aspera-cli 4.13.0 → 4.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +81 -7
  4. data/CONTRIBUTING.md +22 -6
  5. data/README.md +2038 -1080
  6. data/bin/ascli +18 -9
  7. data/bin/asession +12 -14
  8. data/examples/dascli +1 -1
  9. data/examples/proxy.pac +1 -1
  10. data/examples/rubyc +24 -0
  11. data/lib/aspera/aoc.rb +219 -159
  12. data/lib/aspera/ascmd.rb +25 -14
  13. data/lib/aspera/cli/basic_auth_plugin.rb +12 -9
  14. data/lib/aspera/cli/error.rb +17 -0
  15. data/lib/aspera/cli/extended_value.rb +47 -12
  16. data/lib/aspera/cli/formatter.rb +260 -179
  17. data/lib/aspera/cli/hints.rb +80 -0
  18. data/lib/aspera/cli/main.rb +104 -156
  19. data/lib/aspera/cli/manager.rb +259 -209
  20. data/lib/aspera/cli/plugin.rb +123 -63
  21. data/lib/aspera/cli/plugins/alee.rb +2 -3
  22. data/lib/aspera/cli/plugins/aoc.rb +341 -261
  23. data/lib/aspera/cli/plugins/ats.rb +22 -21
  24. data/lib/aspera/cli/plugins/bss.rb +5 -5
  25. data/lib/aspera/cli/plugins/config.rb +578 -627
  26. data/lib/aspera/cli/plugins/console.rb +44 -6
  27. data/lib/aspera/cli/plugins/cos.rb +15 -17
  28. data/lib/aspera/cli/plugins/faspex.rb +114 -100
  29. data/lib/aspera/cli/plugins/faspex5.rb +411 -264
  30. data/lib/aspera/cli/plugins/node.rb +354 -259
  31. data/lib/aspera/cli/plugins/orchestrator.rb +61 -29
  32. data/lib/aspera/cli/plugins/preview.rb +82 -90
  33. data/lib/aspera/cli/plugins/server.rb +79 -32
  34. data/lib/aspera/cli/plugins/shares.rb +55 -42
  35. data/lib/aspera/cli/sync_actions.rb +68 -0
  36. data/lib/aspera/cli/transfer_agent.rb +66 -73
  37. data/lib/aspera/cli/transfer_progress.rb +74 -0
  38. data/lib/aspera/cli/version.rb +1 -1
  39. data/lib/aspera/colors.rb +12 -8
  40. data/lib/aspera/command_line_builder.rb +14 -11
  41. data/lib/aspera/cos_node.rb +3 -2
  42. data/lib/aspera/data/6 +0 -0
  43. data/lib/aspera/environment.rb +24 -9
  44. data/lib/aspera/fasp/agent_aspera.rb +126 -0
  45. data/lib/aspera/fasp/agent_base.rb +31 -77
  46. data/lib/aspera/fasp/agent_connect.rb +25 -21
  47. data/lib/aspera/fasp/agent_direct.rb +89 -103
  48. data/lib/aspera/fasp/agent_httpgw.rb +231 -149
  49. data/lib/aspera/fasp/agent_node.rb +41 -34
  50. data/lib/aspera/fasp/agent_trsdk.rb +75 -32
  51. data/lib/aspera/fasp/error_info.rb +4 -2
  52. data/lib/aspera/fasp/faux_file.rb +52 -0
  53. data/lib/aspera/fasp/installation.rb +53 -195
  54. data/lib/aspera/fasp/management.rb +244 -0
  55. data/lib/aspera/fasp/parameters.rb +71 -37
  56. data/lib/aspera/fasp/parameters.yaml +76 -8
  57. data/lib/aspera/fasp/products.rb +162 -0
  58. data/lib/aspera/fasp/resume_policy.rb +3 -3
  59. data/lib/aspera/fasp/transfer_spec.rb +7 -6
  60. data/lib/aspera/fasp/uri.rb +26 -24
  61. data/lib/aspera/faspex_gw.rb +2 -2
  62. data/lib/aspera/faspex_postproc.rb +2 -2
  63. data/lib/aspera/hash_ext.rb +14 -4
  64. data/lib/aspera/json_rpc.rb +49 -0
  65. data/lib/aspera/keychain/macos_security.rb +13 -13
  66. data/lib/aspera/line_logger.rb +23 -0
  67. data/lib/aspera/log.rb +58 -16
  68. data/lib/aspera/node.rb +157 -92
  69. data/lib/aspera/oauth.rb +37 -19
  70. data/lib/aspera/open_application.rb +4 -4
  71. data/lib/aspera/persistency_action_once.rb +1 -1
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/file_types.rb +4 -2
  74. data/lib/aspera/preview/generator.rb +22 -35
  75. data/lib/aspera/preview/options.rb +2 -0
  76. data/lib/aspera/preview/terminal.rb +73 -16
  77. data/lib/aspera/preview/utils.rb +21 -28
  78. data/lib/aspera/proxy_auto_config.js +2 -2
  79. data/lib/aspera/rest.rb +136 -68
  80. data/lib/aspera/rest_call_error.rb +1 -1
  81. data/lib/aspera/rest_error_analyzer.rb +15 -14
  82. data/lib/aspera/rest_errors_aspera.rb +37 -34
  83. data/lib/aspera/secret_hider.rb +18 -15
  84. data/lib/aspera/ssh.rb +5 -2
  85. data/lib/aspera/sync.rb +127 -119
  86. data/lib/aspera/temp_file_manager.rb +10 -3
  87. data/lib/aspera/web_auth.rb +10 -7
  88. data/lib/aspera/web_server_simple.rb +9 -4
  89. data.tar.gz.sig +0 -0
  90. metadata +34 -17
  91. metadata.gz.sig +0 -0
  92. data/docs/test_env.conf +0 -186
  93. data/lib/aspera/cli/listener/line_dump.rb +0 -19
  94. data/lib/aspera/cli/listener/logger.rb +0 -22
  95. data/lib/aspera/cli/listener/progress.rb +0 -50
  96. data/lib/aspera/cli/listener/progress_multi.rb +0 -84
  97. data/lib/aspera/cli/plugins/sync.rb +0 -44
  98. data/lib/aspera/data/7 +0 -0
  99. data/lib/aspera/fasp/listener.rb +0 -13
@@ -3,11 +3,11 @@
3
3
  # https://github.com/fastlane-community/security
4
4
  require 'aspera/cli/info'
5
5
 
6
- # enhance the gem to support other keychains
6
+ # enhance the gem to support other key chains
7
7
  module Aspera
8
8
  module Keychain
9
9
  module MacosSecurity
10
- # keychain based on macOS keychain, using `security` cmmand line
10
+ # keychain based on macOS keychain, using `security` command line
11
11
  class Keychain
12
12
  DOMAINS = %i[user system common dynamic].freeze
13
13
  LIST_OPTIONS = {
@@ -32,12 +32,12 @@ module Aspera
32
32
  getpass: :g
33
33
  }.freeze
34
34
  class << self
35
- def execute(command, options=nil, supported=nil, lastopt=nil)
35
+ def execute(command, options=nil, supported=nil, last_opt=nil)
36
36
  url = options&.delete(:url)
37
37
  if !url.nil?
38
38
  uri = URI.parse(url)
39
39
  raise 'only https' unless uri.scheme.eql?('https')
40
- options[:protocol] = 'htps'
40
+ options[:protocol] = 'htps' # cspell: disable-line
41
41
  raise 'host required in URL' if uri.host.nil?
42
42
  options[:server] = uri.host
43
43
  options[:path] = uri.path unless ['', '/'].include?(uri.path)
@@ -50,28 +50,28 @@ module Aspera
50
50
  cmd.push("-#{supported[k]}")
51
51
  cmd.push(v.shellescape) unless v.empty?
52
52
  end
53
- cmd.push(lastopt) unless lastopt.nil?
53
+ cmd.push(last_opt) unless last_opt.nil?
54
54
  Log.log.debug{"executing>>#{cmd.join(' ')}"}
55
55
  result = %x(#{cmd.join(' ')} 2>&1)
56
56
  Log.log.debug{"result>>[#{result}]"}
57
57
  return result
58
58
  end
59
59
 
60
- def keychains(output)
60
+ def key_chains(output)
61
61
  output.split("\n").collect { |line| new(line.strip.gsub(/^"|"$/, '')) }
62
62
  end
63
63
 
64
64
  def default
65
- keychains(execute('default-keychain')).first
65
+ key_chains(execute('default-keychain')).first
66
66
  end
67
67
 
68
68
  def login
69
- keychains(execute('login-keychain')).first
69
+ key_chains(execute('login-keychain')).first
70
70
  end
71
71
 
72
72
  def list(options={})
73
73
  raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
74
- keychains(execute('list-keychains', options, LIST_OPTIONS))
74
+ key_chains(execute('list-key_chains', options, LIST_OPTIONS))
75
75
  end
76
76
 
77
77
  def by_name(name)
@@ -88,14 +88,14 @@ module Aspera
88
88
  [string].pack('H*').force_encoding('UTF-8')
89
89
  end
90
90
 
91
- def password(operation, passtype, options)
91
+ def password(operation, pass_type, options)
92
92
  raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
93
- raise "wrong passtype: #{passtype}" unless %i[generic internet].include?(passtype)
93
+ raise "wrong pass_type: #{pass_type}" unless %i[generic internet].include?(pass_type)
94
94
  raise 'options shall be Hash' unless options.is_a?(Hash)
95
95
  missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
96
96
  raise "missing options: #{missing}" unless missing.empty?
97
97
  options[:getpass] = '' if operation.eql?(:find)
98
- output = self.class.execute("#{operation}-#{passtype}-password", options, ADD_PASS_OPTIONS, @path)
98
+ output = self.class.execute("#{operation}-#{pass_type}-password", options, ADD_PASS_OPTIONS, @path)
99
99
  raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
100
100
  return nil unless operation.eql?(:find)
101
101
  attributes = {}
@@ -143,7 +143,7 @@ module Aspera
143
143
  raise 'not found' if info.nil?
144
144
  result = options.clone
145
145
  result[:secret] = info['password']
146
- result[:description] = info['icmt']
146
+ result[:description] = info['icmt'] # cspell: disable-line
147
147
  return result
148
148
  end
149
149
 
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/log'
4
+
5
+ module Aspera
6
+ # used for logging http
7
+ class LineLogger
8
+ def initialize(level)
9
+ @level = level
10
+ @buffer = []
11
+ end
12
+
13
+ def <<(string)
14
+ return if string.nil? || string.empty?
15
+ if !string.end_with?("\n")
16
+ @buffer.push(string)
17
+ return
18
+ end
19
+ Log.log.send(@level, @buffer.join('') + string.chomp)
20
+ @buffer.clear
21
+ end
22
+ end
23
+ end
data/lib/aspera/log.rb CHANGED
@@ -2,17 +2,53 @@
2
2
 
3
3
  require 'aspera/colors'
4
4
  require 'aspera/secret_hider'
5
+ require 'aspera/environment'
5
6
  require 'logger'
6
7
  require 'pp'
7
8
  require 'json'
8
9
  require 'singleton'
9
10
 
11
+ # extend Ruby logger with trace levels
12
+ class Logger
13
+ TRACE_MAX = 2
14
+ # add custom level to logger severity
15
+ module Severity
16
+ 1.upto(TRACE_MAX).each { |level| const_set("TRACE#{level}", - level)}
17
+ end
18
+ # quick access to label
19
+ SEVERITY_LABEL = Severity.constants.each_with_object({}) { |name, hash| hash[Severity.const_get(name)] = name}
20
+ def format_severity(severity)
21
+ SEVERITY_LABEL[severity] || 'ANY'
22
+ end
23
+
24
+ # define methods for a given level
25
+ def self.make_methods(str_level) # rubocop:disable Style/ClassMethodsDefinitions
26
+ int_level = ::Logger.const_get(str_level.upcase)
27
+ str_level = str_level.downcase
28
+ Kernel.send('lave'.reverse, <<-EOM, nil, __FILE__, __LINE__ + 1)
29
+ def #{str_level}(message = nil, &block)
30
+ add(#{int_level}, message, &block)
31
+ end
32
+
33
+ def #{str_level}?
34
+ level <= #{int_level}
35
+ end
36
+
37
+ def #{str_level}!
38
+ self.level = #{int_level}
39
+ end
40
+ EOM
41
+ end
42
+ Logger::Severity.constants.each { |severity| make_methods(severity) }
43
+ end
44
+
10
45
  module Aspera
11
46
  # Singleton object for logging
12
47
  class Log
13
48
  include Singleton
14
49
  # where logs are sent to
15
50
  LOG_TYPES = %i[stderr stdout syslog].freeze
51
+ @@format = :json # rubocop:disable Style/ClassVars
16
52
  # class methods
17
53
  class << self
18
54
  # levels are :debug,:info,:warn,:error,fatal,:unknown
@@ -21,24 +57,23 @@ module Aspera
21
57
  # get the logger object of singleton
22
58
  def log; instance.logger; end
23
59
 
24
- # dump object in debug mode
60
+ # dump object suitable for Log.log.debug
25
61
  # @param name string or symbol
26
62
  # @param format either pp or json format
27
- def dump(name, object, format=:json)
28
- log.debug do
29
- result =
30
- case format
31
- when :json
32
- JSON.pretty_generate(object) rescue PP.pp(object, +'')
33
- when :ruby
34
- PP.pp(object, +'')
35
- else
36
- raise 'wrong parameter, expect pp or json'
37
- end
38
- "#{name.to_s.green} (#{format})=\n#{result}"
39
- end
63
+ def dump(name, object)
64
+ result =
65
+ case @@format
66
+ when :json
67
+ JSON.pretty_generate(object) rescue PP.pp(object, +'')
68
+ when :ruby
69
+ PP.pp(object, +'')
70
+ else
71
+ raise 'wrong parameter, expect ruby or json'
72
+ end
73
+ "#{name.to_s.green} (#{@@format})=\n#{result}"
40
74
  end
41
75
 
76
+ # Capture the output of $stderr and log it at debug level
42
77
  def capture_stderr
43
78
  real_stderr = $stderr
44
79
  $stderr = StringIO.new
@@ -69,16 +104,23 @@ module Aspera
69
104
  # change underlying logger, but keep log level
70
105
  def logger_type=(new_log_type)
71
106
  current_severity_integer = @logger.level unless @logger.nil?
72
- current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
107
+ current_severity_integer = ENV.fetch('AS_LOG_LEVEL', nil) if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
73
108
  current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
74
109
  case new_log_type
75
110
  when :stderr
76
- # typed: Logger
77
111
  @logger = Logger.new($stderr)
78
112
  when :stdout
79
113
  @logger = Logger.new($stdout)
80
114
  when :syslog
81
115
  require 'syslog/logger'
116
+ # the syslog class automatically creates methods from the severity names
117
+ # we just need to add the mapping (but syslog lowest is DEBUG)
118
+ 1.upto(Logger::TRACE_MAX).each do |level|
119
+ Syslog::Logger.const_get(:LEVEL_MAP)[Logger.const_get("TRACE#{level}")] = Syslog::LOG_DEBUG
120
+ end
121
+ Logger::Severity.constants.each do |severity|
122
+ Syslog::Logger.make_methods(severity.downcase)
123
+ end
82
124
  @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
83
125
  else
84
126
  raise "unknown log type: #{new_log_type.class} #{new_log_type}"
data/lib/aspera/node.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/cli/error'
3
4
  require 'aspera/fasp/transfer_spec'
4
5
  require 'aspera/rest'
5
6
  require 'aspera/oauth'
@@ -13,13 +14,22 @@ module Aspera
13
14
  class Node < Aspera::Rest
14
15
  # permissions
15
16
  ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
16
- # prefix for ruby code for filter
17
+ # prefix for ruby code for filter (deprecated)
17
18
  MATCH_EXEC_PREFIX = 'exec:'
19
+ MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
18
20
  HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
19
21
  PATH_SEPARATOR = '/'
22
+ TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
23
+ SCOPE_USER = 'user:all'
24
+ SCOPE_ADMIN = 'admin:all'
25
+ SCOPE_PREFIX = 'node.'
26
+ SCOPE_SEPARATOR = ':'
27
+ SIGNATURE_DELIMITER = '==SIGNATURE=='
28
+ BEARER_TOKEN_VALIDITY_DEFAULT = 86400
29
+ BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
20
30
 
21
31
  # register node special token decoder
22
- Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
32
+ Oauth.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
23
33
 
24
34
  # class instance variable, access with accessors on class
25
35
  @use_standard_ports = true
@@ -27,16 +37,79 @@ module Aspera
27
37
  class << self
28
38
  attr_accessor :use_standard_ports
29
39
 
30
- # for access keys: provide expression to match entry in folder
31
- # if no prefix: regex
32
- # if prefix: ruby code
33
- # if expression is nil, then always match
40
+ # For access keys: provide expression to match entry in folder
34
41
  def file_matcher(match_expression)
35
- match_expression ||= "#{MATCH_EXEC_PREFIX}true"
36
- if match_expression.start_with?(MATCH_EXEC_PREFIX)
37
- return Environment.secure_eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}")
42
+ case match_expression
43
+ when Proc then return match_expression
44
+ when Regexp then return ->(f){f['name'].match?(match_expression)}
45
+ when String
46
+ if match_expression.start_with?(MATCH_EXEC_PREFIX)
47
+ code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
48
+ Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
49
+ return Environment.secure_eval(code, __FILE__, __LINE__)
50
+ end
51
+ return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
52
+ when NilClass then return ->(_){true}
53
+ else raise Cli::BadArgument, "Invalid match expression type: #{match_expression.class}"
54
+ end
55
+ end
56
+
57
+ def file_matcher_from_argument(options)
58
+ return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
59
+ end
60
+
61
+ # node API scopes
62
+ def token_scope(access_key, scope)
63
+ return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
64
+ end
65
+
66
+ def decode_scope(scope)
67
+ items = scope.split(SCOPE_SEPARATOR, 2)
68
+ raise "invalid scope: #{scope}" unless items.length.eql?(2)
69
+ raise "invalid scope: #{scope}" unless items[0].start_with?(SCOPE_PREFIX)
70
+ return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
71
+ end
72
+
73
+ # Create an Aspera Node bearer token
74
+ # @param payload [String] JSON payload to be included in the token
75
+ # @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
76
+ def bearer_token(access_key:, payload:, private_key:)
77
+ raise 'payload shall be Hash' unless payload.is_a?(Hash)
78
+ raise 'missing user_id' unless payload.key?('user_id')
79
+ raise 'user_id must be a String' unless payload['user_id'].is_a?(String)
80
+ raise 'user_id must not be empty' if payload['user_id'].empty?
81
+ raise 'private_key shall be OpenSSL::PKey::RSA' unless private_key.is_a?(OpenSSL::PKey::RSA)
82
+ # manage convenience parameters
83
+ expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
84
+ payload.delete('_validity')
85
+ scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
86
+ payload.delete('_scope')
87
+ payload['scope'] ||= token_scope(access_key, scope)
88
+ payload['auth_type'] ||= 'access_key'
89
+ payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
90
+ payload_json = JSON.generate(payload)
91
+ return Base64.strict_encode64(Zlib::Deflate.deflate([
92
+ payload_json,
93
+ SIGNATURE_DELIMITER,
94
+ Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
95
+ ''
96
+ ].join("\n")))
97
+ end
98
+
99
+ def decode_bearer_token(token)
100
+ return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
101
+ end
102
+
103
+ def bearer_headers(bearer_auth, access_key: nil)
104
+ # if username is not provided, use the access key from the token
105
+ if access_key.nil?
106
+ access_key = Aspera::Node.decode_scope(Aspera::Node.decode_bearer_token(Oauth.bearer_extract(bearer_auth))['scope'])[:access_key]
107
+ raise "internal error #{access_key}" if access_key.nil?
38
108
  end
39
- return lambda{|f|f['name'].match(/#{match_expression}/)}
109
+ return {
110
+ Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
111
+ 'Authorization' => bearer_auth
112
+ }
40
113
  end
41
114
  end
42
115
 
@@ -51,6 +124,7 @@ module Aspera
51
124
  # @param params [Hash] Rest parameters
52
125
  # @param app_info [Hash,NilClass] special processing for AoC
53
126
  def initialize(params:, app_info: nil, add_tspec: nil)
127
+ # init Rest
54
128
  super(params)
55
129
  @app_info = app_info
56
130
  # this is added to transfer spec, for instance to add tags (COS)
@@ -84,17 +158,18 @@ module Aspera
84
158
  return nil
85
159
  end
86
160
 
87
- # recursively browse in a folder (with non-recursive method)
161
+ # Recursively browse in a folder (with non-recursive method)
88
162
  # sub folders are processed if the processing method returns true
89
163
  # @param state [Object] state object sent to processing method
90
- # @param method [Symbol] processing method name
91
164
  # @param top_file_id [String] file id to start at (default = access key root file id)
92
165
  # @param top_file_path [String] path of top folder (default = /)
93
- def process_folder_tree(state:, method:, top_file_id:, top_file_path: '/')
166
+ # @param block [Proc] processing method, arguments: entry, path, state
167
+ def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
94
168
  raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
95
- raise "INTERNAL ERROR: Missing method #{method}" unless respond_to?(method)
169
+ raise 'INTERNAL ERROR: Missing block' unless block
170
+ # start at top folder
96
171
  folders_to_explore = [{id: top_file_id, path: top_file_path}]
97
- Log.dump(:folders_to_explore, folders_to_explore)
172
+ Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
98
173
  until folders_to_explore.empty?
99
174
  current_item = folders_to_explore.shift
100
175
  Log.log.debug{"searching #{current_item[:path]}".bg_green}
@@ -106,12 +181,12 @@ module Aspera
106
181
  Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
107
182
  []
108
183
  end
109
- Log.dump(:folder_contents, folder_contents)
184
+ Log.log.debug{Log.dump(:folder_contents, folder_contents)}
110
185
  folder_contents.each do |entry|
111
186
  relative_path = File.join(current_item[:path], entry['name'])
112
- Log.log.debug{"looking #{relative_path}".bg_green}
187
+ Log.log.debug{"process_folder_tree checking #{relative_path}"}
113
188
  # continue only if method returns true
114
- next unless send(method, entry, relative_path, state)
189
+ next unless yield(entry, relative_path, state)
115
190
  # entry type is file, folder or link
116
191
  case entry['type']
117
192
  when 'folder'
@@ -119,85 +194,74 @@ module Aspera
119
194
  when 'link'
120
195
  node_id_to_node(entry['target_node_id'])&.process_folder_tree(
121
196
  state: state,
122
- method: method,
123
197
  top_file_id: entry['target_id'],
124
- top_file_path: relative_path)
198
+ top_file_path: relative_path,
199
+ &block)
125
200
  end
126
201
  end
127
202
  end
128
203
  end # process_folder_tree
129
204
 
130
- # processing method to resolve a file path to id
131
- # @returns true if processing need to continue
132
- def process_resolve_node_path(entry, _path, state)
133
- # stop digging here if not in right path
134
- return false unless entry['name'].eql?(state[:path].first)
135
- # ok it matches, so we remove the match
136
- state[:path].shift
137
- case entry['type']
138
- when 'file'
139
- # file must be terminal
140
- raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
141
- # it's terminal, we found it
142
- state[:result] = {api: self, file_id: entry['id']}
143
- return false
144
- when 'folder'
145
- if state[:path].empty?
146
- # we found it
147
- state[:result] = {api: self, file_id: entry['id']}
148
- return false
149
- end
150
- when 'link'
151
- if state[:path].empty?
152
- # we found it
153
- other_node = node_id_to_node(entry['target_node_id'])
154
- raise 'cannot resolve link' if other_node.nil?
155
- state[:result] = {api: other_node, file_id: entry['target_id']}
156
- return false
157
- end
158
- else
159
- Log.log.warn{"Unknown element type: #{entry['type']}"}
160
- end
161
- # continue to dig folder
162
- return true
163
- end
164
-
165
205
  # Navigate the path from given file id
166
206
  # @param top_file_id [String] id initial file id
167
207
  # @param path [String] file path
168
208
  # @return [Hash] {.api,.file_id}
169
209
  def resolve_api_fid(top_file_id, path)
170
210
  raise 'file id shall be String' unless top_file_id.is_a?(String)
211
+ process_last_link = path.end_with?(PATH_SEPARATOR)
171
212
  path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
172
213
  return {api: self, file_id: top_file_id} if path_elements.empty?
173
214
  resolve_state = {path: path_elements, result: nil}
174
- process_folder_tree(state: resolve_state, method: :process_resolve_node_path, top_file_id: top_file_id)
175
- raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
176
- return resolve_state[:result]
177
- end
178
-
179
- # add entry to list if test block is success
180
- # @return [TrueClass,FalseClass]
181
- def process_find_files(entry, path, state)
182
- begin
183
- # add to result if match filter
184
- state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
185
- # process link
186
- if entry[:type].eql?('link')
187
- other_node = node_id_to_node(entry['target_node_id'])
188
- other_node.process_folder_tree(state: state, method: process_find_files, top_file_id: entry['target_id'], top_file_path: path)
215
+ process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
216
+ # this block is called recursively for each entry in folder
217
+ # stop digging here if not in right path
218
+ next false unless entry['name'].eql?(state[:path].first)
219
+ # ok it matches, so we remove the match
220
+ state[:path].shift
221
+ case entry['type']
222
+ when 'file'
223
+ # file must be terminal
224
+ raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
225
+ # it's terminal, we found it
226
+ state[:result] = {api: self, file_id: entry['id']}
227
+ next false
228
+ when 'folder'
229
+ if state[:path].empty?
230
+ # we found it
231
+ state[:result] = {api: self, file_id: entry['id']}
232
+ next false
233
+ end
234
+ when 'link'
235
+ if state[:path].empty?
236
+ if process_last_link
237
+ # we found it
238
+ other_node = node_id_to_node(entry['target_node_id'])
239
+ raise 'cannot resolve link' if other_node.nil?
240
+ state[:result] = {api: other_node, file_id: entry['target_id']}
241
+ else
242
+ # we found it but we do not process the link
243
+ state[:result] = {api: self, file_id: entry['id']}
244
+ end
245
+ next false
246
+ end
247
+ else
248
+ Log.log.warn{"Unknown element type: #{entry['type']}"}
189
249
  end
190
- rescue StandardError => e
191
- Log.log.error{"#{path}: #{e.message}"}
250
+ # continue to dig folder
251
+ next true
192
252
  end
193
- # process all folders
194
- return true
253
+ raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
254
+ return resolve_state[:result]
195
255
  end
196
256
 
197
257
  def find_files(top_file_id, test_block)
198
258
  Log.log.debug{"find_files: file id=#{top_file_id}"}
199
259
  find_state = {found: [], test_block: test_block}
200
- process_folder_tree(state: find_state, method: :process_find_files, top_file_id: top_file_id)
260
+ process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
261
+ state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
262
+ # test all files deeply
263
+ true
264
+ end
201
265
  return find_state[:found]
202
266
  end
203
267
 
@@ -212,6 +276,8 @@ module Aspera
212
276
  case params[:auth][:type]
213
277
  when :basic
214
278
  ak_name = params[:auth][:username]
279
+ raise 'ERROR: no secret in node object' unless params[:auth][:password]
280
+ ak_token = Rest.basic_token(params[:auth][:username], params[:auth][:password])
215
281
  when :oauth2
216
282
  ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
217
283
  # TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
@@ -235,39 +301,38 @@ module Aspera
235
301
  add_tspec_info(transfer_spec)
236
302
  transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
237
303
  # add application specific tags (AoC)
238
- the_app = app_info
239
- the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
240
- # add basic token
241
- if transfer_spec['token'].nil?
242
- ts_basic_token(transfer_spec)
243
- end
304
+ app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
244
305
  # add remote host info
245
306
  if self.class.use_standard_ports
246
307
  # get default TCP/UDP ports and transfer user
247
308
  transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
248
309
  # by default: same address as node API
249
310
  transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
311
+ # AoC allows specification of other url
250
312
  if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
251
313
  transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
252
314
  end
315
+ info = read('info')[:data]
316
+ # get the transfer user from info on access key
317
+ transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
318
+ # get settings from name.value array to hash key.value
319
+ settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
320
+ # check WSS ports
321
+ %w[wss_enabled wss_port].each do |i|
322
+ transfer_spec[i] = settings[i] if settings.key?(i)
323
+ end if settings.is_a?(Hash)
253
324
  else
254
- # retrieve values from API
255
- std_t_spec = create(
325
+ # retrieve values from API (and keep a copy/cache)
326
+ @std_t_spec_cache ||= create(
256
327
  'files/download_setup',
257
328
  {transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
258
329
  )[:data]['transfer_specs'].first['transfer_spec']
259
330
  # copy some parts
260
- %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].each {|i| transfer_spec[i] = std_t_spec[i] if std_t_spec.key?(i)}
331
+ TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
261
332
  end
333
+ Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
334
+ unless transfer_spec['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
262
335
  return transfer_spec
263
336
  end
264
-
265
- # set basic token in transfer spec
266
- def ts_basic_token(ts)
267
- Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
268
- unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
269
- raise 'ERROR: no secret in node object' unless params[:auth][:password]
270
- ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
271
- end
272
337
  end
273
338
  end