aspera-cli 4.13.0 → 4.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +81 -7
  4. data/CONTRIBUTING.md +22 -6
  5. data/README.md +2038 -1080
  6. data/bin/ascli +18 -9
  7. data/bin/asession +12 -14
  8. data/examples/dascli +1 -1
  9. data/examples/proxy.pac +1 -1
  10. data/examples/rubyc +24 -0
  11. data/lib/aspera/aoc.rb +219 -159
  12. data/lib/aspera/ascmd.rb +25 -14
  13. data/lib/aspera/cli/basic_auth_plugin.rb +12 -9
  14. data/lib/aspera/cli/error.rb +17 -0
  15. data/lib/aspera/cli/extended_value.rb +47 -12
  16. data/lib/aspera/cli/formatter.rb +260 -179
  17. data/lib/aspera/cli/hints.rb +80 -0
  18. data/lib/aspera/cli/main.rb +104 -156
  19. data/lib/aspera/cli/manager.rb +259 -209
  20. data/lib/aspera/cli/plugin.rb +123 -63
  21. data/lib/aspera/cli/plugins/alee.rb +2 -3
  22. data/lib/aspera/cli/plugins/aoc.rb +341 -261
  23. data/lib/aspera/cli/plugins/ats.rb +22 -21
  24. data/lib/aspera/cli/plugins/bss.rb +5 -5
  25. data/lib/aspera/cli/plugins/config.rb +578 -627
  26. data/lib/aspera/cli/plugins/console.rb +44 -6
  27. data/lib/aspera/cli/plugins/cos.rb +15 -17
  28. data/lib/aspera/cli/plugins/faspex.rb +114 -100
  29. data/lib/aspera/cli/plugins/faspex5.rb +411 -264
  30. data/lib/aspera/cli/plugins/node.rb +354 -259
  31. data/lib/aspera/cli/plugins/orchestrator.rb +61 -29
  32. data/lib/aspera/cli/plugins/preview.rb +82 -90
  33. data/lib/aspera/cli/plugins/server.rb +79 -32
  34. data/lib/aspera/cli/plugins/shares.rb +55 -42
  35. data/lib/aspera/cli/sync_actions.rb +68 -0
  36. data/lib/aspera/cli/transfer_agent.rb +66 -73
  37. data/lib/aspera/cli/transfer_progress.rb +74 -0
  38. data/lib/aspera/cli/version.rb +1 -1
  39. data/lib/aspera/colors.rb +12 -8
  40. data/lib/aspera/command_line_builder.rb +14 -11
  41. data/lib/aspera/cos_node.rb +3 -2
  42. data/lib/aspera/data/6 +0 -0
  43. data/lib/aspera/environment.rb +24 -9
  44. data/lib/aspera/fasp/agent_aspera.rb +126 -0
  45. data/lib/aspera/fasp/agent_base.rb +31 -77
  46. data/lib/aspera/fasp/agent_connect.rb +25 -21
  47. data/lib/aspera/fasp/agent_direct.rb +89 -103
  48. data/lib/aspera/fasp/agent_httpgw.rb +231 -149
  49. data/lib/aspera/fasp/agent_node.rb +41 -34
  50. data/lib/aspera/fasp/agent_trsdk.rb +75 -32
  51. data/lib/aspera/fasp/error_info.rb +4 -2
  52. data/lib/aspera/fasp/faux_file.rb +52 -0
  53. data/lib/aspera/fasp/installation.rb +53 -195
  54. data/lib/aspera/fasp/management.rb +244 -0
  55. data/lib/aspera/fasp/parameters.rb +71 -37
  56. data/lib/aspera/fasp/parameters.yaml +76 -8
  57. data/lib/aspera/fasp/products.rb +162 -0
  58. data/lib/aspera/fasp/resume_policy.rb +3 -3
  59. data/lib/aspera/fasp/transfer_spec.rb +7 -6
  60. data/lib/aspera/fasp/uri.rb +26 -24
  61. data/lib/aspera/faspex_gw.rb +2 -2
  62. data/lib/aspera/faspex_postproc.rb +2 -2
  63. data/lib/aspera/hash_ext.rb +14 -4
  64. data/lib/aspera/json_rpc.rb +49 -0
  65. data/lib/aspera/keychain/macos_security.rb +13 -13
  66. data/lib/aspera/line_logger.rb +23 -0
  67. data/lib/aspera/log.rb +58 -16
  68. data/lib/aspera/node.rb +157 -92
  69. data/lib/aspera/oauth.rb +37 -19
  70. data/lib/aspera/open_application.rb +4 -4
  71. data/lib/aspera/persistency_action_once.rb +1 -1
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/file_types.rb +4 -2
  74. data/lib/aspera/preview/generator.rb +22 -35
  75. data/lib/aspera/preview/options.rb +2 -0
  76. data/lib/aspera/preview/terminal.rb +73 -16
  77. data/lib/aspera/preview/utils.rb +21 -28
  78. data/lib/aspera/proxy_auto_config.js +2 -2
  79. data/lib/aspera/rest.rb +136 -68
  80. data/lib/aspera/rest_call_error.rb +1 -1
  81. data/lib/aspera/rest_error_analyzer.rb +15 -14
  82. data/lib/aspera/rest_errors_aspera.rb +37 -34
  83. data/lib/aspera/secret_hider.rb +18 -15
  84. data/lib/aspera/ssh.rb +5 -2
  85. data/lib/aspera/sync.rb +127 -119
  86. data/lib/aspera/temp_file_manager.rb +10 -3
  87. data/lib/aspera/web_auth.rb +10 -7
  88. data/lib/aspera/web_server_simple.rb +9 -4
  89. data.tar.gz.sig +0 -0
  90. metadata +34 -17
  91. metadata.gz.sig +0 -0
  92. data/docs/test_env.conf +0 -186
  93. data/lib/aspera/cli/listener/line_dump.rb +0 -19
  94. data/lib/aspera/cli/listener/logger.rb +0 -22
  95. data/lib/aspera/cli/listener/progress.rb +0 -50
  96. data/lib/aspera/cli/listener/progress_multi.rb +0 -84
  97. data/lib/aspera/cli/plugins/sync.rb +0 -44
  98. data/lib/aspera/data/7 +0 -0
  99. data/lib/aspera/fasp/listener.rb +0 -13
@@ -3,11 +3,11 @@
3
3
  # https://github.com/fastlane-community/security
4
4
  require 'aspera/cli/info'
5
5
 
6
- # enhance the gem to support other keychains
6
+ # enhance the gem to support other key chains
7
7
  module Aspera
8
8
  module Keychain
9
9
  module MacosSecurity
10
- # keychain based on macOS keychain, using `security` cmmand line
10
+ # keychain based on macOS keychain, using `security` command line
11
11
  class Keychain
12
12
  DOMAINS = %i[user system common dynamic].freeze
13
13
  LIST_OPTIONS = {
@@ -32,12 +32,12 @@ module Aspera
32
32
  getpass: :g
33
33
  }.freeze
34
34
  class << self
35
- def execute(command, options=nil, supported=nil, lastopt=nil)
35
+ def execute(command, options=nil, supported=nil, last_opt=nil)
36
36
  url = options&.delete(:url)
37
37
  if !url.nil?
38
38
  uri = URI.parse(url)
39
39
  raise 'only https' unless uri.scheme.eql?('https')
40
- options[:protocol] = 'htps'
40
+ options[:protocol] = 'htps' # cspell: disable-line
41
41
  raise 'host required in URL' if uri.host.nil?
42
42
  options[:server] = uri.host
43
43
  options[:path] = uri.path unless ['', '/'].include?(uri.path)
@@ -50,28 +50,28 @@ module Aspera
50
50
  cmd.push("-#{supported[k]}")
51
51
  cmd.push(v.shellescape) unless v.empty?
52
52
  end
53
- cmd.push(lastopt) unless lastopt.nil?
53
+ cmd.push(last_opt) unless last_opt.nil?
54
54
  Log.log.debug{"executing>>#{cmd.join(' ')}"}
55
55
  result = %x(#{cmd.join(' ')} 2>&1)
56
56
  Log.log.debug{"result>>[#{result}]"}
57
57
  return result
58
58
  end
59
59
 
60
- def keychains(output)
60
+ def key_chains(output)
61
61
  output.split("\n").collect { |line| new(line.strip.gsub(/^"|"$/, '')) }
62
62
  end
63
63
 
64
64
  def default
65
- keychains(execute('default-keychain')).first
65
+ key_chains(execute('default-keychain')).first
66
66
  end
67
67
 
68
68
  def login
69
- keychains(execute('login-keychain')).first
69
+ key_chains(execute('login-keychain')).first
70
70
  end
71
71
 
72
72
  def list(options={})
73
73
  raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
74
- keychains(execute('list-keychains', options, LIST_OPTIONS))
74
+ key_chains(execute('list-key_chains', options, LIST_OPTIONS))
75
75
  end
76
76
 
77
77
  def by_name(name)
@@ -88,14 +88,14 @@ module Aspera
88
88
  [string].pack('H*').force_encoding('UTF-8')
89
89
  end
90
90
 
91
- def password(operation, passtype, options)
91
+ def password(operation, pass_type, options)
92
92
  raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
93
- raise "wrong passtype: #{passtype}" unless %i[generic internet].include?(passtype)
93
+ raise "wrong pass_type: #{pass_type}" unless %i[generic internet].include?(pass_type)
94
94
  raise 'options shall be Hash' unless options.is_a?(Hash)
95
95
  missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
96
96
  raise "missing options: #{missing}" unless missing.empty?
97
97
  options[:getpass] = '' if operation.eql?(:find)
98
- output = self.class.execute("#{operation}-#{passtype}-password", options, ADD_PASS_OPTIONS, @path)
98
+ output = self.class.execute("#{operation}-#{pass_type}-password", options, ADD_PASS_OPTIONS, @path)
99
99
  raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
100
100
  return nil unless operation.eql?(:find)
101
101
  attributes = {}
@@ -143,7 +143,7 @@ module Aspera
143
143
  raise 'not found' if info.nil?
144
144
  result = options.clone
145
145
  result[:secret] = info['password']
146
- result[:description] = info['icmt']
146
+ result[:description] = info['icmt'] # cspell: disable-line
147
147
  return result
148
148
  end
149
149
 
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/log'
4
+
5
+ module Aspera
6
+ # used for logging http
7
+ class LineLogger
8
+ def initialize(level)
9
+ @level = level
10
+ @buffer = []
11
+ end
12
+
13
+ def <<(string)
14
+ return if string.nil? || string.empty?
15
+ if !string.end_with?("\n")
16
+ @buffer.push(string)
17
+ return
18
+ end
19
+ Log.log.send(@level, @buffer.join('') + string.chomp)
20
+ @buffer.clear
21
+ end
22
+ end
23
+ end
data/lib/aspera/log.rb CHANGED
@@ -2,17 +2,53 @@
2
2
 
3
3
  require 'aspera/colors'
4
4
  require 'aspera/secret_hider'
5
+ require 'aspera/environment'
5
6
  require 'logger'
6
7
  require 'pp'
7
8
  require 'json'
8
9
  require 'singleton'
9
10
 
11
+ # extend Ruby logger with trace levels
12
+ class Logger
13
+ TRACE_MAX = 2
14
+ # add custom level to logger severity
15
+ module Severity
16
+ 1.upto(TRACE_MAX).each { |level| const_set("TRACE#{level}", - level)}
17
+ end
18
+ # quick access to label
19
+ SEVERITY_LABEL = Severity.constants.each_with_object({}) { |name, hash| hash[Severity.const_get(name)] = name}
20
+ def format_severity(severity)
21
+ SEVERITY_LABEL[severity] || 'ANY'
22
+ end
23
+
24
+ # define methods for a given level
25
+ def self.make_methods(str_level) # rubocop:disable Style/ClassMethodsDefinitions
26
+ int_level = ::Logger.const_get(str_level.upcase)
27
+ str_level = str_level.downcase
28
+ Kernel.send('lave'.reverse, <<-EOM, nil, __FILE__, __LINE__ + 1)
29
+ def #{str_level}(message = nil, &block)
30
+ add(#{int_level}, message, &block)
31
+ end
32
+
33
+ def #{str_level}?
34
+ level <= #{int_level}
35
+ end
36
+
37
+ def #{str_level}!
38
+ self.level = #{int_level}
39
+ end
40
+ EOM
41
+ end
42
+ Logger::Severity.constants.each { |severity| make_methods(severity) }
43
+ end
44
+
10
45
  module Aspera
11
46
  # Singleton object for logging
12
47
  class Log
13
48
  include Singleton
14
49
  # where logs are sent to
15
50
  LOG_TYPES = %i[stderr stdout syslog].freeze
51
+ @@format = :json # rubocop:disable Style/ClassVars
16
52
  # class methods
17
53
  class << self
18
54
  # levels are :debug,:info,:warn,:error,fatal,:unknown
@@ -21,24 +57,23 @@ module Aspera
21
57
  # get the logger object of singleton
22
58
  def log; instance.logger; end
23
59
 
24
- # dump object in debug mode
60
+ # dump object suitable for Log.log.debug
25
61
  # @param name string or symbol
26
62
  # @param format either pp or json format
27
- def dump(name, object, format=:json)
28
- log.debug do
29
- result =
30
- case format
31
- when :json
32
- JSON.pretty_generate(object) rescue PP.pp(object, +'')
33
- when :ruby
34
- PP.pp(object, +'')
35
- else
36
- raise 'wrong parameter, expect pp or json'
37
- end
38
- "#{name.to_s.green} (#{format})=\n#{result}"
39
- end
63
+ def dump(name, object)
64
+ result =
65
+ case @@format
66
+ when :json
67
+ JSON.pretty_generate(object) rescue PP.pp(object, +'')
68
+ when :ruby
69
+ PP.pp(object, +'')
70
+ else
71
+ raise 'wrong parameter, expect ruby or json'
72
+ end
73
+ "#{name.to_s.green} (#{@@format})=\n#{result}"
40
74
  end
41
75
 
76
+ # Capture the output of $stderr and log it at debug level
42
77
  def capture_stderr
43
78
  real_stderr = $stderr
44
79
  $stderr = StringIO.new
@@ -69,16 +104,23 @@ module Aspera
69
104
  # change underlying logger, but keep log level
70
105
  def logger_type=(new_log_type)
71
106
  current_severity_integer = @logger.level unless @logger.nil?
72
- current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
107
+ current_severity_integer = ENV.fetch('AS_LOG_LEVEL', nil) if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
73
108
  current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
74
109
  case new_log_type
75
110
  when :stderr
76
- # typed: Logger
77
111
  @logger = Logger.new($stderr)
78
112
  when :stdout
79
113
  @logger = Logger.new($stdout)
80
114
  when :syslog
81
115
  require 'syslog/logger'
116
+ # the syslog class automatically creates methods from the severity names
117
+ # we just need to add the mapping (but syslog lowest is DEBUG)
118
+ 1.upto(Logger::TRACE_MAX).each do |level|
119
+ Syslog::Logger.const_get(:LEVEL_MAP)[Logger.const_get("TRACE#{level}")] = Syslog::LOG_DEBUG
120
+ end
121
+ Logger::Severity.constants.each do |severity|
122
+ Syslog::Logger.make_methods(severity.downcase)
123
+ end
82
124
  @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
83
125
  else
84
126
  raise "unknown log type: #{new_log_type.class} #{new_log_type}"
data/lib/aspera/node.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/cli/error'
3
4
  require 'aspera/fasp/transfer_spec'
4
5
  require 'aspera/rest'
5
6
  require 'aspera/oauth'
@@ -13,13 +14,22 @@ module Aspera
13
14
  class Node < Aspera::Rest
14
15
  # permissions
15
16
  ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
16
- # prefix for ruby code for filter
17
+ # prefix for ruby code for filter (deprecated)
17
18
  MATCH_EXEC_PREFIX = 'exec:'
19
+ MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
18
20
  HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
19
21
  PATH_SEPARATOR = '/'
22
+ TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
23
+ SCOPE_USER = 'user:all'
24
+ SCOPE_ADMIN = 'admin:all'
25
+ SCOPE_PREFIX = 'node.'
26
+ SCOPE_SEPARATOR = ':'
27
+ SIGNATURE_DELIMITER = '==SIGNATURE=='
28
+ BEARER_TOKEN_VALIDITY_DEFAULT = 86400
29
+ BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
20
30
 
21
31
  # register node special token decoder
22
- Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
32
+ Oauth.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
23
33
 
24
34
  # class instance variable, access with accessors on class
25
35
  @use_standard_ports = true
@@ -27,16 +37,79 @@ module Aspera
27
37
  class << self
28
38
  attr_accessor :use_standard_ports
29
39
 
30
- # for access keys: provide expression to match entry in folder
31
- # if no prefix: regex
32
- # if prefix: ruby code
33
- # if expression is nil, then always match
40
+ # For access keys: provide expression to match entry in folder
34
41
  def file_matcher(match_expression)
35
- match_expression ||= "#{MATCH_EXEC_PREFIX}true"
36
- if match_expression.start_with?(MATCH_EXEC_PREFIX)
37
- return Environment.secure_eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}")
42
+ case match_expression
43
+ when Proc then return match_expression
44
+ when Regexp then return ->(f){f['name'].match?(match_expression)}
45
+ when String
46
+ if match_expression.start_with?(MATCH_EXEC_PREFIX)
47
+ code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
48
+ Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
49
+ return Environment.secure_eval(code, __FILE__, __LINE__)
50
+ end
51
+ return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
52
+ when NilClass then return ->(_){true}
53
+ else raise Cli::BadArgument, "Invalid match expression type: #{match_expression.class}"
54
+ end
55
+ end
56
+
57
+ def file_matcher_from_argument(options)
58
+ return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
59
+ end
60
+
61
+ # node API scopes
62
+ def token_scope(access_key, scope)
63
+ return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
64
+ end
65
+
66
+ def decode_scope(scope)
67
+ items = scope.split(SCOPE_SEPARATOR, 2)
68
+ raise "invalid scope: #{scope}" unless items.length.eql?(2)
69
+ raise "invalid scope: #{scope}" unless items[0].start_with?(SCOPE_PREFIX)
70
+ return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
71
+ end
72
+
73
+ # Create an Aspera Node bearer token
74
+ # @param payload [String] JSON payload to be included in the token
75
+ # @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
76
+ def bearer_token(access_key:, payload:, private_key:)
77
+ raise 'payload shall be Hash' unless payload.is_a?(Hash)
78
+ raise 'missing user_id' unless payload.key?('user_id')
79
+ raise 'user_id must be a String' unless payload['user_id'].is_a?(String)
80
+ raise 'user_id must not be empty' if payload['user_id'].empty?
81
+ raise 'private_key shall be OpenSSL::PKey::RSA' unless private_key.is_a?(OpenSSL::PKey::RSA)
82
+ # manage convenience parameters
83
+ expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
84
+ payload.delete('_validity')
85
+ scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
86
+ payload.delete('_scope')
87
+ payload['scope'] ||= token_scope(access_key, scope)
88
+ payload['auth_type'] ||= 'access_key'
89
+ payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
90
+ payload_json = JSON.generate(payload)
91
+ return Base64.strict_encode64(Zlib::Deflate.deflate([
92
+ payload_json,
93
+ SIGNATURE_DELIMITER,
94
+ Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
95
+ ''
96
+ ].join("\n")))
97
+ end
98
+
99
+ def decode_bearer_token(token)
100
+ return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
101
+ end
102
+
103
+ def bearer_headers(bearer_auth, access_key: nil)
104
+ # if username is not provided, use the access key from the token
105
+ if access_key.nil?
106
+ access_key = Aspera::Node.decode_scope(Aspera::Node.decode_bearer_token(Oauth.bearer_extract(bearer_auth))['scope'])[:access_key]
107
+ raise "internal error #{access_key}" if access_key.nil?
38
108
  end
39
- return lambda{|f|f['name'].match(/#{match_expression}/)}
109
+ return {
110
+ Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
111
+ 'Authorization' => bearer_auth
112
+ }
40
113
  end
41
114
  end
42
115
 
@@ -51,6 +124,7 @@ module Aspera
51
124
  # @param params [Hash] Rest parameters
52
125
  # @param app_info [Hash,NilClass] special processing for AoC
53
126
  def initialize(params:, app_info: nil, add_tspec: nil)
127
+ # init Rest
54
128
  super(params)
55
129
  @app_info = app_info
56
130
  # this is added to transfer spec, for instance to add tags (COS)
@@ -84,17 +158,18 @@ module Aspera
84
158
  return nil
85
159
  end
86
160
 
87
- # recursively browse in a folder (with non-recursive method)
161
+ # Recursively browse in a folder (with non-recursive method)
88
162
  # sub folders are processed if the processing method returns true
89
163
  # @param state [Object] state object sent to processing method
90
- # @param method [Symbol] processing method name
91
164
  # @param top_file_id [String] file id to start at (default = access key root file id)
92
165
  # @param top_file_path [String] path of top folder (default = /)
93
- def process_folder_tree(state:, method:, top_file_id:, top_file_path: '/')
166
+ # @param block [Proc] processing method, arguments: entry, path, state
167
+ def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
94
168
  raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
95
- raise "INTERNAL ERROR: Missing method #{method}" unless respond_to?(method)
169
+ raise 'INTERNAL ERROR: Missing block' unless block
170
+ # start at top folder
96
171
  folders_to_explore = [{id: top_file_id, path: top_file_path}]
97
- Log.dump(:folders_to_explore, folders_to_explore)
172
+ Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
98
173
  until folders_to_explore.empty?
99
174
  current_item = folders_to_explore.shift
100
175
  Log.log.debug{"searching #{current_item[:path]}".bg_green}
@@ -106,12 +181,12 @@ module Aspera
106
181
  Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
107
182
  []
108
183
  end
109
- Log.dump(:folder_contents, folder_contents)
184
+ Log.log.debug{Log.dump(:folder_contents, folder_contents)}
110
185
  folder_contents.each do |entry|
111
186
  relative_path = File.join(current_item[:path], entry['name'])
112
- Log.log.debug{"looking #{relative_path}".bg_green}
187
+ Log.log.debug{"process_folder_tree checking #{relative_path}"}
113
188
  # continue only if method returns true
114
- next unless send(method, entry, relative_path, state)
189
+ next unless yield(entry, relative_path, state)
115
190
  # entry type is file, folder or link
116
191
  case entry['type']
117
192
  when 'folder'
@@ -119,85 +194,74 @@ module Aspera
119
194
  when 'link'
120
195
  node_id_to_node(entry['target_node_id'])&.process_folder_tree(
121
196
  state: state,
122
- method: method,
123
197
  top_file_id: entry['target_id'],
124
- top_file_path: relative_path)
198
+ top_file_path: relative_path,
199
+ &block)
125
200
  end
126
201
  end
127
202
  end
128
203
  end # process_folder_tree
129
204
 
130
- # processing method to resolve a file path to id
131
- # @returns true if processing need to continue
132
- def process_resolve_node_path(entry, _path, state)
133
- # stop digging here if not in right path
134
- return false unless entry['name'].eql?(state[:path].first)
135
- # ok it matches, so we remove the match
136
- state[:path].shift
137
- case entry['type']
138
- when 'file'
139
- # file must be terminal
140
- raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
141
- # it's terminal, we found it
142
- state[:result] = {api: self, file_id: entry['id']}
143
- return false
144
- when 'folder'
145
- if state[:path].empty?
146
- # we found it
147
- state[:result] = {api: self, file_id: entry['id']}
148
- return false
149
- end
150
- when 'link'
151
- if state[:path].empty?
152
- # we found it
153
- other_node = node_id_to_node(entry['target_node_id'])
154
- raise 'cannot resolve link' if other_node.nil?
155
- state[:result] = {api: other_node, file_id: entry['target_id']}
156
- return false
157
- end
158
- else
159
- Log.log.warn{"Unknown element type: #{entry['type']}"}
160
- end
161
- # continue to dig folder
162
- return true
163
- end
164
-
165
205
  # Navigate the path from given file id
166
206
  # @param top_file_id [String] id initial file id
167
207
  # @param path [String] file path
168
208
  # @return [Hash] {.api,.file_id}
169
209
  def resolve_api_fid(top_file_id, path)
170
210
  raise 'file id shall be String' unless top_file_id.is_a?(String)
211
+ process_last_link = path.end_with?(PATH_SEPARATOR)
171
212
  path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
172
213
  return {api: self, file_id: top_file_id} if path_elements.empty?
173
214
  resolve_state = {path: path_elements, result: nil}
174
- process_folder_tree(state: resolve_state, method: :process_resolve_node_path, top_file_id: top_file_id)
175
- raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
176
- return resolve_state[:result]
177
- end
178
-
179
- # add entry to list if test block is success
180
- # @return [TrueClass,FalseClass]
181
- def process_find_files(entry, path, state)
182
- begin
183
- # add to result if match filter
184
- state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
185
- # process link
186
- if entry[:type].eql?('link')
187
- other_node = node_id_to_node(entry['target_node_id'])
188
- other_node.process_folder_tree(state: state, method: process_find_files, top_file_id: entry['target_id'], top_file_path: path)
215
+ process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
216
+ # this block is called recursively for each entry in folder
217
+ # stop digging here if not in right path
218
+ next false unless entry['name'].eql?(state[:path].first)
219
+ # ok it matches, so we remove the match
220
+ state[:path].shift
221
+ case entry['type']
222
+ when 'file'
223
+ # file must be terminal
224
+ raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
225
+ # it's terminal, we found it
226
+ state[:result] = {api: self, file_id: entry['id']}
227
+ next false
228
+ when 'folder'
229
+ if state[:path].empty?
230
+ # we found it
231
+ state[:result] = {api: self, file_id: entry['id']}
232
+ next false
233
+ end
234
+ when 'link'
235
+ if state[:path].empty?
236
+ if process_last_link
237
+ # we found it
238
+ other_node = node_id_to_node(entry['target_node_id'])
239
+ raise 'cannot resolve link' if other_node.nil?
240
+ state[:result] = {api: other_node, file_id: entry['target_id']}
241
+ else
242
+ # we found it but we do not process the link
243
+ state[:result] = {api: self, file_id: entry['id']}
244
+ end
245
+ next false
246
+ end
247
+ else
248
+ Log.log.warn{"Unknown element type: #{entry['type']}"}
189
249
  end
190
- rescue StandardError => e
191
- Log.log.error{"#{path}: #{e.message}"}
250
+ # continue to dig folder
251
+ next true
192
252
  end
193
- # process all folders
194
- return true
253
+ raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
254
+ return resolve_state[:result]
195
255
  end
196
256
 
197
257
  def find_files(top_file_id, test_block)
198
258
  Log.log.debug{"find_files: file id=#{top_file_id}"}
199
259
  find_state = {found: [], test_block: test_block}
200
- process_folder_tree(state: find_state, method: :process_find_files, top_file_id: top_file_id)
260
+ process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
261
+ state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
262
+ # test all files deeply
263
+ true
264
+ end
201
265
  return find_state[:found]
202
266
  end
203
267
 
@@ -212,6 +276,8 @@ module Aspera
212
276
  case params[:auth][:type]
213
277
  when :basic
214
278
  ak_name = params[:auth][:username]
279
+ raise 'ERROR: no secret in node object' unless params[:auth][:password]
280
+ ak_token = Rest.basic_token(params[:auth][:username], params[:auth][:password])
215
281
  when :oauth2
216
282
  ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
217
283
  # TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
@@ -235,39 +301,38 @@ module Aspera
235
301
  add_tspec_info(transfer_spec)
236
302
  transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
237
303
  # add application specific tags (AoC)
238
- the_app = app_info
239
- the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
240
- # add basic token
241
- if transfer_spec['token'].nil?
242
- ts_basic_token(transfer_spec)
243
- end
304
+ app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
244
305
  # add remote host info
245
306
  if self.class.use_standard_ports
246
307
  # get default TCP/UDP ports and transfer user
247
308
  transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
248
309
  # by default: same address as node API
249
310
  transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
311
+ # AoC allows specification of other url
250
312
  if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
251
313
  transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
252
314
  end
315
+ info = read('info')[:data]
316
+ # get the transfer user from info on access key
317
+ transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
318
+ # get settings from name.value array to hash key.value
319
+ settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
320
+ # check WSS ports
321
+ %w[wss_enabled wss_port].each do |i|
322
+ transfer_spec[i] = settings[i] if settings.key?(i)
323
+ end if settings.is_a?(Hash)
253
324
  else
254
- # retrieve values from API
255
- std_t_spec = create(
325
+ # retrieve values from API (and keep a copy/cache)
326
+ @std_t_spec_cache ||= create(
256
327
  'files/download_setup',
257
328
  {transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
258
329
  )[:data]['transfer_specs'].first['transfer_spec']
259
330
  # copy some parts
260
- %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].each {|i| transfer_spec[i] = std_t_spec[i] if std_t_spec.key?(i)}
331
+ TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
261
332
  end
333
+ Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
334
+ unless transfer_spec['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
262
335
  return transfer_spec
263
336
  end
264
-
265
- # set basic token in transfer spec
266
- def ts_basic_token(ts)
267
- Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
268
- unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
269
- raise 'ERROR: no secret in node object' unless params[:auth][:password]
270
- ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
271
- end
272
337
  end
273
338
  end