aspera-cli 4.12.0 → 4.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +45 -5
  4. data/CONTRIBUTING.md +113 -22
  5. data/README.md +1289 -754
  6. data/bin/ascli +3 -3
  7. data/examples/dascli +1 -1
  8. data/examples/rubyc +24 -0
  9. data/lib/aspera/aoc.rb +63 -74
  10. data/lib/aspera/ascmd.rb +5 -3
  11. data/lib/aspera/cli/basic_auth_plugin.rb +6 -6
  12. data/lib/aspera/cli/extended_value.rb +24 -37
  13. data/lib/aspera/cli/formatter.rb +23 -25
  14. data/lib/aspera/cli/info.rb +2 -4
  15. data/lib/aspera/cli/main.rb +27 -27
  16. data/lib/aspera/cli/manager.rb +143 -120
  17. data/lib/aspera/cli/plugin.rb +88 -43
  18. data/lib/aspera/cli/plugins/alee.rb +2 -2
  19. data/lib/aspera/cli/plugins/aoc.rb +235 -104
  20. data/lib/aspera/cli/plugins/ats.rb +16 -18
  21. data/lib/aspera/cli/plugins/bss.rb +3 -3
  22. data/lib/aspera/cli/plugins/config.rb +190 -373
  23. data/lib/aspera/cli/plugins/console.rb +4 -6
  24. data/lib/aspera/cli/plugins/cos.rb +12 -13
  25. data/lib/aspera/cli/plugins/faspex.rb +21 -21
  26. data/lib/aspera/cli/plugins/faspex5.rb +399 -150
  27. data/lib/aspera/cli/plugins/node.rb +260 -174
  28. data/lib/aspera/cli/plugins/orchestrator.rb +15 -18
  29. data/lib/aspera/cli/plugins/preview.rb +40 -62
  30. data/lib/aspera/cli/plugins/server.rb +33 -16
  31. data/lib/aspera/cli/plugins/shares.rb +24 -33
  32. data/lib/aspera/cli/plugins/sync.rb +6 -6
  33. data/lib/aspera/cli/transfer_agent.rb +47 -30
  34. data/lib/aspera/cli/version.rb +2 -1
  35. data/lib/aspera/colors.rb +9 -7
  36. data/lib/aspera/command_line_builder.rb +2 -1
  37. data/lib/aspera/cos_node.rb +1 -1
  38. data/lib/aspera/data/6 +0 -0
  39. data/lib/aspera/environment.rb +7 -3
  40. data/lib/aspera/fasp/agent_connect.rb +6 -1
  41. data/lib/aspera/fasp/agent_direct.rb +17 -17
  42. data/lib/aspera/fasp/agent_httpgw.rb +138 -60
  43. data/lib/aspera/fasp/agent_node.rb +14 -4
  44. data/lib/aspera/fasp/agent_trsdk.rb +2 -0
  45. data/lib/aspera/fasp/error_info.rb +2 -0
  46. data/lib/aspera/fasp/installation.rb +19 -19
  47. data/lib/aspera/fasp/parameters.rb +29 -20
  48. data/lib/aspera/fasp/parameters.yaml +5 -2
  49. data/lib/aspera/fasp/resume_policy.rb +3 -3
  50. data/lib/aspera/fasp/transfer_spec.rb +8 -5
  51. data/lib/aspera/fasp/uri.rb +23 -21
  52. data/lib/aspera/faspex_gw.rb +1 -0
  53. data/lib/aspera/faspex_postproc.rb +3 -3
  54. data/lib/aspera/hash_ext.rb +12 -2
  55. data/lib/aspera/keychain/macos_security.rb +13 -13
  56. data/lib/aspera/log.rb +1 -0
  57. data/lib/aspera/node.rb +73 -84
  58. data/lib/aspera/oauth.rb +4 -3
  59. data/lib/aspera/persistency_action_once.rb +1 -1
  60. data/lib/aspera/preview/file_types.rb +8 -6
  61. data/lib/aspera/preview/generator.rb +23 -11
  62. data/lib/aspera/preview/options.rb +3 -2
  63. data/lib/aspera/preview/terminal.rb +80 -0
  64. data/lib/aspera/preview/utils.rb +11 -11
  65. data/lib/aspera/proxy_auto_config.js +2 -2
  66. data/lib/aspera/rest.rb +42 -4
  67. data/lib/aspera/rest_call_error.rb +3 -1
  68. data/lib/aspera/secret_hider.rb +10 -5
  69. data/lib/aspera/ssh.rb +1 -1
  70. data/lib/aspera/sync.rb +41 -33
  71. data/lib/aspera/web_server_simple.rb +22 -18
  72. data.tar.gz.sig +0 -0
  73. metadata +40 -48
  74. metadata.gz.sig +0 -0
  75. data/docs/test_env.conf +0 -179
  76. data/examples/aoc.rb +0 -30
  77. data/examples/faspex4.rb +0 -94
  78. data/examples/node.rb +0 -96
  79. data/examples/server.rb +0 -93
  80. data/lib/aspera/data/7 +0 -0
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # cspell:words httpport targetrate minrate bwcap createpath lockpolicy lockminrate
4
+
3
5
  require 'aspera/log'
4
6
  require 'aspera/command_line_builder'
5
7
 
@@ -7,8 +9,8 @@ module Aspera
7
9
  module Fasp
8
10
  # translates a "faspe:" URI (used in Faspex 4) into transfer spec hash
9
11
  class Uri
10
- def initialize(fasplink)
11
- @fasp_uri = URI.parse(fasplink.gsub(' ', '%20'))
12
+ def initialize(fasp_link)
13
+ @fasp_uri = URI.parse(fasp_link.gsub(' ', '%20'))
12
14
  # TODO: check scheme is faspe
13
15
  end
14
16
 
@@ -18,32 +20,32 @@ module Aspera
18
20
  result_ts['remote_user'] = @fasp_uri.user
19
21
  result_ts['ssh_port'] = @fasp_uri.port
20
22
  result_ts['paths'] = [{'source' => URI.decode_www_form_component(@fasp_uri.path)}]
21
- # faspex does not encode trailing base64 encoded tags, fix that
23
+ # faspex does not encode trailing base64 padding, fix that to be able to decode properly
22
24
  fixed_query = @fasp_uri.query.gsub(/(=+)$/){|x|'%3D' * x.length}
23
25
 
24
26
  URI.decode_www_form(fixed_query).each do |i|
25
27
  name = i[0]
26
28
  value = i[1]
27
29
  case name
28
- when 'cookie' then result_ts['cookie'] = value
29
- when 'token' then result_ts['token'] = value
30
- when 'sshfp' then result_ts['sshfp'] = value
31
- when 'policy' then result_ts['rate_policy'] = value
32
- when 'httpport' then result_ts['http_fallback_port'] = value.to_i
33
- when 'targetrate' then result_ts['target_rate_kbps'] = value.to_i
34
- when 'minrate' then result_ts['min_rate_kbps'] = value.to_i
35
- when 'port' then result_ts['fasp_port'] = value.to_i
36
- when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
37
- when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub(/--/, '-')
38
- when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
39
- when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
40
- when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
41
- when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
30
+ when 'cookie' then result_ts['cookie'] = value
31
+ when 'token' then result_ts['token'] = value
32
+ when 'sshfp' then result_ts['sshfp'] = value
33
+ when 'policy' then result_ts['rate_policy'] = value
34
+ when 'httpport' then result_ts['http_fallback_port'] = value.to_i
35
+ when 'targetrate' then result_ts['target_rate_kbps'] = value.to_i
36
+ when 'minrate' then result_ts['min_rate_kbps'] = value.to_i
37
+ when 'port' then result_ts['fasp_port'] = value.to_i
38
+ when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
39
+ when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub(/--/, '-')
40
+ when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
41
+ when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
42
+ when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
43
+ when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
42
44
  when 'lockminrate' then result_ts['lock_min_rate'] = CommandLineBuilder.yes_to_true(value)
43
- when 'auth' then Log.log.debug{"ignoring auth #{name}=#{value}"} # TODO: translate into transfer spec ? yes/no
44
- when 'v' then Log.log.debug{"ignoring v #{name}=#{value}"} # TODO: translate into transfer spec ? 2
45
- when 'protect' then Log.log.debug{"ignoring protect #{name}=#{value}"} # TODO: translate into transfer spec ?
46
- else Log.log.warn{"URI parameter ignored: #{name} = #{value}"}
45
+ when 'auth' then Log.log.debug{"ignoring #{name}=#{value}"} # Not used (yes/no)
46
+ when 'v' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch Not used (2)
47
+ when 'protect' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch TODO: what is this ?
48
+ else Log.log.warn{"URI parameter ignored: #{name} = #{value}"}
47
49
  end
48
50
  end
49
51
  return result_ts
@@ -84,6 +84,7 @@ module Aspera
84
84
  response['Content-Type'] = 'application/json'
85
85
  response.body = {error: e.message}.to_json
86
86
  Log.log.error(e.message)
87
+ Log.log.debug{e.backtrace.join("\n")}
87
88
  end
88
89
  else
89
90
  response.status = 400
@@ -13,13 +13,13 @@ module Aspera
13
13
  def initialize(server, parameters)
14
14
  raise 'parameters must be Hash' unless parameters.is_a?(Hash)
15
15
  @parameters = parameters.symbolize_keys
16
- Log.dump(:postproc_parameters, @parameters)
16
+ Log.dump(:post_proc_parameters, @parameters)
17
17
  raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
18
18
  @parameters[:script_folder] ||= '.'
19
19
  @parameters[:fail_on_error] ||= false
20
20
  @parameters[:timeout_seconds] ||= 60
21
21
  super(server)
22
- Log.log.debug{"Faspex4PostProcServlet initialized"}
22
+ Log.log.debug{'Faspex4PostProcServlet initialized'}
23
23
  end
24
24
 
25
25
  def do_POST(request, response)
@@ -39,7 +39,7 @@ module Aspera
39
39
  return
40
40
  end
41
41
  # build script path by removing domain, and adding script folder
42
- script_file = request.path[@parameters[:root].size .. ]
42
+ script_file = request.path[@parameters[:root].size..]
43
43
  Log.log.debug{"script file=#{script_file}"}
44
44
  script_path = File.join(@parameters[:script_folder], script_file)
45
45
  Log.log.debug{"script=#{script_path}"}
@@ -2,11 +2,21 @@
2
2
 
3
3
  class ::Hash
4
4
  def deep_merge(second)
5
- merge(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
5
+ merge(second){|_key, v1, v2|v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge(v2) : v2}
6
6
  end
7
7
 
8
8
  def deep_merge!(second)
9
- merge!(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
9
+ merge!(second){|_key, v1, v2|v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge!(v2) : v2}
10
+ end
11
+
12
+ def deep_do(memory=nil, &block)
13
+ each do |key, value|
14
+ if value.is_a?(Hash)
15
+ value.deep_do(memory, &block)
16
+ else
17
+ yield(self, key, value, memory)
18
+ end
19
+ end
10
20
  end
11
21
  end
12
22
 
@@ -3,11 +3,11 @@
3
3
  # https://github.com/fastlane-community/security
4
4
  require 'aspera/cli/info'
5
5
 
6
- # enhance the gem to support other keychains
6
+ # enhance the gem to support other key chains
7
7
  module Aspera
8
8
  module Keychain
9
9
  module MacosSecurity
10
- # keychain based on macOS keychain, using `security` cmmand line
10
+ # keychain based on macOS keychain, using `security` command line
11
11
  class Keychain
12
12
  DOMAINS = %i[user system common dynamic].freeze
13
13
  LIST_OPTIONS = {
@@ -32,12 +32,12 @@ module Aspera
32
32
  getpass: :g
33
33
  }.freeze
34
34
  class << self
35
- def execute(command, options=nil, supported=nil, lastopt=nil)
35
+ def execute(command, options=nil, supported=nil, last_opt=nil)
36
36
  url = options&.delete(:url)
37
37
  if !url.nil?
38
38
  uri = URI.parse(url)
39
39
  raise 'only https' unless uri.scheme.eql?('https')
40
- options[:protocol] = 'htps'
40
+ options[:protocol] = 'htps' # cspell: disable-line
41
41
  raise 'host required in URL' if uri.host.nil?
42
42
  options[:server] = uri.host
43
43
  options[:path] = uri.path unless ['', '/'].include?(uri.path)
@@ -50,28 +50,28 @@ module Aspera
50
50
  cmd.push("-#{supported[k]}")
51
51
  cmd.push(v.shellescape) unless v.empty?
52
52
  end
53
- cmd.push(lastopt) unless lastopt.nil?
53
+ cmd.push(last_opt) unless last_opt.nil?
54
54
  Log.log.debug{"executing>>#{cmd.join(' ')}"}
55
55
  result = %x(#{cmd.join(' ')} 2>&1)
56
56
  Log.log.debug{"result>>[#{result}]"}
57
57
  return result
58
58
  end
59
59
 
60
- def keychains(output)
60
+ def key_chains(output)
61
61
  output.split("\n").collect { |line| new(line.strip.gsub(/^"|"$/, '')) }
62
62
  end
63
63
 
64
64
  def default
65
- keychains(execute('default-keychain')).first
65
+ key_chains(execute('default-keychain')).first
66
66
  end
67
67
 
68
68
  def login
69
- keychains(execute('login-keychain')).first
69
+ key_chains(execute('login-keychain')).first
70
70
  end
71
71
 
72
72
  def list(options={})
73
73
  raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
74
- keychains(execute('list-keychains', options, LIST_OPTIONS))
74
+ key_chains(execute('list-key_chains', options, LIST_OPTIONS))
75
75
  end
76
76
 
77
77
  def by_name(name)
@@ -88,14 +88,14 @@ module Aspera
88
88
  [string].pack('H*').force_encoding('UTF-8')
89
89
  end
90
90
 
91
- def password(operation, passtype, options)
91
+ def password(operation, pass_type, options)
92
92
  raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
93
- raise "wrong passtype: #{passtype}" unless %i[generic internet].include?(passtype)
93
+ raise "wrong pass_type: #{pass_type}" unless %i[generic internet].include?(pass_type)
94
94
  raise 'options shall be Hash' unless options.is_a?(Hash)
95
95
  missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
96
96
  raise "missing options: #{missing}" unless missing.empty?
97
97
  options[:getpass] = '' if operation.eql?(:find)
98
- output = self.class.execute("#{operation}-#{passtype}-password", options, ADD_PASS_OPTIONS, @path)
98
+ output = self.class.execute("#{operation}-#{pass_type}-password", options, ADD_PASS_OPTIONS, @path)
99
99
  raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
100
100
  return nil unless operation.eql?(:find)
101
101
  attributes = {}
@@ -143,7 +143,7 @@ module Aspera
143
143
  raise 'not found' if info.nil?
144
144
  result = options.clone
145
145
  result[:secret] = info['password']
146
- result[:description] = info['icmt']
146
+ result[:description] = info['icmt'] # cspell: disable-line
147
147
  return result
148
148
  end
149
149
 
data/lib/aspera/log.rb CHANGED
@@ -39,6 +39,7 @@ module Aspera
39
39
  end
40
40
  end
41
41
 
42
+ # Capture the output of $stderr and log it at debug level
42
43
  def capture_stderr
43
44
  real_stderr = $stderr
44
45
  $stderr = StringIO.new
data/lib/aspera/node.rb CHANGED
@@ -17,6 +17,7 @@ module Aspera
17
17
  MATCH_EXEC_PREFIX = 'exec:'
18
18
  HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
19
19
  PATH_SEPARATOR = '/'
20
+ TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
20
21
 
21
22
  # register node special token decoder
22
23
  Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
@@ -40,7 +41,9 @@ module Aspera
40
41
  end
41
42
  end
42
43
 
43
- REQUIRED_APP_INFO_FIELDS = %i[node_info app api workspace_info].freeze
44
+ # fields in @app_info
45
+ REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
46
+ # methods of @app_info[:api]
44
47
  REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
45
48
  private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
46
49
 
@@ -71,21 +74,27 @@ module Aspera
71
74
 
72
75
  # @returns [Aspera::Node] a Node or nil
73
76
  def node_id_to_node(node_id)
74
- return self if !@app_info.nil? && @app_info[:node_info]['id'].eql?(node_id)
75
- return @app_info[:api].node_api_from(node_id: node_id, workspace_info: @app_info[workspace_info]) unless @app_info.nil?
77
+ if !@app_info.nil?
78
+ return self if node_id.eql?(@app_info[:node_info]['id'])
79
+ return @app_info[:api].node_api_from(
80
+ node_id: node_id,
81
+ workspace_id: @app_info[:workspace_id],
82
+ workspace_name: @app_info[:workspace_name])
83
+ end
76
84
  Log.log.warn{"cannot resolve link with node id #{node_id}"}
77
85
  return nil
78
86
  end
79
87
 
80
- # recursively browse in a folder (with non-recursive method)
88
+ # Recursively browse in a folder (with non-recursive method)
81
89
  # sub folders are processed if the processing method returns true
82
90
  # @param state [Object] state object sent to processing method
83
- # @param method [Symbol] processing method name
84
91
  # @param top_file_id [String] file id to start at (default = access key root file id)
85
92
  # @param top_file_path [String] path of top folder (default = /)
86
- def process_folder_tree(state:, method:, top_file_id:, top_file_path: '/')
93
+ # @param block [Proc] processing method, args: entry, path, state
94
+ def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
87
95
  raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
88
- raise "INTERNAL ERROR: Missing method #{method}" unless respond_to?(method)
96
+ raise 'INTERNAL ERROR: Missing block' unless block
97
+ # start at top folder
89
98
  folders_to_explore = [{id: top_file_id, path: top_file_path}]
90
99
  Log.dump(:folders_to_explore, folders_to_explore)
91
100
  until folders_to_explore.empty?
@@ -102,9 +111,9 @@ module Aspera
102
111
  Log.dump(:folder_contents, folder_contents)
103
112
  folder_contents.each do |entry|
104
113
  relative_path = File.join(current_item[:path], entry['name'])
105
- Log.log.debug{"looking #{relative_path}".bg_green}
114
+ Log.log.debug{"process_folder_tree checking #{relative_path}"}
106
115
  # continue only if method returns true
107
- next unless send(method, entry, relative_path, state)
116
+ next unless yield(entry, relative_path, state)
108
117
  # entry type is file, folder or link
109
118
  case entry['type']
110
119
  when 'folder'
@@ -112,85 +121,74 @@ module Aspera
112
121
  when 'link'
113
122
  node_id_to_node(entry['target_node_id'])&.process_folder_tree(
114
123
  state: state,
115
- method: method,
116
124
  top_file_id: entry['target_id'],
117
- top_file_path: relative_path)
125
+ top_file_path: relative_path,
126
+ &block)
118
127
  end
119
128
  end
120
129
  end
121
130
  end # process_folder_tree
122
131
 
123
- # processing method to resolve a file path to id
124
- # @returns true if processing need to continue
125
- def process_resolve_node_path(entry, _path, state)
126
- # stop digging here if not in right path
127
- return false unless entry['name'].eql?(state[:path].first)
128
- # ok it matches, so we remove the match
129
- state[:path].shift
130
- case entry['type']
131
- when 'file'
132
- # file must be terminal
133
- raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
134
- # it's terminal, we found it
135
- state[:result] = {api: self, file_id: entry['id']}
136
- return false
137
- when 'folder'
138
- if state[:path].empty?
139
- # we found it
140
- state[:result] = {api: self, file_id: entry['id']}
141
- return false
142
- end
143
- when 'link'
144
- if state[:path].empty?
145
- # we found it
146
- other_node = node_id_to_node(entry['target_node_id'])
147
- raise 'cannot resolve link' if other_node.nil?
148
- state[:result] = {api: other_node, file_id: entry['target_id']}
149
- return false
150
- end
151
- else
152
- Log.log.warn{"Unknown element type: #{entry['type']}"}
153
- end
154
- # continue to dig folder
155
- return true
156
- end
157
-
158
132
  # Navigate the path from given file id
159
133
  # @param top_file_id [String] id initial file id
160
134
  # @param path [String] file path
161
135
  # @return [Hash] {.api,.file_id}
162
136
  def resolve_api_fid(top_file_id, path)
163
137
  raise 'file id shall be String' unless top_file_id.is_a?(String)
138
+ process_last_link = path.end_with?(PATH_SEPARATOR)
164
139
  path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
165
140
  return {api: self, file_id: top_file_id} if path_elements.empty?
166
141
  resolve_state = {path: path_elements, result: nil}
167
- process_folder_tree(state: resolve_state, method: :process_resolve_node_path, top_file_id: top_file_id)
168
- raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
169
- return resolve_state[:result]
170
- end
171
-
172
- # add entry to list if test block is success
173
- # @return [TrueClass,FalseClass]
174
- def process_find_files(entry, path, state)
175
- begin
176
- # add to result if match filter
177
- state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
178
- # process link
179
- if entry[:type].eql?('link')
180
- other_node = node_id_to_node(entry['target_node_id'])
181
- other_node.process_folder_tree(state: state, method: process_find_files, top_file_id: entry['target_id'], top_file_path: path)
142
+ process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
143
+ # this block is called recursively for each entry in folder
144
+ # stop digging here if not in right path
145
+ next false unless entry['name'].eql?(state[:path].first)
146
+ # ok it matches, so we remove the match
147
+ state[:path].shift
148
+ case entry['type']
149
+ when 'file'
150
+ # file must be terminal
151
+ raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
152
+ # it's terminal, we found it
153
+ state[:result] = {api: self, file_id: entry['id']}
154
+ next false
155
+ when 'folder'
156
+ if state[:path].empty?
157
+ # we found it
158
+ state[:result] = {api: self, file_id: entry['id']}
159
+ next false
160
+ end
161
+ when 'link'
162
+ if state[:path].empty?
163
+ if process_last_link
164
+ # we found it
165
+ other_node = node_id_to_node(entry['target_node_id'])
166
+ raise 'cannot resolve link' if other_node.nil?
167
+ state[:result] = {api: other_node, file_id: entry['target_id']}
168
+ else
169
+ # we found it but we do not process the link
170
+ state[:result] = {api: self, file_id: entry['id']}
171
+ end
172
+ next false
173
+ end
174
+ else
175
+ Log.log.warn{"Unknown element type: #{entry['type']}"}
182
176
  end
183
- rescue StandardError => e
184
- Log.log.error{"#{path}: #{e.message}"}
177
+ # continue to dig folder
178
+ next true
185
179
  end
186
- # process all folders
187
- return true
180
+ raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
181
+ return resolve_state[:result]
188
182
  end
189
183
 
190
184
  def find_files(top_file_id, test_block)
191
185
  Log.log.debug{"find_files: file id=#{top_file_id}"}
192
186
  find_state = {found: [], test_block: test_block}
193
- process_folder_tree(state: find_state, method: :process_find_files, top_file_id: top_file_id)
187
+ process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
188
+ state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
189
+ # test all files deeply
190
+ true
191
+ end
194
192
  return find_state[:found]
195
193
  end
196
194
 
@@ -205,6 +203,8 @@ module Aspera
205
203
  case params[:auth][:type]
206
204
  when :basic
207
205
  ak_name = params[:auth][:username]
206
+ raise 'ERROR: no secret in node object' unless params[:auth][:password]
207
+ ak_token = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
208
208
  when :oauth2
209
209
  ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
210
210
  # TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
@@ -216,7 +216,7 @@ module Aspera
216
216
  'direction' => direction,
217
217
  'token' => ak_token,
218
218
  'tags' => {
219
- 'aspera' => {
219
+ Fasp::TransferSpec::TAG_RESERVED => {
220
220
  'node' => {
221
221
  'access_key' => ak_name,
222
222
  'file_id' => file_id
@@ -228,12 +228,7 @@ module Aspera
228
228
  add_tspec_info(transfer_spec)
229
229
  transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
230
230
  # add application specific tags (AoC)
231
- the_app = app_info
232
- the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
233
- # add basic token
234
- if transfer_spec['token'].nil?
235
- ts_basic_token(transfer_spec)
236
- end
231
+ app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
237
232
  # add remote host info
238
233
  if self.class.use_standard_ports
239
234
  # get default TCP/UDP ports and transfer user
@@ -244,23 +239,17 @@ module Aspera
244
239
  transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
245
240
  end
246
241
  else
247
- # retrieve values from API
248
- std_t_spec = create(
242
+ # retrieve values from API (and keep a copy/cache)
243
+ @std_t_spec_cache ||= create(
249
244
  'files/download_setup',
250
245
  {transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
251
246
  )[:data]['transfer_specs'].first['transfer_spec']
252
247
  # copy some parts
253
- %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].each {|i| transfer_spec[i] = std_t_spec[i] if std_t_spec.key?(i)}
248
+ TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
254
249
  end
250
+ Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
251
+ unless transfer_spec['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
255
252
  return transfer_spec
256
253
  end
257
-
258
- # set basic token in transfer spec
259
- def ts_basic_token(ts)
260
- Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
261
- unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
262
- raise 'ERROR: no secret in node object' unless params[:auth][:password]
263
- ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
264
- end
265
254
  end
266
255
  end
data/lib/aspera/oauth.rb CHANGED
@@ -87,6 +87,7 @@ module Aspera
87
87
  # @param lambda_create called to create token
88
88
  # @param id_create called to generate unique id for token, for cache
89
89
  def register_token_creator(id, lambda_create, id_create)
90
+ Log.log.debug{"registering token creator #{id}"}
90
91
  raise 'ERROR: requites Symbol and 2 lambdas' unless id.is_a?(Symbol) && lambda_create.is_a?(Proc) && id_create.is_a?(Proc)
91
92
  @create_handlers[id] = lambda_create
92
93
  @id_handlers[id] = id_create
@@ -94,7 +95,7 @@ module Aspera
94
95
 
95
96
  # @return one of the registered creators for the given create type
96
97
  def token_creator(id)
97
- raise "token grant method unknown: #{id}/#{id.class}" unless @create_handlers.key?(id)
98
+ raise "token grant method unknown: '#{id}' (#{id.class})" unless @create_handlers.key?(id)
98
99
  @create_handlers[id]
99
100
  end
100
101
 
@@ -191,7 +192,7 @@ module Aspera
191
192
  # replace default values
192
193
  @generic_parameters = DEFAULT_CREATE_PARAMS.deep_merge(a_params)
193
194
  # legacy
194
- @generic_parameters[:grant_method] ||= @generic_parameters.delete(:crtype) if @generic_parameters.key?(:crtype)
195
+ @generic_parameters[:grant_method] ||= @generic_parameters.delete(:crtype) if @generic_parameters.key?(:crtype) # cspell: disable-line
195
196
  # check that type is known
196
197
  self.class.token_creator(@generic_parameters[:grant_method])
197
198
  # specific parameters for the creation type
@@ -272,7 +273,7 @@ module Aspera
272
273
 
273
274
  # an API was already called, but failed, we need to regenerate or refresh
274
275
  if use_refresh_token
275
- if token_data.is_a?(Hash) && token_data.key?('refresh_token')
276
+ if token_data.is_a?(Hash) && token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
276
277
  # save possible refresh token, before deleting the cache
277
278
  refresh_token = token_data['refresh_token']
278
279
  end
@@ -8,7 +8,7 @@ module Aspera
8
8
  class PersistencyActionOnce
9
9
  # @param :manager Mandatory Database
10
10
  # @param :data Mandatory object to persist, must be same object from begin to end (assume array by default)
11
- # @param :id Mandatory identifiers
11
+ # @param :id Mandatory identifiers
12
12
  # @param :delete Optional delete persistency condition
13
13
  # @param :parse Optional parse method (default to JSON)
14
14
  # @param :format Optional dump method (default to JSON)
@@ -12,6 +12,7 @@ module Aspera
12
12
  CONVERSION_TYPES = %i[image office pdf plaintext video].freeze
13
13
 
14
14
  # define how files are processed based on mime type
15
+ # spellchecker:disable
15
16
  SUPPORTED_MIME_TYPES = {
16
17
  'application/json' => :plaintext,
17
18
  'application/mac-binhex40' => :office,
@@ -267,6 +268,7 @@ module Aspera
267
268
  'ycbcra' => :image,
268
269
  'yuv' => :image,
269
270
  'zabw' => :office}.freeze
271
+ # spellchecker:enable
270
272
 
271
273
  private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
272
274
 
@@ -301,12 +303,12 @@ module Aspera
301
303
  def conversion_type(filepath, mimetype)
302
304
  Log.log.debug{"conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})"}
303
305
  # 1- get type from provided mime type, using local mapping
304
- conv_type = SUPPORTED_MIME_TYPES[mimetype] if !mimetype.nil?
306
+ conversion_type = SUPPORTED_MIME_TYPES[mimetype] if !mimetype.nil?
305
307
  # 2- else, from computed mime type (if available)
306
- if conv_type.nil? && @use_mimemagic
308
+ if conversion_type.nil? && @use_mimemagic
307
309
  detected_mime = mime_from_file(filepath)
308
310
  if !detected_mime.nil?
309
- conv_type = SUPPORTED_MIME_TYPES[detected_mime]
311
+ conversion_type = SUPPORTED_MIME_TYPES[detected_mime]
310
312
  if !mimetype.nil?
311
313
  if mimetype.eql?(detected_mime)
312
314
  Log.log.debug('matching mime type per magic number')
@@ -319,9 +321,9 @@ module Aspera
319
321
  end
320
322
  # 3- else, from extensions, using local mapping
321
323
  extension = File.extname(filepath.downcase)[1..-1]
322
- conv_type = SUPPORTED_EXTENSIONS[extension] if conv_type.nil?
323
- Log.log.debug{"conversion_type(#{extension}): #{conv_type.class.name} [#{conv_type}]"}
324
- return conv_type
324
+ conversion_type = SUPPORTED_EXTENSIONS[extension] if conversion_type.nil?
325
+ Log.log.debug{"conversion_type(#{extension}): #{conversion_type.class.name} [#{conversion_type}]"}
326
+ return conversion_type
325
327
  end
326
328
  end
327
329
  end