aspera-cli 4.16.0 → 4.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +50 -19
  4. data/CONTRIBUTING.md +3 -1
  5. data/README.md +965 -793
  6. data/bin/asession +29 -21
  7. data/lib/aspera/{fasp/agent_alpha.rb → agent/alpha.rb} +26 -25
  8. data/lib/aspera/{fasp/agent_base.rb → agent/base.rb} +15 -12
  9. data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
  10. data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +49 -53
  11. data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +20 -19
  12. data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +20 -33
  13. data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb} +11 -11
  14. data/lib/aspera/api/aoc.rb +586 -0
  15. data/lib/aspera/api/ats.rb +46 -0
  16. data/lib/aspera/api/cos_node.rb +95 -0
  17. data/lib/aspera/api/node.rb +344 -0
  18. data/lib/aspera/ascmd.rb +46 -10
  19. data/lib/aspera/{fasp → ascp}/installation.rb +5 -5
  20. data/lib/aspera/{fasp → ascp}/management.rb +3 -8
  21. data/lib/aspera/{fasp → ascp}/products.rb +1 -1
  22. data/lib/aspera/assert.rb +30 -30
  23. data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
  24. data/lib/aspera/cli/extended_value.rb +1 -1
  25. data/lib/aspera/cli/formatter.rb +13 -13
  26. data/lib/aspera/cli/hints.rb +5 -5
  27. data/lib/aspera/cli/main.rb +35 -28
  28. data/lib/aspera/cli/manager.rb +25 -24
  29. data/lib/aspera/cli/plugin.rb +22 -15
  30. data/lib/aspera/cli/plugin_factory.rb +61 -0
  31. data/lib/aspera/cli/plugins/alee.rb +7 -7
  32. data/lib/aspera/cli/plugins/aoc.rb +83 -77
  33. data/lib/aspera/cli/plugins/ats.rb +32 -33
  34. data/lib/aspera/cli/plugins/bss.rb +3 -4
  35. data/lib/aspera/cli/plugins/config.rb +169 -186
  36. data/lib/aspera/cli/plugins/console.rb +8 -6
  37. data/lib/aspera/cli/plugins/cos.rb +19 -18
  38. data/lib/aspera/cli/plugins/faspex.rb +61 -54
  39. data/lib/aspera/cli/plugins/faspex5.rb +150 -103
  40. data/lib/aspera/cli/plugins/node.rb +68 -73
  41. data/lib/aspera/cli/plugins/orchestrator.rb +34 -44
  42. data/lib/aspera/cli/plugins/preview.rb +31 -31
  43. data/lib/aspera/cli/plugins/server.rb +31 -33
  44. data/lib/aspera/cli/plugins/shares.rb +13 -11
  45. data/lib/aspera/cli/sync_actions.rb +8 -8
  46. data/lib/aspera/cli/transfer_agent.rb +32 -19
  47. data/lib/aspera/cli/transfer_progress.rb +1 -1
  48. data/lib/aspera/cli/version.rb +1 -1
  49. data/lib/aspera/colors.rb +5 -0
  50. data/lib/aspera/command_line_builder.rb +14 -14
  51. data/lib/aspera/coverage.rb +1 -2
  52. data/lib/aspera/data_repository.rb +1 -1
  53. data/lib/aspera/environment.rb +2 -3
  54. data/lib/aspera/faspex_gw.rb +5 -6
  55. data/lib/aspera/faspex_postproc.rb +1 -1
  56. data/lib/aspera/id_generator.rb +2 -2
  57. data/lib/aspera/json_rpc.rb +5 -5
  58. data/lib/aspera/keychain/encrypted_hash.rb +6 -6
  59. data/lib/aspera/keychain/macos_security.rb +27 -22
  60. data/lib/aspera/log.rb +2 -2
  61. data/lib/aspera/nagios.rb +3 -3
  62. data/lib/aspera/node_simulator.rb +5 -6
  63. data/lib/aspera/oauth/base.rb +143 -0
  64. data/lib/aspera/oauth/factory.rb +124 -0
  65. data/lib/aspera/oauth/generic.rb +34 -0
  66. data/lib/aspera/oauth/jwt.rb +51 -0
  67. data/lib/aspera/oauth/url_json.rb +31 -0
  68. data/lib/aspera/oauth/web.rb +50 -0
  69. data/lib/aspera/oauth.rb +5 -331
  70. data/lib/aspera/open_application.rb +7 -7
  71. data/lib/aspera/persistency_action_once.rb +4 -4
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/generator.rb +5 -5
  74. data/lib/aspera/preview/terminal.rb +3 -2
  75. data/lib/aspera/preview/utils.rb +3 -3
  76. data/lib/aspera/proxy_auto_config.rb +4 -4
  77. data/lib/aspera/rest.rb +175 -144
  78. data/lib/aspera/rest_errors_aspera.rb +3 -3
  79. data/lib/aspera/resumer.rb +77 -0
  80. data/lib/aspera/ssh.rb +6 -1
  81. data/lib/aspera/{fasp → transfer}/error.rb +3 -3
  82. data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
  83. data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
  84. data/lib/aspera/{fasp → transfer}/parameters.rb +58 -89
  85. data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +18 -16
  86. data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
  87. data/lib/aspera/{fasp → transfer}/sync.rb +32 -32
  88. data/lib/aspera/{fasp → transfer}/uri.rb +9 -8
  89. data/lib/aspera/web_server_simple.rb +11 -3
  90. data.tar.gz.sig +0 -0
  91. metadata +36 -63
  92. metadata.gz.sig +0 -0
  93. data/lib/aspera/aoc.rb +0 -601
  94. data/lib/aspera/ats_api.rb +0 -47
  95. data/lib/aspera/cos_node.rb +0 -94
  96. data/lib/aspera/fasp/resume_policy.rb +0 -79
  97. data/lib/aspera/node.rb +0 -339
@@ -0,0 +1,95 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/log'
4
+ require 'aspera/assert'
5
+ require 'aspera/api/node'
6
+ require 'xmlsimple'
7
+
8
+ module Aspera
9
+ module Api
10
+ class CosNode < Node
11
+ IBM_CLOUD_TOKEN_URL = 'https://iam.cloud.ibm.com/identity'
12
+ TOKEN_FIELD = 'delegated_refresh_token'
13
+ class << self
14
+ def parameters_from_svc_credentials(service_credentials, bucket_region)
15
+ # check necessary contents
16
+ Aspera.assert_type(service_credentials, Hash){'service_credentials'}
17
+ Aspera::Log.dump('service_credentials', service_credentials)
18
+ %w[apikey resource_instance_id endpoints].each do |field|
19
+ Aspera.assert(service_credentials.key?(field)){"service_credentials must have a field: #{field}"}
20
+ end
21
+ # read endpoints from service provided in service credentials
22
+ endpoints = Aspera::Rest.new(base_url: service_credentials['endpoints']).read('')[:data]
23
+ Aspera::Log.dump('endpoints', endpoints)
24
+ endpoint = endpoints.dig('service-endpoints', 'regional', bucket_region, 'public', bucket_region)
25
+ raise "no such region: #{bucket_region}" if endpoint.nil?
26
+ return {
27
+ instance_id: service_credentials['resource_instance_id'],
28
+ api_key: service_credentials['apikey'],
29
+ endpoint: endpoint
30
+ }
31
+ end
32
+ end
33
+
34
+ def initialize(instance_id:, api_key:, endpoint:, bucket:, auth_url: IBM_CLOUD_TOKEN_URL)
35
+ Aspera.assert_type(instance_id, String){'resource instance id (crn)'}
36
+ Aspera.assert_type(endpoint, String){'endpoint'}
37
+ endpoint = "https://#{endpoint}" unless endpoint.start_with?('http')
38
+ @auth_url = auth_url
39
+ @api_key = api_key
40
+ s3_api = Aspera::Rest.new(
41
+ base_url: endpoint,
42
+ not_auth_codes: %w[401 403], # error codes when not authorized
43
+ headers: {'ibm-service-instance-id' => instance_id},
44
+ auth: {
45
+ type: :oauth2,
46
+ grant_method: :generic,
47
+ base_url: @auth_url,
48
+ grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
49
+ response_type: 'cloud_iam',
50
+ apikey: @api_key
51
+ })
52
+ # read FASP connection information for bucket
53
+ xml_result_text = s3_api.call(
54
+ operation: 'GET',
55
+ subpath: bucket,
56
+ headers: {'Accept' => 'application/xml'},
57
+ url_params: {'faspConnectionInfo' => nil}
58
+ )[:http].body
59
+ ats_info = XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
60
+ Aspera::Log.dump('ats_info', ats_info)
61
+ @storage_credentials = {
62
+ 'type' => 'token',
63
+ 'token' => {TOKEN_FIELD => nil}
64
+ }
65
+ super(
66
+ base_url: ats_info['ATSEndpoint'],
67
+ auth: {
68
+ type: :basic,
69
+ username: ats_info['AccessKey']['Id'],
70
+ password: ats_info['AccessKey']['Secret']},
71
+ add_tspec: {'tags'=>{Transfer::Spec::TAG_RESERVED=>{'node'=>{'storage_credentials'=>@storage_credentials}}}}
72
+ )
73
+ # update storage_credentials AND Rest params
74
+ generate_token
75
+ end
76
+
77
+ # potentially call this if delegated token is expired
78
+ def generate_token
79
+ # OAuth API to get delegated token
80
+ delegated_oauth = OAuth::Factory.instance.create(
81
+ base_url: @auth_url,
82
+ grant_method: :generic,
83
+ token_field: TOKEN_FIELD,
84
+ grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
85
+ response_type: 'delegated_refresh_token',
86
+ apikey: @api_key,
87
+ receiver_client_ids: 'aspera_ats'
88
+ )
89
+ # get delegated token to be placed in rest call header and in transfer tags
90
+ @storage_credentials['token'][TOKEN_FIELD] = OAuth::Factory.bearer_extract(delegated_oauth.get_authorization)
91
+ @headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
92
+ end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,344 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/cli/error'
4
+ require 'aspera/transfer/spec'
5
+ require 'aspera/rest'
6
+ require 'aspera/oauth'
7
+ require 'aspera/log'
8
+ require 'aspera/assert'
9
+ require 'aspera/environment'
10
+ require 'zlib'
11
+ require 'base64'
12
+
13
+ module Aspera
14
+ module Api
15
+ # Provides additional functions using node API with gen4 extensions (access keys)
16
+ class Node < Aspera::Rest
17
+ # permissions
18
+ ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
19
+ # prefix for ruby code for filter (deprecated)
20
+ MATCH_EXEC_PREFIX = 'exec:'
21
+ MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
22
+ HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
23
+ PATH_SEPARATOR = '/'
24
+ TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
25
+ SCOPE_USER = 'user:all'
26
+ SCOPE_ADMIN = 'admin:all'
27
+ SCOPE_PREFIX = 'node.'
28
+ SCOPE_SEPARATOR = ':'
29
+ SIGNATURE_DELIMITER = '==SIGNATURE=='
30
+ BEARER_TOKEN_VALIDITY_DEFAULT = 86400
31
+ BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
32
+
33
+ # register node special token decoder
34
+ OAuth::Factory.instance.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
35
+
36
+ # class instance variable, access with accessors on class
37
+ @use_standard_ports = true
38
+
39
+ class << self
40
+ attr_accessor :use_standard_ports
41
+
42
+ # For access keys: provide expression to match entry in folder
43
+ def file_matcher(match_expression)
44
+ case match_expression
45
+ when Proc then return match_expression
46
+ when Regexp then return ->(f){f['name'].match?(match_expression)}
47
+ when String
48
+ if match_expression.start_with?(MATCH_EXEC_PREFIX)
49
+ code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
50
+ Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
51
+ return Environment.secure_eval(code, __FILE__, __LINE__)
52
+ end
53
+ return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
54
+ when NilClass then return ->(_){true}
55
+ else Aspera.error_unexpected_value(match_expression.class.name, exception_class: Cli::BadArgument)
56
+ end
57
+ end
58
+
59
+ def file_matcher_from_argument(options)
60
+ return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
61
+ end
62
+
63
+ # node API scopes
64
+ def token_scope(access_key, scope)
65
+ return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
66
+ end
67
+
68
+ def decode_scope(scope)
69
+ items = scope.split(SCOPE_SEPARATOR, 2)
70
+ Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
71
+ Aspera.assert(items[0].start_with?(SCOPE_PREFIX)){"invalid scope: #{scope}"}
72
+ return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
73
+ end
74
+
75
+ # Create an Aspera Node bearer token
76
+ # @param payload [String] JSON payload to be included in the token
77
+ # @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
78
+ def bearer_token(access_key:, payload:, private_key:)
79
+ Aspera.assert_type(payload, Hash)
80
+ Aspera.assert(payload.key?('user_id'))
81
+ Aspera.assert_type(payload['user_id'], String)
82
+ Aspera.assert(!payload['user_id'].empty?)
83
+ Aspera.assert_type(private_key, OpenSSL::PKey::RSA)
84
+ # manage convenience parameters
85
+ expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
86
+ payload.delete('_validity')
87
+ scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
88
+ payload.delete('_scope')
89
+ payload['scope'] ||= token_scope(access_key, scope)
90
+ payload['auth_type'] ||= 'access_key'
91
+ payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
92
+ payload_json = JSON.generate(payload)
93
+ return Base64.strict_encode64(Zlib::Deflate.deflate([
94
+ payload_json,
95
+ SIGNATURE_DELIMITER,
96
+ Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
97
+ ''
98
+ ].join("\n")))
99
+ end
100
+
101
+ def decode_bearer_token(token)
102
+ return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
103
+ end
104
+
105
+ def bearer_headers(bearer_auth, access_key: nil)
106
+ # if username is not provided, use the access key from the token
107
+ if access_key.nil?
108
+ access_key = Node.decode_scope(Node.decode_bearer_token(OAuth::Factory.bearer_extract(bearer_auth))['scope'])[:access_key]
109
+ Aspera.assert(!access_key.nil?)
110
+ end
111
+ return {
112
+ Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
113
+ 'Authorization' => bearer_auth
114
+ }
115
+ end
116
+ end
117
+
118
+ # fields in @app_info
119
+ REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
120
+ # methods of @app_info[:api]
121
+ REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
122
+ private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
123
+
124
+ attr_reader :app_info
125
+
126
+ # @param base_url [String] Rest parameters
127
+ # @param auth [String,NilClass] Rest parameters
128
+ # @param headers [String,NilClass] Rest parameters
129
+ # @param app_info [Hash,NilClass] Special processing for AoC
130
+ # @param add_tspec [Hash,NilClass] Additional transfer spec
131
+ def initialize(app_info: nil, add_tspec: nil, **rest_args)
132
+ # init Rest
133
+ super(**rest_args)
134
+ @app_info = app_info
135
+ # this is added to transfer spec, for instance to add tags (COS)
136
+ @add_tspec = add_tspec
137
+ if !@app_info.nil?
138
+ REQUIRED_APP_INFO_FIELDS.each do |field|
139
+ Aspera.assert(@app_info.key?(field)){"app_info lacks field #{field}"}
140
+ end
141
+ REQUIRED_APP_API_METHODS.each do |method|
142
+ Aspera.assert(@app_info[:api].respond_to?(method)){"#{@app_info[:api].class} lacks method #{method}"}
143
+ end
144
+ end
145
+ end
146
+
147
+ # update transfer spec with special additional tags
148
+ def add_tspec_info(tspec)
149
+ tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
150
+ return tspec
151
+ end
152
+
153
+ # @returns [Node] a Node or nil
154
+ def node_id_to_node(node_id)
155
+ if !@app_info.nil?
156
+ return self if node_id.eql?(@app_info[:node_info]['id'])
157
+ return @app_info[:api].node_api_from(
158
+ node_id: node_id,
159
+ workspace_id: @app_info[:workspace_id],
160
+ workspace_name: @app_info[:workspace_name])
161
+ end
162
+ Log.log.warn{"cannot resolve link with node id #{node_id}"}
163
+ return nil
164
+ end
165
+
166
+ # Recursively browse in a folder (with non-recursive method)
167
+ # sub folders are processed if the processing method returns true
168
+ # @param state [Object] state object sent to processing method
169
+ # @param top_file_id [String] file id to start at (default = access key root file id)
170
+ # @param top_file_path [String] path of top folder (default = /)
171
+ # @param block [Proc] processing method, arguments: entry, path, state
172
+ def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
173
+ Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
174
+ Aspera.assert(block){'Missing block'}
175
+ # start at top folder
176
+ folders_to_explore = [{id: top_file_id, path: top_file_path}]
177
+ Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
178
+ until folders_to_explore.empty?
179
+ current_item = folders_to_explore.shift
180
+ Log.log.debug{"searching #{current_item[:path]}".bg_green}
181
+ # get folder content
182
+ folder_contents =
183
+ begin
184
+ read("files/#{current_item[:id]}/files")[:data]
185
+ rescue StandardError => e
186
+ Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
187
+ []
188
+ end
189
+ Log.log.debug{Log.dump(:folder_contents, folder_contents)}
190
+ folder_contents.each do |entry|
191
+ relative_path = File.join(current_item[:path], entry['name'])
192
+ Log.log.debug{"process_folder_tree checking #{relative_path}"}
193
+ # continue only if method returns true
194
+ next unless yield(entry, relative_path, state)
195
+ # entry type is file, folder or link
196
+ case entry['type']
197
+ when 'folder'
198
+ folders_to_explore.push({id: entry['id'], path: relative_path})
199
+ when 'link'
200
+ node_id_to_node(entry['target_node_id'])&.process_folder_tree(
201
+ state: state,
202
+ top_file_id: entry['target_id'],
203
+ top_file_path: relative_path,
204
+ &block)
205
+ end
206
+ end
207
+ end
208
+ end # process_folder_tree
209
+
210
+ # Navigate the path from given file id
211
+ # @param top_file_id [String] id initial file id
212
+ # @param path [String] file path
213
+ # @return [Hash] {.api,.file_id}
214
+ def resolve_api_fid(top_file_id, path)
215
+ Aspera.assert_type(top_file_id, String)
216
+ process_last_link = path.end_with?(PATH_SEPARATOR)
217
+ path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
218
+ return {api: self, file_id: top_file_id} if path_elements.empty?
219
+ resolve_state = {path: path_elements, result: nil}
220
+ process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
221
+ # this block is called recursively for each entry in folder
222
+ # stop digging here if not in right path
223
+ next false unless entry['name'].eql?(state[:path].first)
224
+ # ok it matches, so we remove the match
225
+ state[:path].shift
226
+ case entry['type']
227
+ when 'file'
228
+ # file must be terminal
229
+ raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
230
+ # it's terminal, we found it
231
+ state[:result] = {api: self, file_id: entry['id']}
232
+ next false
233
+ when 'folder'
234
+ if state[:path].empty?
235
+ # we found it
236
+ state[:result] = {api: self, file_id: entry['id']}
237
+ next false
238
+ end
239
+ when 'link'
240
+ if state[:path].empty?
241
+ if process_last_link
242
+ # we found it
243
+ other_node = node_id_to_node(entry['target_node_id'])
244
+ raise 'cannot resolve link' if other_node.nil?
245
+ state[:result] = {api: other_node, file_id: entry['target_id']}
246
+ else
247
+ # we found it but we do not process the link
248
+ state[:result] = {api: self, file_id: entry['id']}
249
+ end
250
+ next false
251
+ end
252
+ else
253
+ Log.log.warn{"Unknown element type: #{entry['type']}"}
254
+ end
255
+ # continue to dig folder
256
+ next true
257
+ end
258
+ raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
259
+ return resolve_state[:result]
260
+ end
261
+
262
+ def find_files(top_file_id, test_block)
263
+ Log.log.debug{"find_files: file id=#{top_file_id}"}
264
+ find_state = {found: [], test_block: test_block}
265
+ process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
266
+ state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
267
+ # test all files deeply
268
+ true
269
+ end
270
+ return find_state[:found]
271
+ end
272
+
273
+ def refreshed_transfer_token
274
+ return oauth_token(force_refresh: true)
275
+ end
276
+
277
+ # Create transfer spec for gen4
278
+ def transfer_spec_gen4(file_id, direction, ts_merge=nil)
279
+ ak_name = nil
280
+ ak_token = nil
281
+ case auth_params[:type]
282
+ when :basic
283
+ ak_name = auth_params[:username]
284
+ Aspera.assert(auth_params[:password]){'no secret in node object'}
285
+ ak_token = Rest.basic_token(auth_params[:username], auth_params[:password])
286
+ when :oauth2
287
+ ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
288
+ # TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
289
+ # get bearer token, possibly use cache
290
+ ak_token = oauth_token(force_refresh: false)
291
+ else Aspera.error_unexpected_value(auth_params[:type])
292
+ end
293
+ transfer_spec = {
294
+ 'direction' => direction,
295
+ 'token' => ak_token,
296
+ 'tags' => {
297
+ Transfer::Spec::TAG_RESERVED => {
298
+ 'node' => {
299
+ 'access_key' => ak_name,
300
+ 'file_id' => file_id
301
+ } # node
302
+ } # aspera
303
+ } # tags
304
+ }
305
+ # add specials tags (cos)
306
+ add_tspec_info(transfer_spec)
307
+ transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
308
+ # add application specific tags (AoC)
309
+ app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
310
+ # add remote host info
311
+ if self.class.use_standard_ports
312
+ # get default TCP/UDP ports and transfer user
313
+ transfer_spec.merge!(Transfer::Spec::AK_TSPEC_BASE)
314
+ # by default: same address as node API
315
+ transfer_spec['remote_host'] = URI.parse(base_url).host
316
+ # AoC allows specification of other url
317
+ if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
318
+ transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
319
+ end
320
+ info = read('info')[:data]
321
+ # get the transfer user from info on access key
322
+ transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
323
+ # get settings from name.value array to hash key.value
324
+ settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
325
+ # check WSS ports
326
+ %w[wss_enabled wss_port].each do |i|
327
+ transfer_spec[i] = settings[i] if settings.key?(i)
328
+ end if settings.is_a?(Hash)
329
+ else
330
+ # retrieve values from API (and keep a copy/cache)
331
+ @std_t_spec_cache ||= create(
332
+ 'files/download_setup',
333
+ {transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
334
+ )[:data]['transfer_specs'].first['transfer_spec']
335
+ # copy some parts
336
+ TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
337
+ end
338
+ Log.log.warn{"Expected transfer user: #{Transfer::Spec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
339
+ unless transfer_spec['remote_user'].eql?(Transfer::Spec::ACCESS_KEY_TRANSFER_USER)
340
+ return transfer_spec
341
+ end
342
+ end
343
+ end
344
+ end
data/lib/aspera/ascmd.rb CHANGED
@@ -10,8 +10,20 @@ module Aspera
10
10
  # execute: "ascmd -h" to get syntax
11
11
  # Note: "ls" can take filters: as_ls -f *.txt -f *.bin /
12
12
  class AsCmd
13
+ # number of arguments for each operation
14
+ OPS_ARGS = {
15
+ cp: 2,
16
+ df: 0,
17
+ du: 1,
18
+ info: nil,
19
+ ls: 1,
20
+ md5sum: 1,
21
+ mkdir: 1,
22
+ mv: 2,
23
+ rm: 1
24
+ }.freeze
13
25
  # list of supported actions
14
- OPERATIONS = %i[ls rm mv du info mkdir cp df md5sum].freeze
26
+ OPERATIONS = OPS_ARGS.keys.freeze
15
27
 
16
28
  # @param command_executor [Object] provides the "execute" method, taking a command to execute, and stdin to feed to it, typically: ssh or local
17
29
  def initialize(command_executor)
@@ -22,16 +34,36 @@ module Aspera
22
34
  # @param [Symbol] one of OPERATIONS
23
35
  # @param [Array] parameters for "as" command
24
36
  # @return result of command, type depends on command (bool, array, hash)
25
- def execute_single(action_sym, arguments=nil)
37
+ def execute_single(action_sym, arguments)
38
+ arguments = [] if arguments.nil?
39
+ Log.log.debug{"execute_single:#{action_sym}:#{arguments}"}
40
+ Aspera.assert_type(action_sym, Symbol)
41
+ Aspera.assert_type(arguments, Array)
42
+ Aspera.assert(arguments.all?(String), 'arguments must be strings')
43
+ # lines of commands (String's)
44
+ command_lines = []
26
45
  # add "as_" command
27
- main_command = ["as_#{action_sym}"]
28
- arguments&.each do |v|
46
+ main_command = "as_#{action_sym}"
47
+ arg_batches =
48
+ if OPS_ARGS[action_sym].nil? || OPS_ARGS[action_sym].zero?
49
+ [arguments]
50
+ else
51
+ # split arguments into batches
52
+ arguments.each_slice(OPS_ARGS[action_sym]).to_a
53
+ end
54
+ arg_batches.each do |args|
55
+ command = [main_command]
29
56
  # enclose arguments in double quotes, protect backslash and double quotes
30
- main_command.push(%Q{"#{v.gsub(/["\\]/n){|s|"\\#{s}"}}"})
57
+ args.each do |v|
58
+ command.push(%Q{"#{v.gsub(/["\\]/){|s|"\\#{s}"}}"})
59
+ end
60
+ command_lines.push(command.join(' '))
31
61
  end
62
+ command_lines.push('as_exit')
63
+ command_lines.push('')
32
64
  # execute the main command and then exit
33
- stdin_input = [main_command.join(' '), 'as_exit', ''].join("\n")
34
- Log.log.debug{"execute_single:#{stdin_input}"}
65
+ stdin_input = command_lines.join("\n")
66
+ Log.log.trace1{"execute_single:#{stdin_input}"}
35
67
  # execute, get binary output
36
68
  byte_buffer = @command_executor.execute('ascmd', stdin_input).unpack('C*')
37
69
  raise 'ERROR: empty answer from server' if byte_buffer.empty?
@@ -53,7 +85,11 @@ module Aspera
53
85
  end
54
86
  end
55
87
  # for info, second overrides first, so restore it
56
- case result.keys.length; when 0 then result = system_info; when 1 then result = result[result.keys.first]; else error_unexpected_value(result.keys.length); end
88
+ case result.keys.length
89
+ when 0 then result = system_info
90
+ when 1 then result = result[result.keys.first]
91
+ else Aspera.error_unexpected_value(result.keys.length)
92
+ end
57
93
  # raise error as exception
58
94
  raise Error.new(result[:errno], result[:errstr], action_sym, arguments) if
59
95
  result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{|i|i[:name]}.sort)
@@ -66,7 +102,7 @@ module Aspera
66
102
  super(); @errno = errno; @errstr = errstr; @command = cmd; @arguments = arguments; end # rubocop:disable Style/Semicolon
67
103
 
68
104
  def message; "ascmd: #{@errstr} (#{@errno})"; end
69
- def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments.join(',')}"; end
105
+ def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments&.join(',')}"; end
70
106
  end # Error
71
107
 
72
108
  # description of result structures (see ascmdtypes.h). Base types are big endian
@@ -174,7 +210,7 @@ module Aspera
174
210
  end
175
211
  end
176
212
  end
177
- else error_unexpected_value(type_descr[:decode])
213
+ else Aspera.error_unexpected_value(type_descr[:decode])
178
214
  end # is_a
179
215
  return result
180
216
  end
@@ -3,7 +3,7 @@
3
3
  # cspell:ignore protobuf ckpt
4
4
  require 'aspera/environment'
5
5
  require 'aspera/data_repository'
6
- require 'aspera/fasp/products'
6
+ require 'aspera/ascp/products'
7
7
  require 'aspera/log'
8
8
  require 'aspera/assert'
9
9
  require 'aspera/web_server_simple'
@@ -16,7 +16,7 @@ require 'fileutils'
16
16
  require 'openssl'
17
17
 
18
18
  module Aspera
19
- module Fasp
19
+ module Ascp
20
20
  # Singleton that tells where to find ascp and other local resources (keys..) , using the "path(:name)" method.
21
21
  # It is used by object : AgentDirect to find necessary resources
22
22
  # By default it takes the first Aspera product found
@@ -139,10 +139,10 @@ module Aspera
139
139
  check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
140
140
  end
141
141
  file = k.eql?(:fallback_certificate) ? file_cert : file_key
142
- else error_unexpected_value(k)
142
+ else Aspera.error_unexpected_value(k)
143
143
  end
144
144
  return nil if file_is_optional && !File.exist?(file)
145
- assert(File.exist?(file)){"no such file: #{file}"}
145
+ Aspera.assert(File.exist?(file)){"no such file: #{file}"}
146
146
  return file
147
147
  end
148
148
 
@@ -283,6 +283,6 @@ module Aspera
283
283
  def transferd_filepath
284
284
  return File.join(sdk_folder, 'asperatransferd' + Environment.exe_extension) # cspell:disable-line
285
285
  end
286
- end # Installation
286
+ end
287
287
  end
288
288
  end
@@ -1,8 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Aspera
4
- module Fasp
5
- # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
4
+ module Ascp
5
+ # processing of ascp management port events
6
6
  class Management
7
7
  # cspell: disable
8
8
  OPERATIONS = %w[
@@ -202,12 +202,7 @@ module Aspera
202
202
  # translates mgt port event into (enhanced) typed event
203
203
  def enhanced_event_format(event)
204
204
  return event.keys.each_with_object({}) do |e, h|
205
- # capital_to_snake_case
206
- new_name = e
207
- .gsub(/([a-z\d])([A-Z])/, '\1_\2')
208
- .gsub(/([a-z\d])(usec)$/, '\1_\2')
209
- .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
210
- .downcase
205
+ new_name = e.capital_to_snake.gsub(/(usec)$/, '_\1').downcase
211
206
  value = event[e]
212
207
  value = value.to_i if INTEGER_FIELDS.include?(e)
213
208
  value = value.eql?(BOOLEAN_TRUE) if BOOLEAN_FIELDS.include?(e)
@@ -4,7 +4,7 @@
4
4
  require 'aspera/environment'
5
5
 
6
6
  module Aspera
7
- module Fasp
7
+ module Ascp
8
8
  # find Aspera standard products installation in standard paths
9
9
  class Products
10
10
  # known product names