aspera-cli 4.16.0 → 4.17.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +50 -19
  4. data/CONTRIBUTING.md +3 -1
  5. data/README.md +965 -793
  6. data/bin/asession +29 -21
  7. data/lib/aspera/{fasp/agent_alpha.rb → agent/alpha.rb} +26 -25
  8. data/lib/aspera/{fasp/agent_base.rb → agent/base.rb} +15 -12
  9. data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
  10. data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +49 -53
  11. data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +20 -19
  12. data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +20 -33
  13. data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb} +11 -11
  14. data/lib/aspera/api/aoc.rb +586 -0
  15. data/lib/aspera/api/ats.rb +46 -0
  16. data/lib/aspera/api/cos_node.rb +95 -0
  17. data/lib/aspera/api/node.rb +344 -0
  18. data/lib/aspera/ascmd.rb +46 -10
  19. data/lib/aspera/{fasp → ascp}/installation.rb +5 -5
  20. data/lib/aspera/{fasp → ascp}/management.rb +3 -8
  21. data/lib/aspera/{fasp → ascp}/products.rb +1 -1
  22. data/lib/aspera/assert.rb +30 -30
  23. data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
  24. data/lib/aspera/cli/extended_value.rb +1 -1
  25. data/lib/aspera/cli/formatter.rb +13 -13
  26. data/lib/aspera/cli/hints.rb +5 -5
  27. data/lib/aspera/cli/main.rb +35 -28
  28. data/lib/aspera/cli/manager.rb +25 -24
  29. data/lib/aspera/cli/plugin.rb +22 -15
  30. data/lib/aspera/cli/plugin_factory.rb +61 -0
  31. data/lib/aspera/cli/plugins/alee.rb +7 -7
  32. data/lib/aspera/cli/plugins/aoc.rb +83 -77
  33. data/lib/aspera/cli/plugins/ats.rb +32 -33
  34. data/lib/aspera/cli/plugins/bss.rb +3 -4
  35. data/lib/aspera/cli/plugins/config.rb +169 -186
  36. data/lib/aspera/cli/plugins/console.rb +8 -6
  37. data/lib/aspera/cli/plugins/cos.rb +19 -18
  38. data/lib/aspera/cli/plugins/faspex.rb +61 -54
  39. data/lib/aspera/cli/plugins/faspex5.rb +150 -103
  40. data/lib/aspera/cli/plugins/node.rb +68 -73
  41. data/lib/aspera/cli/plugins/orchestrator.rb +34 -44
  42. data/lib/aspera/cli/plugins/preview.rb +31 -31
  43. data/lib/aspera/cli/plugins/server.rb +31 -33
  44. data/lib/aspera/cli/plugins/shares.rb +13 -11
  45. data/lib/aspera/cli/sync_actions.rb +8 -8
  46. data/lib/aspera/cli/transfer_agent.rb +32 -19
  47. data/lib/aspera/cli/transfer_progress.rb +1 -1
  48. data/lib/aspera/cli/version.rb +1 -1
  49. data/lib/aspera/colors.rb +5 -0
  50. data/lib/aspera/command_line_builder.rb +14 -14
  51. data/lib/aspera/coverage.rb +1 -2
  52. data/lib/aspera/data_repository.rb +1 -1
  53. data/lib/aspera/environment.rb +2 -3
  54. data/lib/aspera/faspex_gw.rb +5 -6
  55. data/lib/aspera/faspex_postproc.rb +1 -1
  56. data/lib/aspera/id_generator.rb +2 -2
  57. data/lib/aspera/json_rpc.rb +5 -5
  58. data/lib/aspera/keychain/encrypted_hash.rb +6 -6
  59. data/lib/aspera/keychain/macos_security.rb +27 -22
  60. data/lib/aspera/log.rb +2 -2
  61. data/lib/aspera/nagios.rb +3 -3
  62. data/lib/aspera/node_simulator.rb +5 -6
  63. data/lib/aspera/oauth/base.rb +143 -0
  64. data/lib/aspera/oauth/factory.rb +124 -0
  65. data/lib/aspera/oauth/generic.rb +34 -0
  66. data/lib/aspera/oauth/jwt.rb +51 -0
  67. data/lib/aspera/oauth/url_json.rb +31 -0
  68. data/lib/aspera/oauth/web.rb +50 -0
  69. data/lib/aspera/oauth.rb +5 -331
  70. data/lib/aspera/open_application.rb +7 -7
  71. data/lib/aspera/persistency_action_once.rb +4 -4
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/generator.rb +5 -5
  74. data/lib/aspera/preview/terminal.rb +3 -2
  75. data/lib/aspera/preview/utils.rb +3 -3
  76. data/lib/aspera/proxy_auto_config.rb +4 -4
  77. data/lib/aspera/rest.rb +175 -144
  78. data/lib/aspera/rest_errors_aspera.rb +3 -3
  79. data/lib/aspera/resumer.rb +77 -0
  80. data/lib/aspera/ssh.rb +6 -1
  81. data/lib/aspera/{fasp → transfer}/error.rb +3 -3
  82. data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
  83. data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
  84. data/lib/aspera/{fasp → transfer}/parameters.rb +58 -89
  85. data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +18 -16
  86. data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
  87. data/lib/aspera/{fasp → transfer}/sync.rb +32 -32
  88. data/lib/aspera/{fasp → transfer}/uri.rb +9 -8
  89. data/lib/aspera/web_server_simple.rb +11 -3
  90. data.tar.gz.sig +0 -0
  91. metadata +36 -63
  92. metadata.gz.sig +0 -0
  93. data/lib/aspera/aoc.rb +0 -601
  94. data/lib/aspera/ats_api.rb +0 -47
  95. data/lib/aspera/cos_node.rb +0 -94
  96. data/lib/aspera/fasp/resume_policy.rb +0 -79
  97. data/lib/aspera/node.rb +0 -339
@@ -0,0 +1,95 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/log'
4
+ require 'aspera/assert'
5
+ require 'aspera/api/node'
6
+ require 'xmlsimple'
7
+
8
+ module Aspera
9
+ module Api
10
+ class CosNode < Node
11
+ IBM_CLOUD_TOKEN_URL = 'https://iam.cloud.ibm.com/identity'
12
+ TOKEN_FIELD = 'delegated_refresh_token'
13
+ class << self
14
+ def parameters_from_svc_credentials(service_credentials, bucket_region)
15
+ # check necessary contents
16
+ Aspera.assert_type(service_credentials, Hash){'service_credentials'}
17
+ Aspera::Log.dump('service_credentials', service_credentials)
18
+ %w[apikey resource_instance_id endpoints].each do |field|
19
+ Aspera.assert(service_credentials.key?(field)){"service_credentials must have a field: #{field}"}
20
+ end
21
+ # read endpoints from service provided in service credentials
22
+ endpoints = Aspera::Rest.new(base_url: service_credentials['endpoints']).read('')[:data]
23
+ Aspera::Log.dump('endpoints', endpoints)
24
+ endpoint = endpoints.dig('service-endpoints', 'regional', bucket_region, 'public', bucket_region)
25
+ raise "no such region: #{bucket_region}" if endpoint.nil?
26
+ return {
27
+ instance_id: service_credentials['resource_instance_id'],
28
+ api_key: service_credentials['apikey'],
29
+ endpoint: endpoint
30
+ }
31
+ end
32
+ end
33
+
34
+ def initialize(instance_id:, api_key:, endpoint:, bucket:, auth_url: IBM_CLOUD_TOKEN_URL)
35
+ Aspera.assert_type(instance_id, String){'resource instance id (crn)'}
36
+ Aspera.assert_type(endpoint, String){'endpoint'}
37
+ endpoint = "https://#{endpoint}" unless endpoint.start_with?('http')
38
+ @auth_url = auth_url
39
+ @api_key = api_key
40
+ s3_api = Aspera::Rest.new(
41
+ base_url: endpoint,
42
+ not_auth_codes: %w[401 403], # error codes when not authorized
43
+ headers: {'ibm-service-instance-id' => instance_id},
44
+ auth: {
45
+ type: :oauth2,
46
+ grant_method: :generic,
47
+ base_url: @auth_url,
48
+ grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
49
+ response_type: 'cloud_iam',
50
+ apikey: @api_key
51
+ })
52
+ # read FASP connection information for bucket
53
+ xml_result_text = s3_api.call(
54
+ operation: 'GET',
55
+ subpath: bucket,
56
+ headers: {'Accept' => 'application/xml'},
57
+ url_params: {'faspConnectionInfo' => nil}
58
+ )[:http].body
59
+ ats_info = XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
60
+ Aspera::Log.dump('ats_info', ats_info)
61
+ @storage_credentials = {
62
+ 'type' => 'token',
63
+ 'token' => {TOKEN_FIELD => nil}
64
+ }
65
+ super(
66
+ base_url: ats_info['ATSEndpoint'],
67
+ auth: {
68
+ type: :basic,
69
+ username: ats_info['AccessKey']['Id'],
70
+ password: ats_info['AccessKey']['Secret']},
71
+ add_tspec: {'tags'=>{Transfer::Spec::TAG_RESERVED=>{'node'=>{'storage_credentials'=>@storage_credentials}}}}
72
+ )
73
+ # update storage_credentials AND Rest params
74
+ generate_token
75
+ end
76
+
77
+ # potentially call this if delegated token is expired
78
+ def generate_token
79
+ # OAuth API to get delegated token
80
+ delegated_oauth = OAuth::Factory.instance.create(
81
+ base_url: @auth_url,
82
+ grant_method: :generic,
83
+ token_field: TOKEN_FIELD,
84
+ grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
85
+ response_type: 'delegated_refresh_token',
86
+ apikey: @api_key,
87
+ receiver_client_ids: 'aspera_ats'
88
+ )
89
+ # get delegated token to be placed in rest call header and in transfer tags
90
+ @storage_credentials['token'][TOKEN_FIELD] = OAuth::Factory.bearer_extract(delegated_oauth.get_authorization)
91
+ @headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
92
+ end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,344 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/cli/error'
4
+ require 'aspera/transfer/spec'
5
+ require 'aspera/rest'
6
+ require 'aspera/oauth'
7
+ require 'aspera/log'
8
+ require 'aspera/assert'
9
+ require 'aspera/environment'
10
+ require 'zlib'
11
+ require 'base64'
12
+
13
+ module Aspera
14
+ module Api
15
+ # Provides additional functions using node API with gen4 extensions (access keys)
16
+ class Node < Aspera::Rest
17
+ # permissions
18
+ ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
19
+ # prefix for ruby code for filter (deprecated)
20
+ MATCH_EXEC_PREFIX = 'exec:'
21
+ MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
22
+ HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
23
+ PATH_SEPARATOR = '/'
24
+ TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
25
+ SCOPE_USER = 'user:all'
26
+ SCOPE_ADMIN = 'admin:all'
27
+ SCOPE_PREFIX = 'node.'
28
+ SCOPE_SEPARATOR = ':'
29
+ SIGNATURE_DELIMITER = '==SIGNATURE=='
30
+ BEARER_TOKEN_VALIDITY_DEFAULT = 86400
31
+ BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
32
+
33
+ # register node special token decoder
34
+ OAuth::Factory.instance.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
35
+
36
+ # class instance variable, access with accessors on class
37
+ @use_standard_ports = true
38
+
39
+ class << self
40
+ attr_accessor :use_standard_ports
41
+
42
+ # For access keys: provide expression to match entry in folder
43
+ def file_matcher(match_expression)
44
+ case match_expression
45
+ when Proc then return match_expression
46
+ when Regexp then return ->(f){f['name'].match?(match_expression)}
47
+ when String
48
+ if match_expression.start_with?(MATCH_EXEC_PREFIX)
49
+ code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
50
+ Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
51
+ return Environment.secure_eval(code, __FILE__, __LINE__)
52
+ end
53
+ return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
54
+ when NilClass then return ->(_){true}
55
+ else Aspera.error_unexpected_value(match_expression.class.name, exception_class: Cli::BadArgument)
56
+ end
57
+ end
58
+
59
+ def file_matcher_from_argument(options)
60
+ return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
61
+ end
62
+
63
+ # node API scopes
64
+ def token_scope(access_key, scope)
65
+ return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
66
+ end
67
+
68
+ def decode_scope(scope)
69
+ items = scope.split(SCOPE_SEPARATOR, 2)
70
+ Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
71
+ Aspera.assert(items[0].start_with?(SCOPE_PREFIX)){"invalid scope: #{scope}"}
72
+ return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
73
+ end
74
+
75
+ # Create an Aspera Node bearer token
76
+ # @param payload [String] JSON payload to be included in the token
77
+ # @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
78
+ def bearer_token(access_key:, payload:, private_key:)
79
+ Aspera.assert_type(payload, Hash)
80
+ Aspera.assert(payload.key?('user_id'))
81
+ Aspera.assert_type(payload['user_id'], String)
82
+ Aspera.assert(!payload['user_id'].empty?)
83
+ Aspera.assert_type(private_key, OpenSSL::PKey::RSA)
84
+ # manage convenience parameters
85
+ expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
86
+ payload.delete('_validity')
87
+ scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
88
+ payload.delete('_scope')
89
+ payload['scope'] ||= token_scope(access_key, scope)
90
+ payload['auth_type'] ||= 'access_key'
91
+ payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
92
+ payload_json = JSON.generate(payload)
93
+ return Base64.strict_encode64(Zlib::Deflate.deflate([
94
+ payload_json,
95
+ SIGNATURE_DELIMITER,
96
+ Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
97
+ ''
98
+ ].join("\n")))
99
+ end
100
+
101
+ def decode_bearer_token(token)
102
+ return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
103
+ end
104
+
105
+ def bearer_headers(bearer_auth, access_key: nil)
106
+ # if username is not provided, use the access key from the token
107
+ if access_key.nil?
108
+ access_key = Node.decode_scope(Node.decode_bearer_token(OAuth::Factory.bearer_extract(bearer_auth))['scope'])[:access_key]
109
+ Aspera.assert(!access_key.nil?)
110
+ end
111
+ return {
112
+ Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
113
+ 'Authorization' => bearer_auth
114
+ }
115
+ end
116
+ end
117
+
118
+ # fields in @app_info
119
+ REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
120
+ # methods of @app_info[:api]
121
+ REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
122
+ private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
123
+
124
+ attr_reader :app_info
125
+
126
+ # @param base_url [String] Rest parameters
127
+ # @param auth [String,NilClass] Rest parameters
128
+ # @param headers [String,NilClass] Rest parameters
129
+ # @param app_info [Hash,NilClass] Special processing for AoC
130
+ # @param add_tspec [Hash,NilClass] Additional transfer spec
131
+ def initialize(app_info: nil, add_tspec: nil, **rest_args)
132
+ # init Rest
133
+ super(**rest_args)
134
+ @app_info = app_info
135
+ # this is added to transfer spec, for instance to add tags (COS)
136
+ @add_tspec = add_tspec
137
+ if !@app_info.nil?
138
+ REQUIRED_APP_INFO_FIELDS.each do |field|
139
+ Aspera.assert(@app_info.key?(field)){"app_info lacks field #{field}"}
140
+ end
141
+ REQUIRED_APP_API_METHODS.each do |method|
142
+ Aspera.assert(@app_info[:api].respond_to?(method)){"#{@app_info[:api].class} lacks method #{method}"}
143
+ end
144
+ end
145
+ end
146
+
147
+ # update transfer spec with special additional tags
148
+ def add_tspec_info(tspec)
149
+ tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
150
+ return tspec
151
+ end
152
+
153
+ # @returns [Node] a Node or nil
154
+ def node_id_to_node(node_id)
155
+ if !@app_info.nil?
156
+ return self if node_id.eql?(@app_info[:node_info]['id'])
157
+ return @app_info[:api].node_api_from(
158
+ node_id: node_id,
159
+ workspace_id: @app_info[:workspace_id],
160
+ workspace_name: @app_info[:workspace_name])
161
+ end
162
+ Log.log.warn{"cannot resolve link with node id #{node_id}"}
163
+ return nil
164
+ end
165
+
166
+ # Recursively browse in a folder (with non-recursive method)
167
+ # sub folders are processed if the processing method returns true
168
+ # @param state [Object] state object sent to processing method
169
+ # @param top_file_id [String] file id to start at (default = access key root file id)
170
+ # @param top_file_path [String] path of top folder (default = /)
171
+ # @param block [Proc] processing method, arguments: entry, path, state
172
+ def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
173
+ Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
174
+ Aspera.assert(block){'Missing block'}
175
+ # start at top folder
176
+ folders_to_explore = [{id: top_file_id, path: top_file_path}]
177
+ Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
178
+ until folders_to_explore.empty?
179
+ current_item = folders_to_explore.shift
180
+ Log.log.debug{"searching #{current_item[:path]}".bg_green}
181
+ # get folder content
182
+ folder_contents =
183
+ begin
184
+ read("files/#{current_item[:id]}/files")[:data]
185
+ rescue StandardError => e
186
+ Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
187
+ []
188
+ end
189
+ Log.log.debug{Log.dump(:folder_contents, folder_contents)}
190
+ folder_contents.each do |entry|
191
+ relative_path = File.join(current_item[:path], entry['name'])
192
+ Log.log.debug{"process_folder_tree checking #{relative_path}"}
193
+ # continue only if method returns true
194
+ next unless yield(entry, relative_path, state)
195
+ # entry type is file, folder or link
196
+ case entry['type']
197
+ when 'folder'
198
+ folders_to_explore.push({id: entry['id'], path: relative_path})
199
+ when 'link'
200
+ node_id_to_node(entry['target_node_id'])&.process_folder_tree(
201
+ state: state,
202
+ top_file_id: entry['target_id'],
203
+ top_file_path: relative_path,
204
+ &block)
205
+ end
206
+ end
207
+ end
208
+ end # process_folder_tree
209
+
210
+ # Navigate the path from given file id
211
+ # @param top_file_id [String] id initial file id
212
+ # @param path [String] file path
213
+ # @return [Hash] {.api,.file_id}
214
+ def resolve_api_fid(top_file_id, path)
215
+ Aspera.assert_type(top_file_id, String)
216
+ process_last_link = path.end_with?(PATH_SEPARATOR)
217
+ path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
218
+ return {api: self, file_id: top_file_id} if path_elements.empty?
219
+ resolve_state = {path: path_elements, result: nil}
220
+ process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
221
+ # this block is called recursively for each entry in folder
222
+ # stop digging here if not in right path
223
+ next false unless entry['name'].eql?(state[:path].first)
224
+ # ok it matches, so we remove the match
225
+ state[:path].shift
226
+ case entry['type']
227
+ when 'file'
228
+ # file must be terminal
229
+ raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
230
+ # it's terminal, we found it
231
+ state[:result] = {api: self, file_id: entry['id']}
232
+ next false
233
+ when 'folder'
234
+ if state[:path].empty?
235
+ # we found it
236
+ state[:result] = {api: self, file_id: entry['id']}
237
+ next false
238
+ end
239
+ when 'link'
240
+ if state[:path].empty?
241
+ if process_last_link
242
+ # we found it
243
+ other_node = node_id_to_node(entry['target_node_id'])
244
+ raise 'cannot resolve link' if other_node.nil?
245
+ state[:result] = {api: other_node, file_id: entry['target_id']}
246
+ else
247
+ # we found it but we do not process the link
248
+ state[:result] = {api: self, file_id: entry['id']}
249
+ end
250
+ next false
251
+ end
252
+ else
253
+ Log.log.warn{"Unknown element type: #{entry['type']}"}
254
+ end
255
+ # continue to dig folder
256
+ next true
257
+ end
258
+ raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
259
+ return resolve_state[:result]
260
+ end
261
+
262
+ def find_files(top_file_id, test_block)
263
+ Log.log.debug{"find_files: file id=#{top_file_id}"}
264
+ find_state = {found: [], test_block: test_block}
265
+ process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
266
+ state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
267
+ # test all files deeply
268
+ true
269
+ end
270
+ return find_state[:found]
271
+ end
272
+
273
+ def refreshed_transfer_token
274
+ return oauth_token(force_refresh: true)
275
+ end
276
+
277
+ # Create transfer spec for gen4
278
+ def transfer_spec_gen4(file_id, direction, ts_merge=nil)
279
+ ak_name = nil
280
+ ak_token = nil
281
+ case auth_params[:type]
282
+ when :basic
283
+ ak_name = auth_params[:username]
284
+ Aspera.assert(auth_params[:password]){'no secret in node object'}
285
+ ak_token = Rest.basic_token(auth_params[:username], auth_params[:password])
286
+ when :oauth2
287
+ ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
288
+ # TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
289
+ # get bearer token, possibly use cache
290
+ ak_token = oauth_token(force_refresh: false)
291
+ else Aspera.error_unexpected_value(auth_params[:type])
292
+ end
293
+ transfer_spec = {
294
+ 'direction' => direction,
295
+ 'token' => ak_token,
296
+ 'tags' => {
297
+ Transfer::Spec::TAG_RESERVED => {
298
+ 'node' => {
299
+ 'access_key' => ak_name,
300
+ 'file_id' => file_id
301
+ } # node
302
+ } # aspera
303
+ } # tags
304
+ }
305
+ # add specials tags (cos)
306
+ add_tspec_info(transfer_spec)
307
+ transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
308
+ # add application specific tags (AoC)
309
+ app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
310
+ # add remote host info
311
+ if self.class.use_standard_ports
312
+ # get default TCP/UDP ports and transfer user
313
+ transfer_spec.merge!(Transfer::Spec::AK_TSPEC_BASE)
314
+ # by default: same address as node API
315
+ transfer_spec['remote_host'] = URI.parse(base_url).host
316
+ # AoC allows specification of other url
317
+ if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
318
+ transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
319
+ end
320
+ info = read('info')[:data]
321
+ # get the transfer user from info on access key
322
+ transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
323
+ # get settings from name.value array to hash key.value
324
+ settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
325
+ # check WSS ports
326
+ %w[wss_enabled wss_port].each do |i|
327
+ transfer_spec[i] = settings[i] if settings.key?(i)
328
+ end if settings.is_a?(Hash)
329
+ else
330
+ # retrieve values from API (and keep a copy/cache)
331
+ @std_t_spec_cache ||= create(
332
+ 'files/download_setup',
333
+ {transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
334
+ )[:data]['transfer_specs'].first['transfer_spec']
335
+ # copy some parts
336
+ TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
337
+ end
338
+ Log.log.warn{"Expected transfer user: #{Transfer::Spec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
339
+ unless transfer_spec['remote_user'].eql?(Transfer::Spec::ACCESS_KEY_TRANSFER_USER)
340
+ return transfer_spec
341
+ end
342
+ end
343
+ end
344
+ end
data/lib/aspera/ascmd.rb CHANGED
@@ -10,8 +10,20 @@ module Aspera
10
10
  # execute: "ascmd -h" to get syntax
11
11
  # Note: "ls" can take filters: as_ls -f *.txt -f *.bin /
12
12
  class AsCmd
13
+ # number of arguments for each operation
14
+ OPS_ARGS = {
15
+ cp: 2,
16
+ df: 0,
17
+ du: 1,
18
+ info: nil,
19
+ ls: 1,
20
+ md5sum: 1,
21
+ mkdir: 1,
22
+ mv: 2,
23
+ rm: 1
24
+ }.freeze
13
25
  # list of supported actions
14
- OPERATIONS = %i[ls rm mv du info mkdir cp df md5sum].freeze
26
+ OPERATIONS = OPS_ARGS.keys.freeze
15
27
 
16
28
  # @param command_executor [Object] provides the "execute" method, taking a command to execute, and stdin to feed to it, typically: ssh or local
17
29
  def initialize(command_executor)
@@ -22,16 +34,36 @@ module Aspera
22
34
  # @param [Symbol] one of OPERATIONS
23
35
  # @param [Array] parameters for "as" command
24
36
  # @return result of command, type depends on command (bool, array, hash)
25
- def execute_single(action_sym, arguments=nil)
37
+ def execute_single(action_sym, arguments)
38
+ arguments = [] if arguments.nil?
39
+ Log.log.debug{"execute_single:#{action_sym}:#{arguments}"}
40
+ Aspera.assert_type(action_sym, Symbol)
41
+ Aspera.assert_type(arguments, Array)
42
+ Aspera.assert(arguments.all?(String), 'arguments must be strings')
43
+ # lines of commands (String's)
44
+ command_lines = []
26
45
  # add "as_" command
27
- main_command = ["as_#{action_sym}"]
28
- arguments&.each do |v|
46
+ main_command = "as_#{action_sym}"
47
+ arg_batches =
48
+ if OPS_ARGS[action_sym].nil? || OPS_ARGS[action_sym].zero?
49
+ [arguments]
50
+ else
51
+ # split arguments into batches
52
+ arguments.each_slice(OPS_ARGS[action_sym]).to_a
53
+ end
54
+ arg_batches.each do |args|
55
+ command = [main_command]
29
56
  # enclose arguments in double quotes, protect backslash and double quotes
30
- main_command.push(%Q{"#{v.gsub(/["\\]/n){|s|"\\#{s}"}}"})
57
+ args.each do |v|
58
+ command.push(%Q{"#{v.gsub(/["\\]/){|s|"\\#{s}"}}"})
59
+ end
60
+ command_lines.push(command.join(' '))
31
61
  end
62
+ command_lines.push('as_exit')
63
+ command_lines.push('')
32
64
  # execute the main command and then exit
33
- stdin_input = [main_command.join(' '), 'as_exit', ''].join("\n")
34
- Log.log.debug{"execute_single:#{stdin_input}"}
65
+ stdin_input = command_lines.join("\n")
66
+ Log.log.trace1{"execute_single:#{stdin_input}"}
35
67
  # execute, get binary output
36
68
  byte_buffer = @command_executor.execute('ascmd', stdin_input).unpack('C*')
37
69
  raise 'ERROR: empty answer from server' if byte_buffer.empty?
@@ -53,7 +85,11 @@ module Aspera
53
85
  end
54
86
  end
55
87
  # for info, second overrides first, so restore it
56
- case result.keys.length; when 0 then result = system_info; when 1 then result = result[result.keys.first]; else error_unexpected_value(result.keys.length); end
88
+ case result.keys.length
89
+ when 0 then result = system_info
90
+ when 1 then result = result[result.keys.first]
91
+ else Aspera.error_unexpected_value(result.keys.length)
92
+ end
57
93
  # raise error as exception
58
94
  raise Error.new(result[:errno], result[:errstr], action_sym, arguments) if
59
95
  result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{|i|i[:name]}.sort)
@@ -66,7 +102,7 @@ module Aspera
66
102
  super(); @errno = errno; @errstr = errstr; @command = cmd; @arguments = arguments; end # rubocop:disable Style/Semicolon
67
103
 
68
104
  def message; "ascmd: #{@errstr} (#{@errno})"; end
69
- def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments.join(',')}"; end
105
+ def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments&.join(',')}"; end
70
106
  end # Error
71
107
 
72
108
  # description of result structures (see ascmdtypes.h). Base types are big endian
@@ -174,7 +210,7 @@ module Aspera
174
210
  end
175
211
  end
176
212
  end
177
- else error_unexpected_value(type_descr[:decode])
213
+ else Aspera.error_unexpected_value(type_descr[:decode])
178
214
  end # is_a
179
215
  return result
180
216
  end
@@ -3,7 +3,7 @@
3
3
  # cspell:ignore protobuf ckpt
4
4
  require 'aspera/environment'
5
5
  require 'aspera/data_repository'
6
- require 'aspera/fasp/products'
6
+ require 'aspera/ascp/products'
7
7
  require 'aspera/log'
8
8
  require 'aspera/assert'
9
9
  require 'aspera/web_server_simple'
@@ -16,7 +16,7 @@ require 'fileutils'
16
16
  require 'openssl'
17
17
 
18
18
  module Aspera
19
- module Fasp
19
+ module Ascp
20
20
  # Singleton that tells where to find ascp and other local resources (keys..) , using the "path(:name)" method.
21
21
  # It is used by object : AgentDirect to find necessary resources
22
22
  # By default it takes the first Aspera product found
@@ -139,10 +139,10 @@ module Aspera
139
139
  check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
140
140
  end
141
141
  file = k.eql?(:fallback_certificate) ? file_cert : file_key
142
- else error_unexpected_value(k)
142
+ else Aspera.error_unexpected_value(k)
143
143
  end
144
144
  return nil if file_is_optional && !File.exist?(file)
145
- assert(File.exist?(file)){"no such file: #{file}"}
145
+ Aspera.assert(File.exist?(file)){"no such file: #{file}"}
146
146
  return file
147
147
  end
148
148
 
@@ -283,6 +283,6 @@ module Aspera
283
283
  def transferd_filepath
284
284
  return File.join(sdk_folder, 'asperatransferd' + Environment.exe_extension) # cspell:disable-line
285
285
  end
286
- end # Installation
286
+ end
287
287
  end
288
288
  end
@@ -1,8 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Aspera
4
- module Fasp
5
- # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
4
+ module Ascp
5
+ # processing of ascp management port events
6
6
  class Management
7
7
  # cspell: disable
8
8
  OPERATIONS = %w[
@@ -202,12 +202,7 @@ module Aspera
202
202
  # translates mgt port event into (enhanced) typed event
203
203
  def enhanced_event_format(event)
204
204
  return event.keys.each_with_object({}) do |e, h|
205
- # capital_to_snake_case
206
- new_name = e
207
- .gsub(/([a-z\d])([A-Z])/, '\1_\2')
208
- .gsub(/([a-z\d])(usec)$/, '\1_\2')
209
- .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
210
- .downcase
205
+ new_name = e.capital_to_snake.gsub(/(usec)$/, '_\1').downcase
211
206
  value = event[e]
212
207
  value = value.to_i if INTEGER_FIELDS.include?(e)
213
208
  value = value.eql?(BOOLEAN_TRUE) if BOOLEAN_FIELDS.include?(e)
@@ -4,7 +4,7 @@
4
4
  require 'aspera/environment'
5
5
 
6
6
  module Aspera
7
- module Fasp
7
+ module Ascp
8
8
  # find Aspera standard products installation in standard paths
9
9
  class Products
10
10
  # known product names