aspera-cli 4.21.2 → 4.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +402 -374
  5. data/CONTRIBUTING.md +6 -10
  6. data/README.md +1018 -687
  7. data/lib/aspera/agent/base.rb +9 -5
  8. data/lib/aspera/agent/connect.rb +30 -28
  9. data/lib/aspera/agent/desktop.rb +29 -25
  10. data/lib/aspera/agent/direct.rb +137 -125
  11. data/lib/aspera/agent/httpgw.rb +22 -26
  12. data/lib/aspera/agent/node.rb +14 -11
  13. data/lib/aspera/agent/transferd.rb +6 -2
  14. data/lib/aspera/api/aoc.rb +15 -18
  15. data/lib/aspera/api/cos_node.rb +1 -1
  16. data/lib/aspera/api/httpgw.rb +15 -7
  17. data/lib/aspera/api/node.rb +6 -4
  18. data/lib/aspera/ascmd.rb +17 -9
  19. data/lib/aspera/ascp/installation.rb +21 -19
  20. data/lib/aspera/ascp/management.rb +1 -1
  21. data/lib/aspera/assert.rb +14 -5
  22. data/lib/aspera/cli/error.rb +2 -2
  23. data/lib/aspera/cli/extended_value.rb +38 -19
  24. data/lib/aspera/cli/formatter.rb +48 -48
  25. data/lib/aspera/cli/hints.rb +10 -2
  26. data/lib/aspera/cli/main.rb +190 -168
  27. data/lib/aspera/cli/manager.rb +16 -16
  28. data/lib/aspera/cli/plugin.rb +24 -21
  29. data/lib/aspera/cli/plugin_factory.rb +1 -1
  30. data/lib/aspera/cli/plugins/alee.rb +1 -1
  31. data/lib/aspera/cli/plugins/aoc.rb +173 -126
  32. data/lib/aspera/cli/plugins/ats.rb +19 -17
  33. data/lib/aspera/cli/plugins/config.rb +87 -98
  34. data/lib/aspera/cli/plugins/console.rb +5 -3
  35. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  36. data/lib/aspera/cli/plugins/faspex5.rb +104 -80
  37. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  38. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  39. data/lib/aspera/cli/plugins/node.rb +336 -205
  40. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  41. data/lib/aspera/cli/plugins/preview.rb +3 -3
  42. data/lib/aspera/cli/plugins/server.rb +7 -6
  43. data/lib/aspera/cli/plugins/shares.rb +5 -5
  44. data/lib/aspera/cli/sync_actions.rb +19 -18
  45. data/lib/aspera/cli/transfer_agent.rb +11 -15
  46. data/lib/aspera/cli/transfer_progress.rb +2 -2
  47. data/lib/aspera/cli/version.rb +1 -1
  48. data/lib/aspera/command_line_builder.rb +116 -95
  49. data/lib/aspera/coverage.rb +4 -3
  50. data/lib/aspera/data_repository.rb +1 -0
  51. data/lib/aspera/environment.rb +7 -6
  52. data/lib/aspera/faspex_gw.rb +14 -14
  53. data/lib/aspera/faspex_postproc.rb +7 -6
  54. data/lib/aspera/hash_ext.rb +2 -2
  55. data/lib/aspera/json_rpc.rb +1 -1
  56. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  57. data/lib/aspera/keychain/factory.rb +41 -0
  58. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  59. data/lib/aspera/keychain/macos_security.rb +19 -11
  60. data/lib/aspera/log.rb +29 -34
  61. data/lib/aspera/nagios.rb +6 -6
  62. data/lib/aspera/node_simulator.rb +8 -8
  63. data/lib/aspera/oauth/base.rb +10 -6
  64. data/lib/aspera/oauth/factory.rb +6 -6
  65. data/lib/aspera/oauth/url_json.rb +6 -6
  66. data/lib/aspera/persistency_action_once.rb +6 -4
  67. data/lib/aspera/persistency_folder.rb +2 -2
  68. data/lib/aspera/preview/file_types.rb +40 -33
  69. data/lib/aspera/preview/generator.rb +1 -1
  70. data/lib/aspera/preview/options.rb +16 -16
  71. data/lib/aspera/preview/terminal.rb +3 -3
  72. data/lib/aspera/preview/utils.rb +11 -13
  73. data/lib/aspera/products/connect.rb +2 -1
  74. data/lib/aspera/products/desktop.rb +1 -1
  75. data/lib/aspera/products/transferd.rb +1 -1
  76. data/lib/aspera/proxy_auto_config.rb +2 -2
  77. data/lib/aspera/rest.rb +70 -50
  78. data/lib/aspera/rest_error_analyzer.rb +1 -0
  79. data/lib/aspera/rest_errors_aspera.rb +1 -1
  80. data/lib/aspera/secret_hider.rb +5 -5
  81. data/lib/aspera/ssh.rb +5 -5
  82. data/lib/aspera/temp_file_manager.rb +1 -0
  83. data/lib/aspera/timer_limiter.rb +7 -5
  84. data/lib/aspera/transfer/async_conf.schema.yaml +716 -0
  85. data/lib/aspera/transfer/convert.rb +29 -0
  86. data/lib/aspera/transfer/error_info.rb +66 -66
  87. data/lib/aspera/transfer/parameters.rb +13 -68
  88. data/lib/aspera/transfer/spec.rb +5 -6
  89. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  90. data/lib/aspera/transfer/spec_doc.rb +62 -0
  91. data/lib/aspera/transfer/sync.rb +37 -76
  92. data/lib/aspera/transfer/sync_instance.schema.yaml +20 -0
  93. data/lib/aspera/transfer/sync_session.schema.yaml +86 -0
  94. data/lib/aspera/transfer/uri.rb +6 -6
  95. data/lib/aspera/uri_reader.rb +1 -1
  96. data/lib/aspera/web_auth.rb +1 -1
  97. data/lib/aspera/web_server_simple.rb +53 -44
  98. data.tar.gz.sig +0 -0
  99. metadata +38 -7
  100. metadata.gz.sig +0 -0
  101. data/examples/build_package.sh +0 -28
  102. data/examples/dascli +0 -30
  103. data/examples/get_proto_file.rb +0 -8
  104. data/examples/proxy.pac +0 -60
  105. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -9,9 +9,28 @@ require 'aspera/assert'
9
9
  module Aspera
10
10
  module Agent
11
11
  class Httpgw < Base
12
- # start FASP transfer based on transfer spec (hash table)
13
- # note that it is asynchronous
12
+ def initialize(
13
+ url:,
14
+ api_version: Api::Httpgw::API_V2,
15
+ upload_chunk_size: 64_000,
16
+ synchronous: false,
17
+ **base_options
18
+ )
19
+ super(**base_options)
20
+ @gw_api = Api::Httpgw.new(
21
+ # remove /v1 from end of user-provided GW url: we need the base url only
22
+ url: url,
23
+ api_version: api_version,
24
+ upload_chunk_size: upload_chunk_size,
25
+ synchronous: synchronous,
26
+ notify_cb: ->(*pa, **ka){notify_progress(*pa, **ka)}
27
+ )
28
+ end
29
+
30
+ # Start FASP transfer based on transfer spec (hash table)
31
+ # note that this should be asynchronous, but it is not
14
32
  # HTTP download only supports file list
33
+ # :reek:UnusedParameters token_regenerator
15
34
  def start_transfer(transfer_spec, token_regenerator: nil)
16
35
  raise 'GW URL must be set' if @gw_api.nil?
17
36
  Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
@@ -27,35 +46,12 @@ module Aspera
27
46
  end
28
47
  end
29
48
 
30
- # wait for completion of all jobs started
49
+ # Wait for completion of all jobs started
31
50
  # @return list of :success or error message
32
51
  def wait_for_transfers_completion
33
52
  # well ... transfer was done in "start"
34
53
  return [:success]
35
54
  end
36
-
37
- # TODO: is that useful?
38
- # def url=(api_url); end
39
-
40
- private
41
-
42
- def initialize(
43
- url:,
44
- api_version: Api::Httpgw::API_V2,
45
- upload_chunk_size: 64_000,
46
- synchronous: false,
47
- **base_options
48
- )
49
- super(**base_options)
50
- @gw_api = Api::Httpgw.new(
51
- # remove /v1 from end of user-provided GW url: we need the base url only
52
- url: url,
53
- api_version: api_version,
54
- upload_chunk_size: upload_chunk_size,
55
- synchronous: synchronous,
56
- notify_cb: ->(*pa, **ka) { notify_progress(*pa, **ka) }
57
- )
58
- end
59
55
  end
60
56
  end
61
57
  end
@@ -13,10 +13,10 @@ module Aspera
13
13
  # this singleton class is used by the CLI to provide a common interface to start a transfer
14
14
  # before using it, the use must set the `node_api` member.
15
15
  class Node < Base
16
- # @param url [String] the base url of the node api
17
- # @param username [String] the username to use for the node api
18
- # @param password [String] the password to use for the node api
19
- # @param root_id [String] root file id if the node is an access key
16
+ # @param url [String] the base url of the node api
17
+ # @param username [String] the username to use for the node api
18
+ # @param password [String] the password to use for the node api
19
+ # @param root_id [String] root file id if the node is an access key
20
20
  # @param base_options [Hash] options for base class
21
21
  def initialize(
22
22
  url:,
@@ -28,7 +28,7 @@ module Aspera
28
28
  super(**base_options)
29
29
  # root id is required for access key
30
30
  @root_id = root_id
31
- rest_params = { base_url: url}
31
+ rest_params = {base_url: url}
32
32
  if OAuth::Factory.bearer?(password)
33
33
  Aspera.assert(!@root_id.nil?){'root_id not allowed for access key'}
34
34
  rest_params[:headers] = Api::Node.bearer_headers(password, access_key: username)
@@ -44,13 +44,8 @@ module Aspera
44
44
  @transfer_id = nil
45
45
  end
46
46
 
47
- # used internally to ensure node api is set before using.
48
- def node_api_
49
- Aspera.assert(!@node_api.nil?){'Before using this object, set the node_api attribute to a Aspera::Rest object'}
50
- return @node_api
51
- end
52
-
53
47
  # generic method
48
+ # :reek:UnusedParameters token_regenerator
54
49
  def start_transfer(transfer_spec, token_regenerator: nil)
55
50
  # add root id if access key
56
51
  if !@root_id.nil?
@@ -119,6 +114,14 @@ module Aspera
119
114
  # TODO: get status of sessions
120
115
  return []
121
116
  end
117
+
118
+ private
119
+
120
+ # used internally to ensure node api is set before using.
121
+ def node_api_
122
+ Aspera.assert(!@node_api.nil?){'Before using this object, set the node_api attribute to a Aspera::Rest object'}
123
+ return @node_api
124
+ end
122
125
  end
123
126
  end
124
127
  end
@@ -31,6 +31,7 @@ module Aspera
31
31
  **base
32
32
  )
33
33
  super(**base)
34
+ @transfer_id = nil
34
35
  @stop = stop
35
36
  is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
36
37
  raise 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
@@ -94,7 +95,7 @@ module Aspera
94
95
  end
95
96
  Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
96
97
  Process.detach(@daemon_pid) unless @stop
97
- at_exit {shutdown}
98
+ at_exit{shutdown}
98
99
  # update port for next connection attempt (if auto high port was requested)
99
100
  daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
100
101
  # local daemon started, try again
@@ -102,6 +103,7 @@ module Aspera
102
103
  end
103
104
  end
104
105
 
106
+ # :reek:UnusedParameters token_regenerator
105
107
  def start_transfer(transfer_spec, token_regenerator: nil)
106
108
  # create a transfer request
107
109
  transfer_request = ::Transferd::Api::TransferRequest.new(
@@ -156,10 +158,12 @@ module Aspera
156
158
  stop_daemon if @stop
157
159
  end
158
160
 
161
+ private
162
+
159
163
  def stop_daemon
160
164
  if !@daemon_pid.nil?
161
165
  Log.log.debug("Stopping daemon #{@daemon_pid}")
162
- Process.kill('INT', @daemon_pid)
166
+ Process.kill(:INT, @daemon_pid)
163
167
  _, status = Process.wait2(@daemon_pid)
164
168
  Log.log.debug("daemon stopped #{status}")
165
169
  @daemon_pid = nil
@@ -22,7 +22,7 @@ module Aspera
22
22
  MAX_AOC_URL_REDIRECT = 10
23
23
  CLIENT_ID_PREFIX = 'aspera.'
24
24
  # Well-known AoC global client apps
25
- GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{|i|i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
25
+ GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{ |i| i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
26
26
  # cookie prefix so that console can decode identity
27
27
  COOKIE_PREFIX_CONSOLE_AOC = 'aspera.aoc'
28
28
  # path in URL of public links
@@ -34,6 +34,8 @@ module Aspera
34
34
  # types of events for shared folder creation
35
35
  # Node events: permission.created permission.modified permission.deleted
36
36
  PERMISSIONS_CREATED = ['permission.created'].freeze
37
+ # Special name when creating workspace shared folders
38
+ ID_AK_ADMIN = 'ASPERA_ACCESS_KEY_ADMIN'
37
39
 
38
40
  private_constant :MAX_AOC_URL_REDIRECT,
39
41
  :CLIENT_ID_PREFIX,
@@ -43,7 +45,8 @@ module Aspera
43
45
  :JWT_AUDIENCE,
44
46
  :OAUTH_API_SUBPATH,
45
47
  :USER_INFO_FIELDS_MIN,
46
- :PERMISSIONS_CREATED
48
+ :PERMISSIONS_CREATED,
49
+ :ID_AK_ADMIN
47
50
 
48
51
  # various API scopes supported
49
52
  SCOPE_FILES_SELF = 'self'
@@ -237,7 +240,7 @@ module Aspera
237
240
  Log.log.debug{"ignoring error: #{e}"}
238
241
  {}
239
242
  end
240
- USER_INFO_FIELDS_MIN.each{|f|@cache_user_info[f] = nil if @cache_user_info[f].nil?}
243
+ USER_INFO_FIELDS_MIN.each{ |f| @cache_user_info[f] = nil if @cache_user_info[f].nil?}
241
244
  return @cache_user_info
242
245
  end
243
246
 
@@ -283,7 +286,7 @@ module Aspera
283
286
  if ws_info.nil?
284
287
  {
285
288
  id: nil,
286
- name: 'Shared folders'
289
+ name: "Shared #{application}"
287
290
  }
288
291
  else
289
292
  {
@@ -386,10 +389,10 @@ module Aspera
386
389
  Aspera.assert(field.key?('name')){'metadata field must have name'}
387
390
  Aspera.assert(field.key?('values')){'metadata field must have values'}
388
391
  Aspera.assert_type(field['values'], Array){'metadata field values'}
389
- Aspera.assert(!meta_schema.none?{|i|i['name'].eql?(field['name'])}){"unknown metadata field: #{field['name']}"}
392
+ Aspera.assert(!meta_schema.none?{ |i| i['name'].eql?(field['name'])}){"unknown metadata field: #{field['name']}"}
390
393
  end
391
394
  meta_schema.each do |field|
392
- provided = pkg_meta.select{|i|i['name'].eql?(field['name'])}
395
+ provided = pkg_meta.select{ |i| i['name'].eql?(field['name'])}
393
396
  raise "only one field with name #{field['name']} allowed" if provided.count > 1
394
397
  raise "missing mandatory field: #{field['name']}" if field['required'] && provided.empty?
395
398
  end
@@ -519,7 +522,7 @@ module Aspera
519
522
  # Console cookie
520
523
  ################
521
524
  # we are sure that fields are not nil
522
- cookie_elements = [app_info[:app], current_user_info['name'] || 'public link', current_user_info['email'] || 'none'].map{|e|Base64.strict_encode64(e)}
525
+ cookie_elements = [app_info[:app], current_user_info['name'] || 'public link', current_user_info['email'] || 'none'].map{ |e| Base64.strict_encode64(e)}
523
526
  cookie_elements.unshift(COOKIE_PREFIX_CONSOLE_AOC)
524
527
  transfer_spec['cookie'] = cookie_elements.join(':')
525
528
  # Application tags
@@ -546,17 +549,12 @@ module Aspera
546
549
  transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['app'] = app_info[:app]
547
550
  end
548
551
 
549
- ID_AK_ADMIN = 'ASPERA_ACCESS_KEY_ADMIN'
550
552
  # Callback from Plugins::Node
551
553
  # add application specific tags to permissions creation
552
554
  # @param perm_data [Hash] parameters for creating permissions
553
555
  # @param app_info [Hash] application information
554
556
  def permissions_set_create_params(perm_data:, app_info:)
555
- # workspace shared folder:
556
- # access_id = "#{ID_AK_ADMIN}_WS_#{app_info[:workspace_id]}"
557
557
  defaults = {
558
- # 'access_type' => 'user', # mandatory: user or group
559
- # 'access_id' => access_id, # id of user or group or special
560
558
  'tags' => {
561
559
  Transfer::Spec::TAG_RESERVED => {
562
560
  'files' => {
@@ -567,8 +565,6 @@ module Aspera
567
565
  'shared_by_user_id' => current_user_info['id'],
568
566
  'shared_by_name' => current_user_info['name'],
569
567
  'shared_by_email' => current_user_info['email'],
570
- # 'shared_with_name' => access_id,
571
- # 'share_as' => new_name_for_folder,
572
568
  'access_key' => app_info[:node_info]['access_key'],
573
569
  'node' => app_info[:node_info]['name']
574
570
  }
@@ -578,19 +574,20 @@ module Aspera
578
574
  }
579
575
  perm_data.deep_merge!(defaults)
580
576
  tag_workspace = perm_data['tags'][Transfer::Spec::TAG_RESERVED]['files']['workspace']
581
- case perm_data['with']
577
+ shared_with = perm_data.delete('with')
578
+ case shared_with
582
579
  when NilClass
583
580
  when ''
581
+ # workspace shared folder
584
582
  perm_data['access_type'] = 'user'
585
583
  perm_data['access_id'] = "#{ID_AK_ADMIN}_WS_#{app_info[:workspace_id]}"
586
584
  tag_workspace['shared_with_name'] = perm_data['access_id']
587
585
  else
588
- entity_info = lookup_by_name('contacts', perm_data['with'], query: {'current_workspace_id' => app_info[:workspace_id]})
586
+ entity_info = lookup_by_name('contacts', shared_with, query: {'current_workspace_id' => app_info[:workspace_id]})
589
587
  perm_data['access_type'] = entity_info['source_type']
590
588
  perm_data['access_id'] = entity_info['source_id']
591
- tag_workspace['shared_with_name'] = entity_info['email']
589
+ tag_workspace['shared_with_name'] = entity_info['email'] # TODO: check that ???
592
590
  end
593
- perm_data.delete('with')
594
591
  if perm_data.key?('as')
595
592
  tag_workspace['share_as'] = perm_data['as']
596
593
  perm_data.delete('as')
@@ -88,7 +88,7 @@ module Aspera
88
88
  )
89
89
  # get delegated token to be placed in rest call header and in transfer tags
90
90
  @storage_credentials['token'][TOKEN_FIELD] = delegated_oauth.token
91
- @headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
91
+ headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
92
92
  end
93
93
  end
94
94
  end
@@ -171,7 +171,7 @@ module Aspera
171
171
  cond_var: ConditionVariable.new
172
172
  }
173
173
  # start read thread after handshake
174
- @ws_read_thread = Thread.new {process_read_thread}
174
+ @ws_read_thread = Thread.new{process_read_thread}
175
175
  @notify_cb&.call(:session_start, session_id: session_id)
176
176
  @notify_cb&.call(:session_size, session_id: session_id, info: total_bytes_to_transfer)
177
177
  sleep(1)
@@ -244,18 +244,22 @@ module Aspera
244
244
  end
245
245
 
246
246
  def download(transfer_spec)
247
- transfer_spec['zip_required'] ||= false
248
247
  transfer_spec['source_root'] ||= '/'
248
+ default_file_name = transfer_spec['paths'].first['source']
249
+ source_is_folder = %w[. /].include?(default_file_name)
250
+ default_file_name = 'http_download' if source_is_folder
251
+ transfer_spec['zip_required'] ||= source_is_folder || transfer_spec['paths'].length > 1
249
252
  # is normally provided by application, like package name
250
253
  if !transfer_spec.key?('download_name')
251
254
  # by default it is the name of first file
252
- download_name = File.basename(transfer_spec['paths'].first['source'], '.*')
253
- # ands add indication of number of files if there is more than one
255
+ download_name = File.basename(default_file_name, '.*')
256
+ # add indication of number of files if there is more than one
254
257
  if transfer_spec['paths'].length > 1
255
258
  download_name += " #{transfer_spec['paths'].length} Files"
256
259
  end
257
260
  transfer_spec['download_name'] = download_name
258
261
  end
262
+ # start transfer session on httpgw
259
263
  creation = create('download', {'transfer_spec' => transfer_spec})
260
264
  transfer_uuid = creation['url'].split('/').last
261
265
  file_name =
@@ -264,7 +268,7 @@ module Aspera
264
268
  transfer_spec['download_name'] + '.zip'
265
269
  else
266
270
  # it is a plain file if we don't require zip and there is only one file
267
- File.basename(transfer_spec['paths'].first['source'])
271
+ File.basename(default_file_name)
268
272
  end
269
273
  file_path = File.join(transfer_spec['destination_root'], file_name)
270
274
  call(operation: 'GET', subpath: "download/#{transfer_uuid}", save_to_file: file_path)
@@ -309,11 +313,15 @@ module Aspera
309
313
  # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
310
314
  # is the latest supported? else revert to old api
311
315
  if !@upload_version.eql?(API_V1)
312
- if !@api_info['endpoints'].any?{|i|i.include?(@upload_version)}
316
+ if !@api_info['endpoints'].any?{ |i| i.include?(@upload_version)}
313
317
  Log.log.warn{"API version #{@upload_version} not supported, reverting to #{API_V1}"}
314
318
  @upload_version = API_V1
315
319
  end
316
320
  end
321
+ @shared_info = nil
322
+ @ws_handshake = nil
323
+ @ws_io = nil
324
+ @ws_read_thread = nil
317
325
  end
318
326
 
319
327
  private
@@ -373,7 +381,7 @@ module Aspera
373
381
  raise "File not found: #{source_path}"
374
382
  end
375
383
  end
376
- transfer_spec['paths'] = files_to_send.map{|i|{'source' => i[:name]}}
384
+ transfer_spec['paths'] = files_to_send.map{ |i| {'source' => i[:name]}}
377
385
  files_to_send.push(total_bytes_to_transfer)
378
386
  return files_to_send
379
387
  end
@@ -27,7 +27,7 @@ module Aspera
27
27
  :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT,
28
28
  :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
29
29
 
30
- # node api permissions
30
+ # Node API permissions
31
31
  ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
32
32
  HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
33
33
  HEADER_X_TOTAL_COUNT = 'X-Total-Count'
@@ -38,7 +38,7 @@ module Aspera
38
38
  PATH_SEPARATOR = '/'
39
39
 
40
40
  # register node special token decoder
41
- OAuth::Factory.instance.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
41
+ OAuth::Factory.instance.register_decoder(lambda{ |token| Node.decode_bearer_token(token)})
42
42
 
43
43
  # class instance variable, access with accessors on class
44
44
  @use_standard_ports = true
@@ -81,6 +81,7 @@ module Aspera
81
81
  end
82
82
 
83
83
  # Create an Aspera Node bearer token
84
+ # @param access_key [String] Access key identifier
84
85
  # @param payload [String] JSON payload to be included in the token
85
86
  # @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
86
87
  def bearer_token(access_key:, payload:, private_key:)
@@ -106,6 +107,7 @@ module Aspera
106
107
  ].join("\n")))
107
108
  end
108
109
 
110
+ # Decode an Aspera Node bearer token
109
111
  def decode_bearer_token(token)
110
112
  return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
111
113
  end
@@ -149,7 +151,7 @@ module Aspera
149
151
 
150
152
  # Call node API, possibly adding cache control header, as globally specified
151
153
  def read_with_cache(subpath, query=nil)
152
- headers = {'Accept' => 'application/json'}
154
+ headers = {'Accept' => Rest::MIME_JSON}
153
155
  headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless self.class.use_node_cache
154
156
  return call(
155
157
  operation: 'GET',
@@ -352,7 +354,7 @@ module Aspera
352
354
  # get the transfer user from info on access key
353
355
  transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
354
356
  # get settings from name.value array to hash key.value
355
- settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
357
+ settings = info['settings']&.each_with_object({}){ |i, h| h[i['name']] = i['value']}
356
358
  # check WSS ports
357
359
  Transfer::Spec::WSS_FIELDS.each do |i|
358
360
  transfer_spec[i] = settings[i] if settings.key?(i)
data/lib/aspera/ascmd.rb CHANGED
@@ -10,7 +10,7 @@ module Aspera
10
10
  # execute: "ascmd -h" to get syntax
11
11
  # Note: "ls" can take filters: as_ls -f *.txt -f *.bin /
12
12
  class AsCmd
13
- # number of arguments for each operation
13
+ # number of arguments for each operation (to allow splitting into batches)
14
14
  OPS_ARGS = {
15
15
  cp: 2,
16
16
  df: 0,
@@ -23,11 +23,11 @@ module Aspera
23
23
  rm: 1
24
24
  }.freeze
25
25
 
26
- # protocol is based on Type-Length-Value
27
- # type start at one, but array index start at zero
26
+ # Protocol is based on Type-Length-Value
27
+ # Type start at one, but array index start at zero
28
28
  ENUM_START = 1
29
29
 
30
- # description of result structures (see ascmdtypes.h).
30
+ # Description of result structures (see ascmdtypes.h).
31
31
  # Base types are big endian
32
32
  # key = name of type
33
33
  # index in array `fields` is the type (minus ENUM_START)
@@ -79,17 +79,25 @@ module Aspera
79
79
  end
80
80
 
81
81
  # execute an "as" command on a remote server
82
+ # Version 2 allows use of reverse proxy with multiple addresses.
82
83
  # @param [Symbol] one of OPERATIONS
83
84
  # @param [Array] parameters for "as" command
84
85
  # @return result of command, type depends on command (bool, array, hash)
85
- def execute_single(action_sym, arguments)
86
+ def execute_single(action_sym, arguments, version: 1, host: nil)
86
87
  arguments = [] if arguments.nil?
87
88
  Log.log.debug{"execute_single:#{action_sym}:#{arguments}"}
88
89
  Aspera.assert_type(action_sym, Symbol)
89
90
  Aspera.assert_type(arguments, Array)
90
91
  Aspera.assert(arguments.all?(String), 'arguments must be strings')
92
+ remote_cmd = 'ascmd'
91
93
  # lines of commands (String's)
92
94
  command_lines = []
95
+ if version.eql?(2)
96
+ cmd = "as_session_init --protocol=#{version}"
97
+ cmd += " --host=#{host}" if host
98
+ command_lines.push(cmd)
99
+ remote_cmd += ' -V2'
100
+ end
93
101
  # add "as_" command
94
102
  main_command = "as_#{action_sym}"
95
103
  arg_batches =
@@ -104,7 +112,7 @@ module Aspera
104
112
  # enclose arguments in double quotes, protect backslash and double quotes
105
113
  # ascmd uses space as token separator, and optional quotes ('") or \ to escape
106
114
  args.each do |v|
107
- command.push(%Q{"#{v.gsub(/["\\]/){|s|"\\#{s}"}}"})
115
+ command.push(%Q{"#{v.gsub(/["\\]/){ |s| "\\#{s}"}}"})
108
116
  end
109
117
  command_lines.push(command.join(' '))
110
118
  end
@@ -114,7 +122,7 @@ module Aspera
114
122
  stdin_input = command_lines.join("\n")
115
123
  Log.log.trace1{"execute_single:#{stdin_input}"}
116
124
  # execute, get binary output
117
- byte_buffer = @command_executor.execute('ascmd', stdin_input).unpack('C*')
125
+ byte_buffer = @command_executor.execute(remote_cmd, stdin_input).unpack('C*')
118
126
  raise 'ERROR: empty answer from server' if byte_buffer.empty?
119
127
  # get hash or table result
120
128
  result = self.class.parse(byte_buffer, :result)
@@ -141,7 +149,7 @@ module Aspera
141
149
  end
142
150
  # raise error as exception
143
151
  raise Error.new(result[:errno], result[:errstr], action_sym, arguments) if
144
- result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{|i|i[:name]}.sort)
152
+ result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{ |i| i[:name]}.sort)
145
153
  return result
146
154
  end
147
155
 
@@ -209,7 +217,7 @@ module Aspera
209
217
  result[field_info[:name]] ||= []
210
218
  result[field_info[:name]].push(parse(typed_buffer[:buffer], field_info[:is_a], indent_level))
211
219
  when :list_tlv_list # field is an array of values in a list of buffers
212
- result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{|r|parse(r[:buffer], field_info[:is_a], indent_level)}
220
+ result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{ |r| parse(r[:buffer], field_info[:is_a], indent_level)}
213
221
  when :list_tlv_restart # field is an array of values, but a new value is started on index 1
214
222
  fl = result[field_info[:name]] = []
215
223
  parse(typed_buffer[:buffer], :blist, indent_level).map do |tb|
@@ -32,6 +32,7 @@ module Aspera
32
32
  # Installation.instance.ascp_path=""
33
33
  class Installation
34
34
  include Singleton
35
+
35
36
  DEFAULT_ASPERA_CONF = <<~END_OF_CONFIG_FILE
36
37
  <?xml version='1.0' encoding='UTF-8'?>
37
38
  <CONF version="2">
@@ -68,8 +69,8 @@ module Aspera
68
69
  # set ascp executable path
69
70
  def ascp_path=(v)
70
71
  Aspera.assert_type(v, String)
71
- Aspera.assert(!v.empty?) {'ascp path cannot be empty: check your config file'}
72
- Aspera.assert(File.exist?(v)) {"No such file: [#{v}]"}
72
+ Aspera.assert(!v.empty?){'ascp path cannot be empty: check your config file'}
73
+ Aspera.assert(File.exist?(v)){"No such file: [#{v}]"}
73
74
  @path_to_ascp = v
74
75
  end
75
76
 
@@ -89,7 +90,7 @@ module Aspera
89
90
  pl = installed_products.first
90
91
  raise "no Aspera transfer module or SDK found.\nRefer to the manual or install SDK with command:\nascli conf ascp install" if pl.nil?
91
92
  else
92
- pl = installed_products.find{|i|i[:name].eql?(product_name)}
93
+ pl = installed_products.find{ |i| i[:name].eql?(product_name)}
93
94
  raise "no such product installed: #{product_name}" if pl.nil?
94
95
  end
95
96
  self.ascp_path = pl[:ascp_path]
@@ -130,22 +131,21 @@ module Aspera
130
131
  when :ssh_private_dsa, :ssh_private_rsa
131
132
  # assume last 3 letters are type
132
133
  type = k.to_s[-3..-1].to_sym
133
- file = check_or_create_sdk_file("aspera_bypass_#{type}.pem") {DataRepository.instance.item(type)}
134
+ file = check_or_create_sdk_file("aspera_bypass_#{type}.pem"){DataRepository.instance.item(type)}
134
135
  when :aspera_license
135
- file = check_or_create_sdk_file('aspera-license') {DataRepository.instance.item(:license)}
136
+ file = check_or_create_sdk_file('aspera-license'){DataRepository.instance.item(:license)}
136
137
  when :aspera_conf
137
- file = check_or_create_sdk_file('aspera.conf') {DEFAULT_ASPERA_CONF}
138
+ file = check_or_create_sdk_file('aspera.conf'){DEFAULT_ASPERA_CONF}
138
139
  when :fallback_certificate, :fallback_private_key
139
140
  file_key = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert_private_key.pem')
140
141
  file_cert = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert.pem')
141
142
  if !File.exist?(file_key) || !File.exist?(file_cert)
142
143
  require 'openssl'
143
144
  # create new self signed certificate for http fallback
144
- cert = OpenSSL::X509::Certificate.new
145
145
  private_key = OpenSSL::PKey::RSA.new(4096)
146
- WebServerSimple.fill_self_signed_cert(cert, private_key)
147
- check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true) {private_key.to_pem}
148
- check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
146
+ cert = WebServerSimple.self_signed_cert(private_key)
147
+ check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true){private_key.to_pem}
148
+ check_or_create_sdk_file('aspera_fallback_cert.pem', force: true){cert.to_pem}
149
149
  end
150
150
  file = k.eql?(:fallback_certificate) ? file_cert : file_key
151
151
  else Aspera.error_unexpected_value(k)
@@ -166,7 +166,7 @@ module Aspera
166
166
  Aspera.assert_values(types, CLIENT_SSH_KEY_OPTIONS)
167
167
  return case types
168
168
  when :dsa_rsa, :rsa
169
- types.to_s.split('_').map{|i|Installation.instance.path("ssh_private_#{i}".to_sym)}
169
+ types.to_s.split('_').map{ |i| Installation.instance.path("ssh_private_#{i}".to_sym)}
170
170
  when :per_client
171
171
  raise 'Not yet implemented'
172
172
  end
@@ -197,6 +197,8 @@ module Aspera
197
197
  last_line = ''
198
198
  while (line = stderr.gets)
199
199
  line.chomp!
200
+ # skip lines that may have accents
201
+ next unless line.valid_encoding?
200
202
  last_line = line
201
203
  case line
202
204
  when /^DBG Path ([^ ]+) (dir|file) +: (.*)$/
@@ -212,7 +214,7 @@ module Aspera
212
214
  data['product_name'] = Regexp.last_match(1)
213
215
  data['product_version'] = Regexp.last_match(2)
214
216
  when /^LOG Initializing FASP version ([^,]+),/
215
- data['sdk_ascp_version'] = Regexp.last_match(1)
217
+ data['ascp_version'] = Regexp.last_match(1)
216
218
  end
217
219
  end
218
220
  if !thread.value.exitstatus.eql?(1) && !data.key?('root')
@@ -227,9 +229,9 @@ module Aspera
227
229
  data = {}
228
230
  File.binread(ascp_path).scan(/[\x20-\x7E]{10,}/) do |bin_string|
229
231
  if (m = bin_string.match(/OPENSSLDIR.*"(.*)"/))
230
- data['openssldir'] = m[1]
232
+ data['ascp_openssl_dir'] = m[1]
231
233
  elsif (m = bin_string.match(/OpenSSL (\d[^ -]+)/))
232
- data['openssl_version'] = m[1]
234
+ data['ascp_openssl_version'] = m[1]
233
235
  end
234
236
  end if File.file?(ascp_path)
235
237
  return data
@@ -247,10 +249,10 @@ module Aspera
247
249
  def sdk_url_for_platform(platform: nil, version: nil)
248
250
  locations = sdk_locations
249
251
  platform = Environment.architecture if platform.nil?
250
- locations = locations.select{|l|l['platform'].eql?(platform)}
252
+ locations = locations.select{ |l| l['platform'].eql?(platform)}
251
253
  raise "No SDK for platform: #{platform}" if locations.empty?
252
- version = locations.max_by { |entry| Gem::Version.new(entry['version']) }['version'] if version.nil?
253
- info = locations.select{|entry| entry['version'].eql?(version)}
254
+ version = locations.max_by{ |entry| Gem::Version.new(entry['version'])}['version'] if version.nil?
255
+ info = locations.select{ |entry| entry['version'].eql?(version)}
254
256
  raise "No such version: #{version} for #{platform}" if info.empty?
255
257
  return info.first['url']
256
258
  end
@@ -300,7 +302,7 @@ module Aspera
300
302
  if subfolder_lambda.nil?
301
303
  # default files to extract directly to main folder if in selected source folders
302
304
  subfolder_lambda = ->(name) do
303
- Products::Transferd::RUNTIME_FOLDERS.any?{|i|name.match?(%r{^[^/]*/#{i}/})} ? '/' : nil
305
+ Products::Transferd::RUNTIME_FOLDERS.any?{ |i| name.match?(%r{^[^/]*/#{i}/})} ? '/' : nil
304
306
  end
305
307
  end
306
308
  # rename old install
@@ -322,7 +324,7 @@ module Aspera
322
324
  end
323
325
  FileUtils.mkdir_p(dest_folder)
324
326
  if link_target.nil?
325
- File.open(dest_file, 'wb') { |output_stream|IO.copy_stream(entry_stream, output_stream)}
327
+ File.open(dest_file, 'wb'){ |output_stream| IO.copy_stream(entry_stream, output_stream)}
326
328
  else
327
329
  File.symlink(link_target, dest_file)
328
330
  end
@@ -227,7 +227,7 @@ module Aspera
227
227
  # TODO: translate enhanced to capitalized ?
228
228
  data
229
229
  .keys
230
- .map{|k|"#{k.capitalize}: #{data[k]}"}
230
+ .map{ |k| "#{k.capitalize}: #{data[k]}"}
231
231
  .unshift(MGT_HEADER)
232
232
  .push('', '')
233
233
  .join("\n")