aspera-cli 4.22.0 → 4.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +405 -364
  4. data/CONTRIBUTING.md +86 -29
  5. data/README.md +1856 -961
  6. data/bin/ascli +2 -1
  7. data/bin/asession +4 -4
  8. data/lib/aspera/agent/base.rb +4 -0
  9. data/lib/aspera/agent/connect.rb +20 -18
  10. data/lib/aspera/agent/desktop.rb +14 -11
  11. data/lib/aspera/agent/direct.rb +39 -31
  12. data/lib/aspera/agent/httpgw.rb +2 -2
  13. data/lib/aspera/agent/node.rb +9 -11
  14. data/lib/aspera/agent/transferd.rb +18 -11
  15. data/lib/aspera/api/aoc.rb +53 -43
  16. data/lib/aspera/api/cos_node.rb +7 -5
  17. data/lib/aspera/api/httpgw.rb +23 -22
  18. data/lib/aspera/api/node.rb +104 -22
  19. data/lib/aspera/ascmd.rb +35 -21
  20. data/lib/aspera/ascp/installation.rb +43 -43
  21. data/lib/aspera/ascp/management.rb +5 -4
  22. data/lib/aspera/assert.rb +55 -24
  23. data/lib/aspera/cli/basic_auth_plugin.rb +8 -7
  24. data/lib/aspera/cli/error.rb +1 -1
  25. data/lib/aspera/cli/extended_value.rb +28 -29
  26. data/lib/aspera/cli/formatter.rb +191 -168
  27. data/lib/aspera/cli/hints.rb +38 -4
  28. data/lib/aspera/cli/main.rb +139 -108
  29. data/lib/aspera/cli/manager.rb +51 -31
  30. data/lib/aspera/cli/plugin.rb +149 -78
  31. data/lib/aspera/cli/plugin_factory.rb +2 -2
  32. data/lib/aspera/cli/plugins/aoc.rb +217 -88
  33. data/lib/aspera/cli/plugins/ats.rb +15 -13
  34. data/lib/aspera/cli/plugins/config.rb +105 -227
  35. data/lib/aspera/cli/plugins/console.rb +49 -18
  36. data/lib/aspera/cli/plugins/cos.rb +4 -4
  37. data/lib/aspera/cli/plugins/faspex.rb +45 -51
  38. data/lib/aspera/cli/plugins/faspex5.rb +162 -163
  39. data/lib/aspera/cli/plugins/faspio.rb +6 -5
  40. data/lib/aspera/cli/plugins/httpgw.rb +2 -2
  41. data/lib/aspera/cli/plugins/node.rb +233 -247
  42. data/lib/aspera/cli/plugins/orchestrator.rb +10 -14
  43. data/lib/aspera/cli/plugins/preview.rb +26 -29
  44. data/lib/aspera/cli/plugins/server.rb +29 -28
  45. data/lib/aspera/cli/plugins/shares.rb +40 -28
  46. data/lib/aspera/cli/sync_actions.rb +101 -80
  47. data/lib/aspera/cli/transfer_agent.rb +55 -58
  48. data/lib/aspera/cli/transfer_progress.rb +29 -20
  49. data/lib/aspera/cli/version.rb +1 -1
  50. data/lib/aspera/cli/wizard.rb +160 -0
  51. data/lib/aspera/colors.rb +13 -8
  52. data/lib/aspera/command_line_builder.rb +28 -22
  53. data/lib/aspera/command_line_converter.rb +31 -0
  54. data/lib/aspera/data_repository.rb +1 -0
  55. data/lib/aspera/environment.rb +144 -100
  56. data/lib/aspera/faspex_gw.rb +1 -1
  57. data/lib/aspera/faspex_postproc.rb +3 -2
  58. data/lib/aspera/hash_ext.rb +1 -1
  59. data/lib/aspera/id_generator.rb +10 -10
  60. data/lib/aspera/keychain/base.rb +18 -0
  61. data/lib/aspera/keychain/encrypted_hash.rb +6 -12
  62. data/lib/aspera/keychain/factory.rb +9 -3
  63. data/lib/aspera/keychain/hashicorp_vault.rb +9 -6
  64. data/lib/aspera/keychain/macos_security.rb +13 -13
  65. data/lib/aspera/log.rb +70 -20
  66. data/lib/aspera/nagios.rb +5 -6
  67. data/lib/aspera/node_simulator.rb +12 -7
  68. data/lib/aspera/oauth/base.rb +6 -2
  69. data/lib/aspera/oauth/factory.rb +25 -18
  70. data/lib/aspera/oauth/jwt.rb +13 -1
  71. data/lib/aspera/oauth/url_json.rb +3 -3
  72. data/lib/aspera/oauth/web.rb +5 -3
  73. data/lib/aspera/persistency_folder.rb +2 -2
  74. data/lib/aspera/preview/file_types.rb +43 -35
  75. data/lib/aspera/preview/generator.rb +26 -13
  76. data/lib/aspera/preview/terminal.rb +10 -7
  77. data/lib/aspera/preview/utils.rb +11 -9
  78. data/lib/aspera/products/connect.rb +2 -1
  79. data/lib/aspera/products/desktop.rb +1 -1
  80. data/lib/aspera/products/other.rb +2 -2
  81. data/lib/aspera/products/transferd.rb +8 -6
  82. data/lib/aspera/proxy_auto_config.rb +1 -1
  83. data/lib/aspera/rest.rb +46 -28
  84. data/lib/aspera/rest_call_error.rb +1 -1
  85. data/lib/aspera/rest_error_analyzer.rb +1 -0
  86. data/lib/aspera/resumer.rb +1 -1
  87. data/lib/aspera/secret_hider.rb +46 -40
  88. data/lib/aspera/ssh.rb +14 -4
  89. data/lib/aspera/sync/args.schema.yaml +102 -0
  90. data/lib/aspera/sync/conf.schema.yaml +701 -0
  91. data/lib/aspera/sync/database.rb +83 -0
  92. data/lib/aspera/{transfer/sync.rb → sync/operations.rb} +145 -68
  93. data/lib/aspera/temp_file_manager.rb +4 -2
  94. data/lib/aspera/timer_limiter.rb +7 -5
  95. data/lib/aspera/transfer/error.rb +1 -1
  96. data/lib/aspera/transfer/error_info.rb +1 -2
  97. data/lib/aspera/transfer/faux_file.rb +11 -10
  98. data/lib/aspera/transfer/parameters.rb +6 -5
  99. data/lib/aspera/transfer/spec.rb +15 -1
  100. data/lib/aspera/transfer/spec.schema.yaml +316 -293
  101. data/lib/aspera/transfer/spec_doc.rb +34 -16
  102. data/lib/aspera/transfer/uri.rb +5 -5
  103. data/lib/aspera/uri_reader.rb +14 -10
  104. data/lib/aspera/web_auth.rb +2 -2
  105. data/lib/aspera/web_server_simple.rb +2 -2
  106. data.tar.gz.sig +0 -0
  107. metadata +15 -15
  108. metadata.gz.sig +0 -0
  109. data/examples/dascli +0 -30
  110. data/examples/get_proto_file.rb +0 -8
  111. data/examples/proxy.pac +0 -60
  112. data/lib/aspera/transfer/convert.rb +0 -29
  113. data/lib/aspera/transfer/sync_instance.schema.yaml +0 -13
  114. data/lib/aspera/transfer/sync_session.schema.yaml +0 -79
data/bin/ascli CHANGED
@@ -12,5 +12,6 @@ $VERBOSE = old_verbose
12
12
  require 'aspera/coverage'
13
13
  require 'aspera/environment'
14
14
  require 'aspera/cli/main'
15
- Aspera::Environment.fix_home
15
+ Aspera::Cli::Main.early_debug_setup(ARGV)
16
+ Aspera::Environment.instance.fix_home
16
17
  Aspera::Cli::Main.new(ARGV).process_command_line
data/bin/asession CHANGED
@@ -21,6 +21,7 @@ SAMPLE_DEMO = '"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ss
21
21
  SAMPLE_DEMO2 = '"direction":"receive","destination_root":"./test.dir"'
22
22
  def assert_usage(assertion, error_message)
23
23
  return if assertion
24
+ # rubocop:disable Style/StderrPuts
24
25
  $stderr.puts('ERROR: '.red.blink + error_message) if error_message
25
26
  $stderr.puts('USAGE')
26
27
  $stderr.puts(' asession')
@@ -47,6 +48,7 @@ def assert_usage(assertion, error_message)
47
48
  $stderr.puts('EXAMPLES')
48
49
  $stderr.puts(%Q( asession @json:'{"#{PARAM_SPEC}":{#{SAMPLE_DEMO},#{SAMPLE_DEMO2},"paths":[{"source":"/aspera-test-dir-tiny/200KB.1"}]}}'))
49
50
  $stderr.puts(%Q( echo '{"#{PARAM_SPEC}":{"remote_host":...}}'|asession @json:@stdin))
51
+ # rubocop:enable Style/StderrPuts
50
52
  Process.exit(0)
51
53
  end
52
54
  parameter_source_err_msg = ' (argument), did you specify: "@json:" ?'
@@ -71,9 +73,7 @@ assert_usage(session_spec[PARAM_SPEC].is_a?(Hash), "The value must contain key #
71
73
  # additional debug capability
72
74
  Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL] if session_spec.key?(PARAM_LOG_LEVEL)
73
75
  # possibly override temp folder
74
- if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
75
- Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER]
76
- end
76
+ Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER] if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
77
77
  session_spec[PARAM_SDK] = File.join(Dir.home, '.aspera', 'sdk') unless session_spec.key?(PARAM_SDK)
78
78
  Aspera::Products::Transferd.sdk_directory = session_spec[PARAM_SDK]
79
79
  session_spec[PARAM_AGENT] = {} unless session_spec.key?(PARAM_AGENT)
@@ -90,7 +90,7 @@ job_id = client.start_transfer(session_spec[PARAM_SPEC])
90
90
  Thread.new do
91
91
  # we assume here a single session
92
92
  session_id = client.sessions_by_job(job_id).first
93
- begin # rubocop:disable Style/RedundantBegin
93
+ begin
94
94
  loop do
95
95
  data = JSON.parse($stdin.gets)
96
96
  client.send_command(session_id, data)
@@ -5,6 +5,10 @@ require 'aspera/assert'
5
5
  module Aspera
6
6
  module Agent
7
7
  # Base class for transfer agents
8
+ # Transfer agents provide methods:
9
+ # - `start_transfer` : take a transfer spec and start a transfer asynchronously
10
+ # - `wait_for_transfers_completion` : waits for all transfer sessions to finish
11
+ # - `notify_progress` : called back by transfer agent to notify transfer progress
8
12
  class Base
9
13
  RUBY_EXT = '.rb'
10
14
  private_constant :RUBY_EXT
@@ -21,27 +21,25 @@ module Aspera
21
21
  @connect_settings = {
22
22
  'app_id' => SecureRandom.uuid
23
23
  }
24
- raise 'Using connect requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
24
+ raise Error, 'Using connect requires a graphical environment' unless Environment.instance.graphical?
25
25
  method_index = 0
26
26
  begin
27
27
  connect_url = connect_api_url
28
28
  Log.log.debug{"found: #{connect_url}"}
29
29
  @connect_api = Rest.new(
30
30
  base_url: "#{connect_url}/v5/connect", # could use v6 also now
31
- headers: {'Origin' => RestParameters.instance.user_agent})
31
+ headers: {'Origin' => RestParameters.instance.user_agent}
32
+ )
32
33
  connect_info = @connect_api.read('info/version')
33
34
  Log.log.info('Connect was reached') if method_index > 0
34
- Log.log.debug{Log.dump(:connect_version, connect_info)}
35
+ Log.dump(:connect_version, connect_info)
35
36
  rescue StandardError => e # Errno::ECONNREFUSED
36
37
  Log.log.debug{"Exception: #{e}"}
37
38
  start_url = CONNECT_START_URIS[method_index]
38
39
  method_index += 1
39
40
  raise StandardError, "Unable to start connect #{method_index} times" if start_url.nil?
40
41
  Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..."}
41
- if !Environment.open_uri_graphical(start_url)
42
- Environment.open_uri_graphical('https://www.ibm.com/aspera/connect/')
43
- raise StandardError, 'Connect is not installed'
44
- end
42
+ Environment.instance.open_uri_graphical(start_url)
45
43
  sleep(SLEEP_SEC_BETWEEN_RETRY)
46
44
  retry
47
45
  end
@@ -57,7 +55,8 @@ module Aspera
57
55
  'title' => 'Select Files',
58
56
  'suggestedName' => '',
59
57
  'allowMultipleSelection' => true,
60
- 'allowedFileTypes' => ''})
58
+ 'allowedFileTypes' => ''
59
+ })
61
60
  transfer_spec['paths'] = selection['dataTransfer']['files'].map{ |i| {'source' => i['name']}}
62
61
  end
63
62
  # if there is a token, we ask connect client to use well known ssh private keys
@@ -70,7 +69,8 @@ module Aspera
70
69
  }),
71
70
  'transfer_specs' => [{
72
71
  'transfer_spec' => transfer_spec
73
- }]}
72
+ }]
73
+ }
74
74
  # asynchronous anyway
75
75
  res = @connect_api.create('transfers/start', connect_transfer_args)
76
76
  @transfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id']
@@ -83,7 +83,7 @@ module Aspera
83
83
  begin
84
84
  loop do
85
85
  tr_info = @connect_api.create("transfers/info/#{@transfer_id}", connect_activity_args)
86
- Log.log.trace1{Log.dump(:tr_info, tr_info)}
86
+ Log.dump(:tr_info, tr_info, level: :trace1)
87
87
  if tr_info['transfer_info'].is_a?(Hash)
88
88
  transfer = tr_info['transfer_info']
89
89
  if transfer.nil?
@@ -93,7 +93,7 @@ module Aspera
93
93
  # TODO: get session id
94
94
  case transfer['status']
95
95
  when 'initiating', 'queued'
96
- notify_progress(:pre_start, session_id: nil, info: transfer['status'])
96
+ notify_progress(:sessions_init, info: transfer['status'])
97
97
  when 'running'
98
98
  if !started
99
99
  notify_progress(:session_start, session_id: @transfer_id)
@@ -106,16 +106,20 @@ module Aspera
106
106
  notify_progress(:transfer, session_id: @transfer_id, info: transfer['bytes_written'])
107
107
  end
108
108
  when 'completed'
109
- notify_progress(:end, session_id: @transfer_id)
109
+ notify_progress(:session_end, session_id: @transfer_id)
110
+ notify_progress(:end)
110
111
  break
111
112
  when 'failed'
112
- notify_progress(:end, session_id: @transfer_id)
113
+ notify_progress(:session_end, session_id: @transfer_id)
114
+ notify_progress(:end)
113
115
  raise Transfer::Error, transfer['error_desc']
114
116
  when 'cancelled'
115
- notify_progress(:end, session_id: @transfer_id)
117
+ notify_progress(:session_end, session_id: @transfer_id)
118
+ notify_progress(:end)
116
119
  raise Transfer::Error, 'Transfer cancelled by user'
117
120
  else
118
- notify_progress(:end, session_id: @transfer_id)
121
+ notify_progress(:session_end, session_id: @transfer_id)
122
+ notify_progress(:end)
119
123
  raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
120
124
  end
121
125
  end
@@ -137,9 +141,7 @@ module Aspera
137
141
  ['', 's'].each do |ext|
138
142
  uri_file = File.join(folder, "http#{ext}.uri")
139
143
  Log.log.debug{"checking connect port file: #{uri_file}"}
140
- if File.exist?(uri_file)
141
- return File.open(uri_file, &:gets).strip
142
- end
144
+ return File.open(uri_file, &:gets).strip if File.exist?(uri_file)
143
145
  end
144
146
  raise "no connect uri file found in #{folder}"
145
147
  end
@@ -5,6 +5,7 @@ require 'aspera/rest'
5
5
  require 'aspera/environment'
6
6
  require 'aspera/json_rpc'
7
7
  require 'aspera/products/desktop'
8
+ require 'aspera/transfer/spec'
8
9
  require 'securerandom'
9
10
 
10
11
  module Aspera
@@ -21,24 +22,21 @@ module Aspera
21
22
  @application_id = SecureRandom.uuid
22
23
  @transfer_id = nil
23
24
  super
24
- raise 'Using client requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
25
+ raise Error, 'Using client requires a graphical environment' unless Environment.instance.graphical?
25
26
  method_index = 0
26
27
  begin
27
28
  # curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
28
29
  # https://playground.open-rpc.org/?schemaUrl=http://127.0.0.1:33024
29
30
  @client_app_api = Aspera::JsonRpcClient.new(Aspera::Rest.new(base_url: aspera_client_api_url))
30
31
  client_info = @client_app_api.get_info
31
- Log.log.debug{Log.dump(:client_version, client_info)}
32
+ Log.dump(:client_version, client_info)
32
33
  Log.log.info('Client was reached') if method_index > 0
33
34
  rescue Errno::ECONNREFUSED => e
34
35
  start_url = START_URIS[method_index]
35
36
  method_index += 1
36
37
  raise StandardError, "Unable to start #{Products::Desktop::APP_NAME} #{method_index} times" if start_url.nil?
37
38
  Log.log.warn{"#{Products::Desktop::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
38
- if !Environment.open_uri_graphical(start_url)
39
- Environment.open_uri_graphical('https://www.ibm.com/aspera/connect/')
40
- raise StandardError, "#{Products::Desktop::APP_NAME} is not installed"
41
- end
39
+ Environment.instance.open_uri_graphical(start_url)
42
40
  sleep(SLEEP_SEC_BETWEEN_RETRY)
43
41
  retry
44
42
  end
@@ -46,6 +44,7 @@ module Aspera
46
44
 
47
45
  # :reek:UnusedParameters token_regenerator
48
46
  def start_transfer(transfer_spec, token_regenerator: nil)
47
+ Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
49
48
  @request_id = SecureRandom.uuid
50
49
  # if there is a token, we ask the client app to use well known ssh private keys
51
50
  # instead of asking password
@@ -62,7 +61,7 @@ module Aspera
62
61
  transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @transfer_id)
63
62
  case transfer['status']
64
63
  when 'initiating', 'queued'
65
- notify_progress(:pre_start, session_id: nil, info: transfer['status'])
64
+ notify_progress(:sessions_init, info: transfer['status'])
66
65
  when 'running'
67
66
  if !started
68
67
  notify_progress(:session_start, session_id: @transfer_id)
@@ -75,16 +74,20 @@ module Aspera
75
74
  notify_progress(:transfer, session_id: @transfer_id, info: transfer['bytes_written'])
76
75
  end
77
76
  when 'completed'
78
- notify_progress(:end, session_id: @transfer_id)
77
+ notify_progress(:session_end, session_id: @transfer_id)
78
+ notify_progress(:end)
79
79
  break
80
80
  when 'failed'
81
- notify_progress(:end, session_id: @transfer_id)
81
+ notify_progress(:session_end, session_id: @transfer_id)
82
+ notify_progress(:end)
82
83
  raise Transfer::Error, transfer['error_desc']
83
84
  when 'cancelled'
84
- notify_progress(:end, session_id: @transfer_id)
85
+ notify_progress(:session_end, session_id: @transfer_id)
86
+ notify_progress(:end)
85
87
  raise Transfer::Error, 'Transfer cancelled by user'
86
88
  else
87
- notify_progress(:end, session_id: @transfer_id)
89
+ notify_progress(:session_end, session_id: @transfer_id)
90
+ notify_progress(:end)
88
91
  raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
89
92
  end
90
93
  sleep(1)
@@ -66,7 +66,7 @@ module Aspera
66
66
  @spawn_timeout_sec = spawn_timeout_sec
67
67
  @spawn_delay_sec = spawn_delay_sec
68
68
  # default is true on Windows, false on other OSes
69
- @multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
69
+ @multi_incr_udp = multi_incr_udp.nil? ? Environment.instance.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
70
70
  @monitor = monitor
71
71
  @management_cb = management_cb
72
72
  @resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
@@ -96,7 +96,7 @@ module Aspera
96
96
  # TODO: useful ? node only ? seems to be a timeout for retry in node
97
97
  transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 3600
98
98
  end
99
- Log.log.debug{Log.dump('ts', transfer_spec)}
99
+ Log.dump(:ts, transfer_spec)
100
100
  # Compute this before using transfer spec because it potentially modifies the transfer spec
101
101
  # (even if the var is not used in single session)
102
102
  multi_session_info = nil
@@ -171,6 +171,7 @@ module Aspera
171
171
  session[:thread].join
172
172
  result.push(session[:error] || :success)
173
173
  end
174
+ notify_progress(:end)
174
175
  Log.log.debug('all transfers joined')
175
176
  # since all are finished and we return the result, clear statuses
176
177
  @sessions.clear
@@ -225,21 +226,24 @@ module Aspera
225
226
  # This is the low level method to start the transfer process.
226
227
  # Typically started in a thread.
227
228
  # Start process with management port.
228
- # @param session this session information, keys :io and :token_regenerator
229
- # @param env [Hash] environment variables (comes from ascp_args)
230
- # @param name [Symbol] name of executable: :ascp, :ascp4 or :async (comes from ascp_args)
231
- # @param args [Array] command line arguments (comes from ascp_args)
229
+ # @param session [Hash] This session information, keys :io and :token_regenerator
230
+ # @param name [Symbol] Name of executable: :ascp, :ascp4 or :async (comes from ascp_args)
231
+ # @param env [Hash] Environment variables (comes from ascp_args)
232
+ # @param args [Array] Command line arguments (comes from ascp_args)
232
233
  # @return [nil] when process has exited
233
234
  # @throw FaspError on error
234
235
  def start_and_monitor_process(
235
236
  session:,
236
- env:,
237
237
  name:,
238
+ env:,
238
239
  args:
239
240
  )
240
241
  Aspera.assert_type(session, Hash)
241
- notify_progress(:pre_start, session_id: nil, info: 'starting')
242
+ notify_progress(:sessions_init, info: 'starting')
242
243
  begin
244
+ capture_stderr = false
245
+ stderr_r, stderr_w = nil
246
+ spawn_args = {}
243
247
  command_pid = nil
244
248
  command_arguments = []
245
249
  if @monitor
@@ -258,13 +262,17 @@ module Aspera
258
262
  end
259
263
  end
260
264
  command_arguments.concat(args)
261
- # capture process stderr
262
- stderr_r, stderr_w = IO.pipe
265
+ if capture_stderr
266
+ # capture process stderr
267
+ stderr_r, stderr_w = IO.pipe
268
+ spawn_args[err] = stderr_w
269
+ end
263
270
  # get location of command executable (ascp, async)
264
271
  command_path = Ascp::Installation.instance.path(name)
265
- command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments, err: stderr_w)
266
- stderr_w.close
267
- notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
272
+ command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments, **spawn_args)
273
+ # close here, but still used in other process (pipe)
274
+ stderr_w&.close
275
+ notify_progress(:sessions_init, info: "waiting for #{name} to start")
268
276
  # "ensure" block will wait for process
269
277
  return unless @monitor
270
278
  # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
@@ -272,7 +280,7 @@ module Aspera
272
280
  Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
273
281
  readable, _, _ = IO.select([mgt_server_socket], nil, nil, @spawn_timeout_sec)
274
282
  Log.log.debug('after select, before accept')
275
- Aspera.assert(readable, exception_class: Transfer::Error){'timeout waiting mgt port connect (select not readable)'}
283
+ Aspera.assert(readable, type: Transfer::Error){'timeout waiting mgt port connect (select not readable)'}
276
284
  # There is a connection to accept
277
285
  client_socket, _client_addrinfo = mgt_server_socket.accept
278
286
  Log.log.debug('after accept')
@@ -288,12 +296,11 @@ module Aspera
288
296
  event = processor.process_line(line.chomp)
289
297
  next unless event
290
298
  # event is ready
291
- Log.log.trace1{Log.dump(:management_port, event)}
299
+ Log.dump(:management_port, event, level: :trace1)
292
300
  # store session identifier
293
301
  session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
294
302
  @management_cb&.call(event)
295
303
  process_progress(event)
296
- Log.log.error(event['Description'].to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
297
304
  end
298
305
  Log.log.debug('management io closed')
299
306
  # check that last status was received before process exit
@@ -311,9 +318,7 @@ module Aspera
311
318
  env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
312
319
  end
313
320
  raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
314
- else
315
- Log.log.error{"unexpected last event type: #{last_event['Type']}"}
316
- # raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
321
+ else Aspera.error_unexpected_value(last_event['Type'], :error){'last event type'}
317
322
  end
318
323
  rescue SystemCallError => e
319
324
  # Process.spawn failed, or socket error
@@ -325,14 +330,20 @@ module Aspera
325
330
  session.delete(:io)
326
331
  # if command was successfully started, check its status
327
332
  unless command_pid.nil?
328
- Process.kill(:INT, command_pid) if @monitor
333
+ Process.kill(:INT, command_pid) if @monitor && !Environment.instance.os.eql?(Environment::OS_WINDOWS)
329
334
  # collect process exit status or wait for termination
330
335
  _, status = Process.wait2(command_pid)
331
- # process stderr of ascp
332
- stderr_r.each_line do |line|
333
- Log.log.error(line.chomp)
336
+ if stderr_r
337
+ # process stderr of ascp
338
+ stderr_flag = false
339
+ stderr_r.each_line do |line|
340
+ Log.log.error{"BEGIN stderr #{name}"} unless stderr_flag
341
+ Log.log.error{line.chomp}
342
+ stderr_flag = true
343
+ end
344
+ Log.log.error{"END stderr #{name}"} if stderr_flag
345
+ stderr_r.close
334
346
  end
335
- stderr_r.close
336
347
  # status is nil if an exception occurred before starting command
337
348
  if !status&.success?
338
349
  message = "#{name} failed (#{status})"
@@ -370,17 +381,14 @@ module Aspera
370
381
  notify_progress(:transfer, session_id: session_id, info: @pre_calc_last_size)
371
382
  when 'DONE', 'ERROR' # end of session
372
383
  total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
373
- if !@pre_calc_sent && !total_size.zero?
374
- notify_progress(:session_size, session_id: session_id, info: total_size)
375
- end
376
- if @pre_calc_last_size != total_size
377
- notify_progress(:transfer, session_id: session_id, info: total_size)
378
- end
379
- notify_progress(:end, session_id: session_id)
384
+ notify_progress(:session_size, session_id: session_id, info: total_size) if !@pre_calc_sent && !total_size.zero?
385
+ notify_progress(:transfer, session_id: session_id, info: total_size) if @pre_calc_last_size != total_size
386
+ notify_progress(:session_end, session_id: session_id)
380
387
  # cspell:disable
381
388
  when 'SESSION'
382
389
  when 'ARGSTOP'
383
390
  when 'FILEERROR'
391
+ Log.log.error{"#{event['Type']} #{event['Description']}"}
384
392
  when 'STOP'
385
393
  # cspell:enable
386
394
  # stop event when one file is completed
@@ -32,10 +32,10 @@ module Aspera
32
32
  # HTTP download only supports file list
33
33
  # :reek:UnusedParameters token_regenerator
34
34
  def start_transfer(transfer_spec, token_regenerator: nil)
35
- raise 'GW URL must be set' if @gw_api.nil?
35
+ Aspera.assert(!@gw_api.nil?){'GW URL must be set'}
36
36
  Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
37
37
  Aspera.assert_type(transfer_spec['token'], String){'only token based transfer is supported in GW'}
38
- Log.log.debug{Log.dump(:user_spec, transfer_spec)}
38
+ Log.dump(:user_spec, transfer_spec)
39
39
  transfer_spec['authentication'] ||= 'token'
40
40
  case transfer_spec['direction']
41
41
  when Transfer::Spec::DIRECTION_SEND
@@ -29,7 +29,7 @@ module Aspera
29
29
  # root id is required for access key
30
30
  @root_id = root_id
31
31
  rest_params = {base_url: url}
32
- if OAuth::Factory.bearer?(password)
32
+ if OAuth::Factory.bearer_auth?(password)
33
33
  Aspera.assert(!@root_id.nil?){'root_id not allowed for access key'}
34
34
  rest_params[:headers] = Api::Node.bearer_headers(password, access_key: username)
35
35
  else
@@ -57,14 +57,10 @@ module Aspera
57
57
  end
58
58
  # add mandatory retry parameter for node api
59
59
  ts_tags = transfer_spec['tags']
60
- if ts_tags.is_a?(Hash) && ts_tags[Transfer::Spec::TAG_RESERVED].is_a?(Hash)
61
- ts_tags[Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 150
62
- end
60
+ ts_tags[Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 150 if ts_tags.is_a?(Hash) && ts_tags[Transfer::Spec::TAG_RESERVED].is_a?(Hash)
63
61
  # Optimization in case of sending to the same node
64
62
  # TODO: probably remove this, as /etc/hosts shall be used for that
65
- if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
66
- transfer_spec['remote_host'] = '127.0.0.1'
67
- end
63
+ transfer_spec['remote_host'] = '127.0.0.1' if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
68
64
  resp = node_api_.create('ops/transfers', transfer_spec)
69
65
  @transfer_id = resp['id']
70
66
  Log.log.debug{"tr_id=#{@transfer_id}"}
@@ -82,7 +78,7 @@ module Aspera
82
78
  transfer_data = node_api_.read("ops/transfers/#{@transfer_id}") || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
83
79
  case transfer_data['status']
84
80
  when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
85
- notify_progress(:pre_start, session_id: nil, info: transfer_data['status'])
81
+ notify_progress(:sessions_init, info: transfer_data['status'])
86
82
  when 'running'
87
83
  if !session_started
88
84
  notify_progress(:session_start, session_id: @transfer_id)
@@ -90,7 +86,7 @@ module Aspera
90
86
  end
91
87
  message = transfer_data['status']
92
88
  message = "#{message} (#{transfer_data['error_desc']})" if !transfer_data['error_desc']&.empty?
93
- notify_progress(:pre_start, session_id: nil, info: message)
89
+ notify_progress(:sessions_init, info: message)
94
90
  if bytes_expected.nil? &&
95
91
  transfer_data['precalc'].is_a?(Hash) &&
96
92
  transfer_data['precalc']['status'].eql?('ready')
@@ -100,10 +96,12 @@ module Aspera
100
96
  notify_progress(:transfer, session_id: @transfer_id, info: transfer_data['bytes_transferred'])
101
97
  when 'completed'
102
98
  notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
103
- notify_progress(:end, session_id: @transfer_id)
99
+ notify_progress(:session_end, session_id: @transfer_id)
100
+ notify_progress(:end)
104
101
  break
105
102
  when 'failed'
106
- notify_progress(:end, session_id: @transfer_id)
103
+ notify_progress(:session_end, session_id: @transfer_id)
104
+ notify_progress(:end)
107
105
  # Bug in HSTS ? transfer is marked failed, but there is no reason
108
106
  break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
109
107
  raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
@@ -4,6 +4,7 @@ require 'aspera/environment'
4
4
  require 'aspera/agent/base'
5
5
  require 'aspera/products/transferd'
6
6
  require 'aspera/temp_file_manager'
7
+ require 'aspera/transfer/spec'
7
8
  require 'json'
8
9
  require 'uri'
9
10
  require 'transferd_services_pb'
@@ -34,11 +35,11 @@ module Aspera
34
35
  @transfer_id = nil
35
36
  @stop = stop
36
37
  is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
37
- raise 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
38
+ raise Error, 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
38
39
  # keep PID for optional shutdown
39
40
  @daemon_pid = nil
40
41
  daemon_endpoint = url
41
- Log.log.debug{Log.dump(:daemon_endpoint, daemon_endpoint)}
42
+ Log.dump(:daemon_endpoint, daemon_endpoint)
42
43
  # retry loop
43
44
  begin
44
45
  # no address: local bind
@@ -82,7 +83,8 @@ module Aspera
82
83
  exec: Ascp::Installation.instance.path(:transferd),
83
84
  args: ['--config', conf_file],
84
85
  out: log_stdout,
85
- err: log_stderr)
86
+ err: log_stderr
87
+ )
86
88
  begin
87
89
  # wait for process to initialize, max 2 seconds
88
90
  Timeout.timeout(2.0) do
@@ -105,15 +107,18 @@ module Aspera
105
107
 
106
108
  # :reek:UnusedParameters token_regenerator
107
109
  def start_transfer(transfer_spec, token_regenerator: nil)
110
+ Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
108
111
  # create a transfer request
109
112
  transfer_request = ::Transferd::Api::TransferRequest.new(
110
113
  transferType: ::Transferd::Api::TransferType::FILE_REGULAR, # transfer type (file/stream)
111
114
  config: ::Transferd::Api::TransferConfig.new, # transfer configuration
112
- transferSpec: transfer_spec.to_json) # transfer definition
115
+ transferSpec: transfer_spec.to_json
116
+ ) # transfer definition
113
117
  # send start transfer request to the transfer manager daemon
114
- start_transfer_response = @transfer_client.start_transfer(transfer_request)
115
- Log.log.debug{"start transfer response #{start_transfer_response}"}
116
- @transfer_id = start_transfer_response.transferId
118
+ start_response = @transfer_client.start_transfer(transfer_request)
119
+ raise Transfer::Error, start_response.error.description if start_response.status.eql?(:FAILED)
120
+ Log.log.debug{"start transfer response #{start_response}"}
121
+ @transfer_id = start_response.transferId
117
122
  Log.log.debug{"transfer started with id #{@transfer_id}"}
118
123
  end
119
124
 
@@ -123,7 +128,7 @@ module Aspera
123
128
  bytes_expected = nil
124
129
  # monitor transfer status
125
130
  @transfer_client.monitor_transfers(::Transferd::Api::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
126
- Log.log.debug{Log.dump(:response, response.to_h)}
131
+ Log.dump(:response, response.to_h)
127
132
  # Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
128
133
  case response.status
129
134
  when :RUNNING
@@ -139,13 +144,15 @@ module Aspera
139
144
  notify_progress(:transfer, session_id: @transfer_id, info: response.transferInfo.bytesTransferred)
140
145
  when :COMPLETED
141
146
  notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
142
- notify_progress(:end, session_id: @transfer_id)
147
+ notify_progress(:session_end, session_id: @transfer_id)
148
+ notify_progress(:end)
143
149
  break
144
150
  when :FAILED, :CANCELED
145
- notify_progress(:end, session_id: @transfer_id)
151
+ notify_progress(:session_end, session_id: @transfer_id)
152
+ notify_progress(:end)
146
153
  raise Transfer::Error, JSON.parse(response.message)['Description']
147
154
  when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
148
- notify_progress(:pre_start, session_id: nil, info: response.status.to_s.downcase)
155
+ notify_progress(:sessions_init, info: response.status.to_s.downcase)
149
156
  else
150
157
  Log.log.error{"unknown status#{response.status}"}
151
158
  end