aspera-cli 4.21.2 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +34 -16
  5. data/CONTRIBUTING.md +6 -10
  6. data/README.md +805 -574
  7. data/examples/get_proto_file.rb +1 -1
  8. data/lib/aspera/agent/base.rb +9 -5
  9. data/lib/aspera/agent/connect.rb +30 -28
  10. data/lib/aspera/agent/desktop.rb +29 -25
  11. data/lib/aspera/agent/direct.rb +137 -125
  12. data/lib/aspera/agent/httpgw.rb +22 -26
  13. data/lib/aspera/agent/node.rb +14 -11
  14. data/lib/aspera/agent/transferd.rb +6 -2
  15. data/lib/aspera/api/aoc.rb +6 -6
  16. data/lib/aspera/api/cos_node.rb +1 -1
  17. data/lib/aspera/api/httpgw.rb +7 -3
  18. data/lib/aspera/api/node.rb +6 -4
  19. data/lib/aspera/ascmd.rb +3 -3
  20. data/lib/aspera/ascp/installation.rb +15 -16
  21. data/lib/aspera/ascp/management.rb +1 -1
  22. data/lib/aspera/assert.rb +11 -2
  23. data/lib/aspera/cli/error.rb +2 -2
  24. data/lib/aspera/cli/extended_value.rb +38 -19
  25. data/lib/aspera/cli/formatter.rb +48 -48
  26. data/lib/aspera/cli/hints.rb +1 -1
  27. data/lib/aspera/cli/main.rb +190 -168
  28. data/lib/aspera/cli/manager.rb +15 -15
  29. data/lib/aspera/cli/plugin.rb +23 -20
  30. data/lib/aspera/cli/plugin_factory.rb +1 -1
  31. data/lib/aspera/cli/plugins/alee.rb +1 -1
  32. data/lib/aspera/cli/plugins/aoc.rb +144 -107
  33. data/lib/aspera/cli/plugins/ats.rb +19 -17
  34. data/lib/aspera/cli/plugins/config.rb +67 -83
  35. data/lib/aspera/cli/plugins/console.rb +5 -3
  36. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  37. data/lib/aspera/cli/plugins/faspex5.rb +104 -80
  38. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  39. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  40. data/lib/aspera/cli/plugins/node.rb +306 -179
  41. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  42. data/lib/aspera/cli/plugins/preview.rb +3 -3
  43. data/lib/aspera/cli/plugins/server.rb +6 -6
  44. data/lib/aspera/cli/plugins/shares.rb +5 -5
  45. data/lib/aspera/cli/sync_actions.rb +19 -18
  46. data/lib/aspera/cli/transfer_agent.rb +5 -5
  47. data/lib/aspera/cli/transfer_progress.rb +2 -2
  48. data/lib/aspera/cli/version.rb +1 -1
  49. data/lib/aspera/command_line_builder.rb +116 -95
  50. data/lib/aspera/coverage.rb +4 -3
  51. data/lib/aspera/environment.rb +6 -6
  52. data/lib/aspera/faspex_gw.rb +14 -14
  53. data/lib/aspera/faspex_postproc.rb +7 -6
  54. data/lib/aspera/hash_ext.rb +2 -2
  55. data/lib/aspera/json_rpc.rb +1 -1
  56. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  57. data/lib/aspera/keychain/factory.rb +41 -0
  58. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  59. data/lib/aspera/keychain/macos_security.rb +19 -11
  60. data/lib/aspera/log.rb +28 -34
  61. data/lib/aspera/nagios.rb +6 -6
  62. data/lib/aspera/node_simulator.rb +8 -8
  63. data/lib/aspera/oauth/base.rb +8 -6
  64. data/lib/aspera/oauth/factory.rb +5 -6
  65. data/lib/aspera/oauth/url_json.rb +6 -6
  66. data/lib/aspera/persistency_action_once.rb +6 -4
  67. data/lib/aspera/persistency_folder.rb +2 -2
  68. data/lib/aspera/preview/generator.rb +1 -1
  69. data/lib/aspera/preview/options.rb +16 -16
  70. data/lib/aspera/preview/terminal.rb +3 -3
  71. data/lib/aspera/preview/utils.rb +11 -13
  72. data/lib/aspera/products/connect.rb +1 -1
  73. data/lib/aspera/products/desktop.rb +1 -1
  74. data/lib/aspera/products/transferd.rb +1 -1
  75. data/lib/aspera/proxy_auto_config.rb +2 -2
  76. data/lib/aspera/rest.rb +52 -43
  77. data/lib/aspera/rest_errors_aspera.rb +1 -1
  78. data/lib/aspera/secret_hider.rb +5 -5
  79. data/lib/aspera/ssh.rb +4 -4
  80. data/lib/aspera/transfer/convert.rb +29 -0
  81. data/lib/aspera/transfer/error_info.rb +66 -66
  82. data/lib/aspera/transfer/parameters.rb +13 -68
  83. data/lib/aspera/transfer/spec.rb +5 -6
  84. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  85. data/lib/aspera/transfer/spec_doc.rb +62 -0
  86. data/lib/aspera/transfer/sync.rb +23 -72
  87. data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
  88. data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
  89. data/lib/aspera/transfer/uri.rb +6 -6
  90. data/lib/aspera/uri_reader.rb +1 -1
  91. data/lib/aspera/web_auth.rb +1 -1
  92. data/lib/aspera/web_server_simple.rb +53 -44
  93. data.tar.gz.sig +1 -2
  94. metadata +37 -4
  95. metadata.gz.sig +0 -0
  96. data/examples/build_package.sh +0 -28
  97. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -5,4 +5,4 @@ require 'aspera/ascp/installation'
5
5
  require 'aspera/cli/transfer_progress'
6
6
  Aspera::RestParameters.instance.progress_bar = Aspera::Cli::TransferProgress.new
7
7
  # Retrieve `transfer.proto` from the web
8
- Aspera::Ascp::Installation.instance.install_sdk(folder: ARGV.first, backup: false, with_exe: false) {|name| name.end_with?('.proto') ? '/' : nil }
8
+ Aspera::Ascp::Installation.instance.install_sdk(folder: ARGV.first, backup: false, with_exe: false){ |name| name.end_with?('.proto') ? '/' : nil}
@@ -21,7 +21,7 @@ module Aspera
21
21
  base_class = File.basename(__FILE__)
22
22
  Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
23
23
  file.end_with?(RUBY_EXT) && !file.eql?(base_class)
24
- end.map{|file|file[0..(-1 - RUBY_EXT.length)].to_sym}
24
+ end.map{ |file| file[0..(-1 - RUBY_EXT.length)].to_sym}
25
25
  end
26
26
  end
27
27
 
@@ -31,16 +31,20 @@ module Aspera
31
31
  statuses = wait_for_transfers_completion
32
32
  @progress&.reset
33
33
  Aspera.assert_type(statuses, Array)
34
- Aspera.assert(statuses.none?{|i|!i.eql?(:success) && !i.is_a?(StandardError)}){"bad statuses content: #{statuses}"}
34
+ Aspera.assert(statuses.none?{ |i| !i.eql?(:success) && !i.is_a?(StandardError)}){"bad statuses content: #{statuses}"}
35
35
  return statuses
36
36
  end
37
37
 
38
38
  private
39
39
 
40
+ Aspera.require_method!(:start_transfer)
41
+ Aspera.require_method!(:wait_for_transfers_completion)
42
+ # method `shutdown` is optional
43
+ def shutdown
44
+ nil
45
+ end
46
+
40
47
  def initialize(progress: nil)
41
- # method `shutdown` is optional
42
- Aspera.assert(respond_to?(:start_transfer))
43
- Aspera.assert(respond_to?(:wait_for_transfers_completion))
44
48
  @progress = progress
45
49
  end
46
50
 
@@ -17,6 +17,7 @@ module Aspera
17
17
  private_constant :CONNECT_START_URIS, :SLEEP_SEC_BETWEEN_RETRY
18
18
  def initialize(**base_options)
19
19
  super
20
+ @transfer_id = nil
20
21
  @connect_settings = {
21
22
  'app_id' => SecureRandom.uuid
22
23
  }
@@ -46,21 +47,7 @@ module Aspera
46
47
  end
47
48
  end
48
49
 
49
- # @return the file path of local connect where API's URI can be read
50
- def connect_api_url
51
- connect_locations = Products::Other.find(Products::Connect.locations).first
52
- raise "Product: #{name} not found, please install." if connect_locations.nil?
53
- folder = File.join(connect_locations[:run_root], 'var', 'run')
54
- ['', 's'].each do |ext|
55
- uri_file = File.join(folder, "http#{ext}.uri")
56
- Log.log.debug{"checking connect port file: #{uri_file}"}
57
- if File.exist?(uri_file)
58
- return File.open(uri_file, &:gets).strip
59
- end
60
- end
61
- raise "no connect uri file found in #{folder}"
62
- end
63
-
50
+ # :reek:UnusedParameters token_regenerator
64
51
  def start_transfer(transfer_spec, token_regenerator: nil)
65
52
  if transfer_spec['direction'] == 'send'
66
53
  Log.log.warn{"Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red}
@@ -71,15 +58,14 @@ module Aspera
71
58
  'suggestedName' => '',
72
59
  'allowMultipleSelection' => true,
73
60
  'allowedFileTypes' => ''})
74
- transfer_spec['paths'] = selection['dataTransfer']['files'].map { |i| {'source' => i['name']}}
61
+ transfer_spec['paths'] = selection['dataTransfer']['files'].map{ |i| {'source' => i['name']}}
75
62
  end
76
- @request_id = SecureRandom.uuid
77
63
  # if there is a token, we ask connect client to use well known ssh private keys
78
64
  # instead of asking password
79
65
  transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
80
66
  connect_transfer_args = {
81
67
  'aspera_connect_settings' => @connect_settings.merge({
82
- 'request_id' => @request_id,
68
+ 'request_id' => SecureRandom.uuid,
83
69
  'allow_dialogs' => true
84
70
  }),
85
71
  'transfer_specs' => [{
@@ -87,17 +73,16 @@ module Aspera
87
73
  }]}
88
74
  # asynchronous anyway
89
75
  res = @connect_api.create('transfers/start', connect_transfer_args)
90
- @xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id']
76
+ @transfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id']
91
77
  end
92
78
 
93
79
  def wait_for_transfers_completion
94
80
  connect_activity_args = {'aspera_connect_settings' => @connect_settings}
95
81
  started = false
96
82
  pre_calc = false
97
- session_id = @xfer_id
98
83
  begin
99
84
  loop do
100
- tr_info = @connect_api.create("transfers/info/#{@xfer_id}", connect_activity_args)
85
+ tr_info = @connect_api.create("transfers/info/#{@transfer_id}", connect_activity_args)
101
86
  Log.log.trace1{Log.dump(:tr_info, tr_info)}
102
87
  if tr_info['transfer_info'].is_a?(Hash)
103
88
  transfer = tr_info['transfer_info']
@@ -111,26 +96,26 @@ module Aspera
111
96
  notify_progress(:pre_start, session_id: nil, info: transfer['status'])
112
97
  when 'running'
113
98
  if !started
114
- notify_progress(:session_start, session_id: session_id)
99
+ notify_progress(:session_start, session_id: @transfer_id)
115
100
  started = true
116
101
  end
117
102
  if !pre_calc && (transfer['bytes_expected'] != 0)
118
- notify_progress(:session_size, session_id: session_id, info: transfer['bytes_expected'])
103
+ notify_progress(:session_size, session_id: @transfer_id, info: transfer['bytes_expected'])
119
104
  pre_calc = true
120
105
  else
121
- notify_progress(:transfer, session_id: session_id, info: transfer['bytes_written'])
106
+ notify_progress(:transfer, session_id: @transfer_id, info: transfer['bytes_written'])
122
107
  end
123
108
  when 'completed'
124
- notify_progress(:end, session_id: session_id)
109
+ notify_progress(:end, session_id: @transfer_id)
125
110
  break
126
111
  when 'failed'
127
- notify_progress(:end, session_id: session_id)
112
+ notify_progress(:end, session_id: @transfer_id)
128
113
  raise Transfer::Error, transfer['error_desc']
129
114
  when 'cancelled'
130
- notify_progress(:end, session_id: session_id)
115
+ notify_progress(:end, session_id: @transfer_id)
131
116
  raise Transfer::Error, 'Transfer cancelled by user'
132
117
  else
133
- notify_progress(:end, session_id: session_id)
118
+ notify_progress(:end, session_id: @transfer_id)
134
119
  raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
135
120
  end
136
121
  end
@@ -141,6 +126,23 @@ module Aspera
141
126
  end
142
127
  return [:success]
143
128
  end
129
+
130
+ private
131
+
132
+ # @return the file path of local connect where API's URI can be read
133
+ def connect_api_url
134
+ connect_locations = Products::Other.find(Products::Connect.locations).first
135
+ raise "Product: #{name} not found, please install." if connect_locations.nil?
136
+ folder = File.join(connect_locations[:run_root], 'var', 'run')
137
+ ['', 's'].each do |ext|
138
+ uri_file = File.join(folder, "http#{ext}.uri")
139
+ Log.log.debug{"checking connect port file: #{uri_file}"}
140
+ if File.exist?(uri_file)
141
+ return File.open(uri_file, &:gets).strip
142
+ end
143
+ end
144
+ raise "no connect uri file found in #{folder}"
145
+ end
144
146
  end
145
147
  end
146
148
  end
@@ -19,7 +19,7 @@ module Aspera
19
19
 
20
20
  def initialize(**base_options)
21
21
  @application_id = SecureRandom.uuid
22
- @xfer_id = nil
22
+ @transfer_id = nil
23
23
  super
24
24
  raise 'Using client requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
25
25
  method_index = 0
@@ -44,28 +44,14 @@ module Aspera
44
44
  end
45
45
  end
46
46
 
47
- def aspera_client_api_url
48
- log_file = Products::Desktop.log_file
49
- url = 'http://127.0.0.1:33024'
50
- File.open(log_file, 'r') do |file|
51
- file.each_line do |line|
52
- line = line.chomp
53
- if (m = line.match(/JSON-RPC server listening on (.*)/))
54
- url = "http://#{m[1]}"
55
- end
56
- end
57
- end
58
- # raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
59
- return url
60
- end
61
-
47
+ # :reek:UnusedParameters token_regenerator
62
48
  def start_transfer(transfer_spec, token_regenerator: nil)
63
49
  @request_id = SecureRandom.uuid
64
50
  # if there is a token, we ask the client app to use well known ssh private keys
65
51
  # instead of asking password
66
52
  transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
67
53
  result = @client_app_api.start_transfer(app_id: @application_id, desktop_spec: {}, transfer_spec: transfer_spec)
68
- @xfer_id = result['uuid']
54
+ @transfer_id = result['uuid']
69
55
  end
70
56
 
71
57
  def wait_for_transfers_completion
@@ -73,32 +59,32 @@ module Aspera
73
59
  pre_calc = false
74
60
  begin
75
61
  loop do
76
- transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @xfer_id)
62
+ transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @transfer_id)
77
63
  case transfer['status']
78
64
  when 'initiating', 'queued'
79
65
  notify_progress(:pre_start, session_id: nil, info: transfer['status'])
80
66
  when 'running'
81
67
  if !started
82
- notify_progress(:session_start, session_id: @xfer_id)
68
+ notify_progress(:session_start, session_id: @transfer_id)
83
69
  started = true
84
70
  end
85
71
  if !pre_calc && (transfer['bytes_expected'] != 0)
86
- notify_progress(:session_size, session_id: @xfer_id, info: transfer['bytes_expected'])
72
+ notify_progress(:session_size, session_id: @transfer_id, info: transfer['bytes_expected'])
87
73
  pre_calc = true
88
74
  else
89
- notify_progress(:transfer, session_id: @xfer_id, info: transfer['bytes_written'])
75
+ notify_progress(:transfer, session_id: @transfer_id, info: transfer['bytes_written'])
90
76
  end
91
77
  when 'completed'
92
- notify_progress(:end, session_id: @xfer_id)
78
+ notify_progress(:end, session_id: @transfer_id)
93
79
  break
94
80
  when 'failed'
95
- notify_progress(:end, session_id: @xfer_id)
81
+ notify_progress(:end, session_id: @transfer_id)
96
82
  raise Transfer::Error, transfer['error_desc']
97
83
  when 'cancelled'
98
- notify_progress(:end, session_id: @xfer_id)
84
+ notify_progress(:end, session_id: @transfer_id)
99
85
  raise Transfer::Error, 'Transfer cancelled by user'
100
86
  else
101
- notify_progress(:end, session_id: @xfer_id)
87
+ notify_progress(:end, session_id: @transfer_id)
102
88
  raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
103
89
  end
104
90
  sleep(1)
@@ -108,6 +94,24 @@ module Aspera
108
94
  end
109
95
  return [:success]
110
96
  end
97
+
98
+ private
99
+
100
+ # @return [String] the url where transferd is listening
101
+ def aspera_client_api_url
102
+ log_file = Products::Desktop.log_file
103
+ url = 'http://127.0.0.1:33024'
104
+ File.open(log_file, 'r') do |file|
105
+ file.each_line do |line|
106
+ line = line.chomp
107
+ if (m = line.match(/JSON-RPC server listening on (.*)/))
108
+ url = "http://#{m[1]}"
109
+ end
110
+ end
111
+ end
112
+ # raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
113
+ return url
114
+ end
111
115
  end
112
116
  end
113
117
  end
@@ -16,15 +16,68 @@ require 'English'
16
16
 
17
17
  module Aspera
18
18
  module Agent
19
- # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
19
+ # executes a local "ascp", create mgt port
20
20
  class Direct < Base
21
+ # ascp started locally, so listen local
21
22
  LISTEN_LOCAL_ADDRESS = '127.0.0.1'
22
- # 0 means: select an available port
23
+ # 0 means: use any available port
23
24
  SELECT_AVAILABLE_PORT = 0
24
- # spellchecker: enable
25
25
  private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
26
26
 
27
- # method of Base
27
+ # options for initialize (same as values in option transfer_info)
28
+ # @param ascp_args [Array] additional arguments to ascp
29
+ # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
30
+ # @param quiet [Boolean] by default no native ascp progress bar
31
+ # @param monitor [Boolean] set to false to eliminate management port
32
+ # @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
33
+ # @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
34
+ # @param check_ignore_cb [Proc] callback with host,port
35
+ # @param spawn_timeout_sec [Integer] timeout for ascp spawn
36
+ # @param spawn_delay_sec [Integer] optional delay to start between sessions
37
+ # @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
38
+ # @param resume [Hash,NilClass] resume policy
39
+ # @param management_cb [Proc] callback for management events
40
+ # @param base_options [Hash] other options for base class
41
+ def initialize(
42
+ ascp_args: nil,
43
+ wss: true,
44
+ quiet: true,
45
+ trusted_certs: nil,
46
+ client_ssh_key: nil,
47
+ check_ignore_cb: nil,
48
+ spawn_timeout_sec: 2,
49
+ spawn_delay_sec: 2,
50
+ multi_incr_udp: nil,
51
+ resume: nil,
52
+ monitor: true,
53
+ management_cb: nil,
54
+ **base_options
55
+ )
56
+ super(**base_options)
57
+ # special transfer parameters
58
+ @tr_opts = {
59
+ ascp_args: ascp_args,
60
+ wss: wss,
61
+ quiet: quiet,
62
+ trusted_certs: trusted_certs,
63
+ client_ssh_key: client_ssh_key,
64
+ check_ignore_cb: check_ignore_cb
65
+ }
66
+ @spawn_timeout_sec = spawn_timeout_sec
67
+ @spawn_delay_sec = spawn_delay_sec
68
+ # default is true on Windows, false on other OSes
69
+ @multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
70
+ @monitor = monitor
71
+ @management_cb = management_cb
72
+ @resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
73
+ # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
74
+ @sessions = []
75
+ # mutex protects global data accessed by threads
76
+ @mutex = Mutex.new
77
+ @pre_calc_sent = false
78
+ @pre_calc_last_size = nil
79
+ end
80
+
28
81
  # start ascp transfer(s) (non blocking), single or multi-session
29
82
  # session information added to @sessions
30
83
  # @param transfer_spec [Hash] aspera transfer specification
@@ -48,11 +101,10 @@ module Aspera
48
101
  # (even if the var is not used in single session)
49
102
  multi_session_info = nil
50
103
  if transfer_spec.key?('multi_session')
104
+ # Managed by multi-session, so delete from transfer spec
51
105
  multi_session_info = {
52
- count: transfer_spec['multi_session'].to_i
106
+ count: transfer_spec.delete('multi_session').to_i
53
107
  }
54
- # Managed by multi-session, so delete from transfer spec
55
- transfer_spec.delete('multi_session')
56
108
  if multi_session_info[:count].negative?
57
109
  Log.log.error{"multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0"}
58
110
  multi_session_info = nil
@@ -77,14 +129,14 @@ module Aspera
77
129
  error: nil, # exception if failed
78
130
  io: nil, # management port server socket
79
131
  token_regenerator: token_regenerator, # regenerate bearer token with oauth
80
- # env vars and args to ascp (from transfer spec)
132
+ # env vars and args for ascp (from transfer spec)
81
133
  env_args: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
82
134
  }
83
135
 
84
136
  if multi_session_info.nil?
85
137
  Log.log.debug('Starting single session thread')
86
138
  # single session for transfer : simple
87
- session[:thread] = Thread.new {transfer_thread_entry(session)}
139
+ session[:thread] = Thread.new{transfer_thread_entry(session)}
88
140
  @sessions.push(session)
89
141
  else
90
142
  Log.log.debug('Starting multi session threads')
@@ -101,7 +153,7 @@ module Aspera
101
153
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
102
154
  args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
103
155
  # finally start the thread
104
- this_session[:thread] = Thread.new {transfer_thread_entry(this_session)}
156
+ this_session[:thread] = Thread.new{transfer_thread_entry(this_session)}
105
157
  @sessions.push(this_session)
106
158
  end
107
159
  end
@@ -132,9 +184,44 @@ module Aspera
132
184
 
133
185
  # @return [Array] list of sessions for a job
134
186
  def sessions_by_job(job_id)
135
- @sessions.select{|session| session[:job_id].eql?(job_id)}
187
+ @sessions.select{ |session| session[:job_id].eql?(job_id)}
136
188
  end
137
189
 
190
+ # send command to management port of command (used in `asession)
191
+ # @param job_id identified transfer process
192
+ # @param session_index index of session (for multi session)
193
+ # @param data command on mgt port, examples:
194
+ # {'type'=>'START','source'=>_path_,'destination'=>_path_}
195
+ # {'type'=>'DONE'}
196
+ def send_command(job_id, data)
197
+ session = @sessions.find{ |session| session[:job_id].eql?(job_id)}
198
+ Log.log.debug{"command: #{data}"}
199
+ session[:io].puts(Ascp::Management.command_to_stream(data))
200
+ end
201
+
202
+ private
203
+
204
+ # transfer thread entry
205
+ # @param session information
206
+ def transfer_thread_entry(session)
207
+ begin
208
+ # set name for logging
209
+ Thread.current[:name] = 'transfer'
210
+ Log.log.debug{"ENTER (#{Thread.current[:name]})"}
211
+ # start transfer with selected resumer policy
212
+ @resume_policy.execute_with_resume do
213
+ start_and_monitor_process(session: session, **session[:env_args])
214
+ end
215
+ Log.log.debug('transfer ok'.bg_green)
216
+ rescue StandardError => e
217
+ session[:error] = e
218
+ Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
219
+ end
220
+ Log.log.debug{"EXIT (#{Thread.current[:name]})"}
221
+ end
222
+
223
+ public
224
+
138
225
  # This is the low level method to start the transfer process.
139
226
  # Typically started in a thread.
140
227
  # Start process with management port.
@@ -154,18 +241,21 @@ module Aspera
154
241
  notify_progress(:pre_start, session_id: nil, info: 'starting')
155
242
  begin
156
243
  command_pid = nil
157
- # we use Socket directly, instead of TCPServer, as it gives access to lower level options
158
- socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
159
- mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
160
- # open any available (0) local TCP port for use as management port
161
- mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
162
- # make port ready to accept connections, before starting ascp
163
- mgt_server_socket.listen(1)
164
- # build arguments and add mgt port
165
- command_arguments = if name.eql?(:async)
166
- ["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
167
- else
168
- ['-M', mgt_server_socket.local_address.ip_port.to_s]
244
+ command_arguments = []
245
+ if @monitor
246
+ # we use Socket directly, instead of TCPServer, as it gives access to lower level options
247
+ socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
248
+ mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
249
+ # open any available (0) local TCP port for use as management port
250
+ mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
251
+ # make port ready to accept connections, before starting ascp
252
+ mgt_server_socket.listen(1)
253
+ # build arguments and add mgt port
254
+ command_arguments = if name.eql?(:async)
255
+ ["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
256
+ else
257
+ ['-M', mgt_server_socket.local_address.ip_port.to_s]
258
+ end
169
259
  end
170
260
  command_arguments.concat(args)
171
261
  # capture process stderr
@@ -175,6 +265,8 @@ module Aspera
175
265
  command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments, err: stderr_w)
176
266
  stderr_w.close
177
267
  notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
268
+ # "ensure" block will wait for process
269
+ return unless @monitor
178
270
  # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
179
271
  # So we use select to detect that anything happens on the socket (connection)
180
272
  Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
@@ -185,9 +277,9 @@ module Aspera
185
277
  client_socket, _client_addrinfo = mgt_server_socket.accept
186
278
  Log.log.debug('after accept')
187
279
  management_port_io = client_socket.to_io
188
- # management messages include file names which may be utf8
189
280
  # by default socket is US-ASCII
190
- # TODO: use same value as Encoding.default_external
281
+ # management messages include file names which may be UTF-8
282
+ # TODO: use same value as Encoding.default_external ?
191
283
  management_port_io.set_encoding(Encoding::UTF_8)
192
284
  session[:io] = management_port_io
193
285
  processor = Ascp::Management.new
@@ -197,7 +289,7 @@ module Aspera
197
289
  next unless event
198
290
  # event is ready
199
291
  Log.log.trace1{Log.dump(:management_port, event)}
200
- # store latest event by type
292
+ # store session identifier
201
293
  session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
202
294
  @management_cb&.call(event)
203
295
  process_progress(event)
@@ -206,12 +298,10 @@ module Aspera
206
298
  Log.log.debug('management io closed')
207
299
  # check that last status was received before process exit
208
300
  last_event = processor.last_event
209
- # process stderr of ascp
210
- stderr_r&.each_line do |line|
211
- Log.log.error(line.chomp)
212
- end
213
- raise Transfer::Error, "internal: no management event (#{last_event.class})" unless last_event.is_a?(Hash)
301
+ raise Transfer::Error, "No management event (#{last_event.class})" unless last_event.is_a?(Hash)
214
302
  case last_event['Type']
303
+ when 'DONE'
304
+ Log.log.trace1{'Graceful shutdown, DONE message received'}
215
305
  when 'ERROR'
216
306
  if /bearer token/i.match?(last_event['Description']) &&
217
307
  session[:token_regenerator].respond_to?(:refreshed_transfer_token)
@@ -221,10 +311,9 @@ module Aspera
221
311
  env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
222
312
  end
223
313
  raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
224
- when 'DONE'
225
- nil
226
314
  else
227
- raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
315
+ Log.log.error{"unexpected last event type: #{last_event['Type']}"}
316
+ # raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
228
317
  end
229
318
  rescue SystemCallError => e
230
319
  # Process.spawn failed, or socket error
@@ -232,18 +321,21 @@ module Aspera
232
321
  rescue Interrupt
233
322
  raise Transfer::Error, 'transfer interrupted by user'
234
323
  ensure
235
- mgt_server_socket.close
236
- stderr_r&.close
324
+ mgt_server_socket&.close
325
+ session.delete(:io)
237
326
  # if command was successfully started, check its status
238
327
  unless command_pid.nil?
239
- # "wait" for process to avoid zombie
240
- Process.wait(command_pid)
241
- status = $CHILD_STATUS
242
- # command_pid = nil
243
- session.delete(:io)
328
+ Process.kill(:INT, command_pid) if @monitor
329
+ # collect process exit status or wait for termination
330
+ _, status = Process.wait2(command_pid)
331
+ # process stderr of ascp
332
+ stderr_r.each_line do |line|
333
+ Log.log.error(line.chomp)
334
+ end
335
+ stderr_r.close
244
336
  # status is nil if an exception occurred before starting command
245
337
  if !status&.success?
246
- message = status.nil? ? "#{name} not started" : "#{name} failed (#{status})"
338
+ message = "#{name} failed (#{status})"
247
339
  # raise error only if there was not already an exception (ERROR_INFO)
248
340
  raise Transfer::Error, message unless $ERROR_INFO
249
341
  # else display this message also, as main exception is already here
@@ -254,10 +346,10 @@ module Aspera
254
346
  nil
255
347
  end
256
348
 
257
- attr_reader :sessions
258
-
259
349
  private
260
350
 
351
+ attr_reader :sessions
352
+
261
353
  # notify progress to callback
262
354
  # @param event management port event
263
355
  # @param session sessin object
@@ -293,88 +385,8 @@ module Aspera
293
385
  # cspell:enable
294
386
  # stop event when one file is completed
295
387
  else
296
- Log.log.debug{"unknown event type #{event['Type']}"}
297
- end
298
- end
299
-
300
- # send command to management port of command (used in `asession)
301
- # @param job_id identified transfer process
302
- # @param session_index index of session (for multi session)
303
- # @param data command on mgt port, examples:
304
- # {'type'=>'START','source'=>_path_,'destination'=>_path_}
305
- # {'type'=>'DONE'}
306
- def send_command(job_id, data)
307
- session = @sessions.find{|session| session[:job_id].eql?(job_id)}
308
- Log.log.debug{"command: #{data}"}
309
- session[:io].puts(Ascp::Management.command_to_stream(data))
310
- end
311
-
312
- # options for initialize (same as values in option transfer_info)
313
- # @param ascp_args [Array] additional arguments to ascp
314
- # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
315
- # @param quiet [Boolean] by default no native ascp progress bar
316
- # @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
317
- # @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
318
- # @param check_ignore_cb [Proc] callback with host,port
319
- # @param spawn_timeout_sec [Integer] timeout for ascp spawn
320
- # @param spawn_delay_sec [Integer] optional delay to start between sessions
321
- # @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
322
- # @param resume [Hash,NilClass] resume policy
323
- # @param management_cb [Proc] callback for management events
324
- # @param base_options [Hash] other options for base class
325
- def initialize(
326
- ascp_args: nil,
327
- wss: true,
328
- quiet: true,
329
- trusted_certs: nil,
330
- client_ssh_key: nil,
331
- check_ignore_cb: nil,
332
- spawn_timeout_sec: 2,
333
- spawn_delay_sec: 2,
334
- multi_incr_udp: nil,
335
- resume: nil,
336
- management_cb: nil,
337
- **base_options
338
- )
339
- super(**base_options)
340
- # special transfer parameters
341
- @tr_opts = {
342
- ascp_args: ascp_args,
343
- wss: wss,
344
- quiet: quiet,
345
- trusted_certs: trusted_certs,
346
- client_ssh_key: client_ssh_key,
347
- check_ignore_cb: check_ignore_cb
348
- }
349
- @spawn_timeout_sec = spawn_timeout_sec
350
- @spawn_delay_sec = spawn_delay_sec
351
- # default is true on windows, false on other platforms
352
- @multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
353
- @management_cb = management_cb
354
- @resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
355
- # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
356
- @sessions = []
357
- # mutex protects global data accessed by threads
358
- @mutex = Mutex.new
359
- end
360
-
361
- # transfer thread entry
362
- # @param session information
363
- def transfer_thread_entry(session)
364
- begin
365
- # set name for logging
366
- Thread.current[:name] = 'transfer'
367
- Log.log.debug{"ENTER (#{Thread.current[:name]})"}
368
- # start transfer with selected resumer policy
369
- @resume_policy.execute_with_resume do
370
- start_and_monitor_process(session: session, **session[:env_args])
371
- end
372
- Log.log.debug('transfer ok'.bg_green)
373
- rescue StandardError => e
374
- session[:error] = e
375
- Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
388
+ Log.log.debug{"Unknown event type for progress: #{event['Type']}"}
376
389
  end
377
- Log.log.debug{"EXIT (#{Thread.current[:name]})"}
378
390
  end
379
391
  end
380
392
  end