aspera-cli 4.20.0 → 4.21.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +41 -3
  4. data/CONTRIBUTING.md +69 -142
  5. data/README.md +687 -461
  6. data/bin/ascli +5 -14
  7. data/bin/asession +3 -5
  8. data/examples/get_proto_file.rb +4 -3
  9. data/examples/proxy.pac +20 -20
  10. data/lib/aspera/agent/base.rb +2 -0
  11. data/lib/aspera/agent/connect.rb +20 -2
  12. data/lib/aspera/agent/{alpha.rb → desktop.rb} +12 -18
  13. data/lib/aspera/agent/direct.rb +30 -31
  14. data/lib/aspera/agent/node.rb +1 -11
  15. data/lib/aspera/agent/{trsdk.rb → transferd.rb} +37 -51
  16. data/lib/aspera/api/alee.rb +1 -1
  17. data/lib/aspera/api/aoc.rb +13 -8
  18. data/lib/aspera/api/cos_node.rb +1 -1
  19. data/lib/aspera/api/node.rb +49 -32
  20. data/lib/aspera/ascp/installation.rb +98 -77
  21. data/lib/aspera/ascp/management.rb +27 -6
  22. data/lib/aspera/cli/extended_value.rb +9 -3
  23. data/lib/aspera/cli/formatter.rb +155 -154
  24. data/lib/aspera/cli/info.rb +2 -1
  25. data/lib/aspera/cli/main.rb +12 -0
  26. data/lib/aspera/cli/manager.rb +4 -4
  27. data/lib/aspera/cli/plugin.rb +2 -2
  28. data/lib/aspera/cli/plugins/aoc.rb +134 -73
  29. data/lib/aspera/cli/plugins/config.rb +114 -83
  30. data/lib/aspera/cli/plugins/cos.rb +1 -0
  31. data/lib/aspera/cli/plugins/faspex.rb +4 -2
  32. data/lib/aspera/cli/plugins/faspex5.rb +29 -14
  33. data/lib/aspera/cli/plugins/node.rb +51 -41
  34. data/lib/aspera/cli/transfer_progress.rb +2 -0
  35. data/lib/aspera/cli/version.rb +1 -1
  36. data/lib/aspera/command_line_builder.rb +1 -1
  37. data/lib/aspera/coverage.rb +5 -3
  38. data/lib/aspera/environment.rb +59 -16
  39. data/lib/aspera/faspex_postproc.rb +3 -5
  40. data/lib/aspera/hash_ext.rb +2 -12
  41. data/lib/aspera/node_simulator.rb +230 -112
  42. data/lib/aspera/oauth/base.rb +40 -48
  43. data/lib/aspera/oauth/factory.rb +41 -2
  44. data/lib/aspera/oauth/jwt.rb +4 -1
  45. data/lib/aspera/persistency_action_once.rb +1 -1
  46. data/lib/aspera/persistency_folder.rb +20 -2
  47. data/lib/aspera/preview/generator.rb +13 -10
  48. data/lib/aspera/preview/options.rb +2 -2
  49. data/lib/aspera/preview/terminal.rb +1 -1
  50. data/lib/aspera/preview/utils.rb +11 -6
  51. data/lib/aspera/products/connect.rb +82 -0
  52. data/lib/aspera/products/desktop.rb +30 -0
  53. data/lib/aspera/products/other.rb +82 -0
  54. data/lib/aspera/products/transferd.rb +61 -0
  55. data/lib/aspera/rest.rb +22 -17
  56. data/lib/aspera/secret_hider.rb +9 -2
  57. data/lib/aspera/ssh.rb +31 -24
  58. data/lib/aspera/temp_file_manager.rb +5 -4
  59. data/lib/aspera/transfer/parameters.rb +2 -1
  60. data/lib/aspera/transfer/spec.yaml +22 -20
  61. data/lib/aspera/transfer/sync.rb +1 -5
  62. data/lib/aspera/transfer/uri.rb +2 -2
  63. data/lib/aspera/uri_reader.rb +18 -1
  64. data/lib/transferd_pb.rb +86 -0
  65. data/lib/transferd_services_pb.rb +84 -0
  66. data.tar.gz.sig +0 -0
  67. metadata +13 -166
  68. metadata.gz.sig +0 -0
  69. data/examples/build_exec +0 -74
  70. data/examples/build_exec_rubyc +0 -40
  71. data/lib/aspera/ascp/products.rb +0 -168
  72. data/lib/transfer_pb.rb +0 -84
  73. data/lib/transfer_services_pb.rb +0 -82
data/bin/ascli CHANGED
@@ -9,17 +9,8 @@ Encoding.default_internal = Encoding::UTF_8
9
9
  Encoding.default_external = Encoding::UTF_8
10
10
  $VERBOSE = old_verbose
11
11
 
12
- begin
13
- gem_lib_folder = File.join(File.dirname(File.dirname(File.realpath(__FILE__))), 'lib')
14
- Kernel.load(File.join(gem_lib_folder, 'aspera/coverage.rb'))
15
- begin
16
- require 'aspera/cli/main'
17
- rescue LoadError
18
- # if in development, add path toward gem
19
- $LOAD_PATH.unshift(gem_lib_folder)
20
- require 'aspera/cli/main'
21
- end
22
- require 'aspera/environment'
23
- Aspera::Environment.fix_home
24
- Aspera::Cli::Main.new(ARGV).process_command_line
25
- end
12
+ require 'aspera/coverage'
13
+ require 'aspera/environment'
14
+ require 'aspera/cli/main'
15
+ Aspera::Environment.fix_home
16
+ Aspera::Cli::Main.new(ARGV).process_command_line
data/bin/asession CHANGED
@@ -1,12 +1,10 @@
1
1
  #!/usr/bin/env ruby
2
2
  # frozen_string_literal: true
3
3
 
4
- gem_lib_folder = File.join(File.dirname(File.dirname(File.realpath(__FILE__))), 'lib')
5
- Kernel.load(File.join(gem_lib_folder, 'aspera/coverage.rb'))
6
- $LOAD_PATH.unshift(gem_lib_folder)
4
+ require 'aspera/coverage'
7
5
  require 'aspera/agent/direct'
8
6
  require 'aspera/cli/extended_value'
9
- require 'aspera/ascp/installation'
7
+ require 'aspera/products/transferd'
10
8
  require 'aspera/log'
11
9
  require 'json'
12
10
  # extended transfer spec parameter (only used in asession)
@@ -77,7 +75,7 @@ if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
77
75
  Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER]
78
76
  end
79
77
  session_spec[PARAM_SDK] = File.join(Dir.home, '.aspera', 'sdk') unless session_spec.key?(PARAM_SDK)
80
- Aspera::Ascp::Installation.instance.sdk_folder = session_spec[PARAM_SDK]
78
+ Aspera::Products::Transferd.sdk_directory = session_spec[PARAM_SDK]
81
79
  session_spec[PARAM_AGENT] = {} unless session_spec.key?(PARAM_AGENT)
82
80
  agent_params = session_spec[PARAM_AGENT]
83
81
  agent_params['quiet'] = true
@@ -1,7 +1,8 @@
1
1
  #!/usr/bin/env ruby
2
2
  # frozen_string_literal: true
3
3
 
4
- # Retrieve `transfer.proto` from the web
5
- $LOAD_PATH.unshift(File.join(File.dirname(File.dirname(File.realpath(__FILE__))), 'lib'))
6
4
  require 'aspera/ascp/installation'
7
- Aspera::Ascp::Installation.instance.install_sdk(folder: ARGV.first, backup: false, with_exe: false) {|name| '/' if name.end_with?('transfer.proto')}
5
+ require 'aspera/cli/transfer_progress'
6
+ Aspera::RestParameters.instance.progress_bar = Aspera::Cli::TransferProgress.new
7
+ # Retrieve `transfer.proto` from the web
8
+ Aspera::Ascp::Installation.instance.install_sdk(folder: ARGV.first, backup: false, with_exe: false) {|name| name.end_with?('.proto') ? '/' : nil }
data/examples/proxy.pac CHANGED
@@ -11,20 +11,20 @@ function FindProxyForURL(url, host) {
11
11
 
12
12
  /* Don't proxy local domains */
13
13
  if (dnsDomainIs(host, ".example1.com") || (host == "example1.com")
14
- || dnsDomainIs(host, ".example2.com") || (host == "example2.com")
15
- || dnsDomainIs(host, ".example3.com") || (host == "example3.com")) {
14
+ || dnsDomainIs(host, ".example2.com") || (host == "example2.com")
15
+ || dnsDomainIs(host, ".example3.com") || (host == "example3.com")) {
16
16
  return 'DIRECT';
17
17
  }
18
18
 
19
19
  /* Don't proxy Windows Update */
20
20
  if ((host == "download.microsoft.com")
21
- || (host == "ntservicepack.microsoft.com")
22
- || (host == "cdm.microsoft.com") || (host == "wustat.windows.com")
23
- || (host == "windowsupdate.microsoft.com")
24
- || (dnsDomainIs(host, ".windowsupdate.microsoft.com"))
25
- || (host == "update.microsoft.com")
26
- || (dnsDomainIs(host, ".update.microsoft.com"))
27
- || (dnsDomainIs(host, ".windowsupdate.com"))) {
21
+ || (host == "ntservicepack.microsoft.com")
22
+ || (host == "cdm.microsoft.com") || (host == "wustat.windows.com")
23
+ || (host == "windowsupdate.microsoft.com")
24
+ || (dnsDomainIs(host, ".windowsupdate.microsoft.com"))
25
+ || (host == "update.microsoft.com")
26
+ || (dnsDomainIs(host, ".update.microsoft.com"))
27
+ || (dnsDomainIs(host, ".windowsupdate.com"))) {
28
28
  return 'DIRECT';
29
29
  }
30
30
 
@@ -33,22 +33,22 @@ function FindProxyForURL(url, host) {
33
33
 
34
34
  /* Don't proxy private addresses (RFC 3330) */
35
35
  if (isInNet(hostIP, '0.0.0.0', '255.0.0.0')
36
- || isInNet(hostIP, '10.0.0.0', '255.0.0.0')
37
- || isInNet(hostIP, '127.0.0.0', '255.0.0.0')
38
- || isInNet(hostIP, '169.254.0.0', '255.255.0.0')
39
- || isInNet(hostIP, '172.16.0.0', '255.240.0.0')
40
- || isInNet(hostIP, '192.0.2.0', '255.255.255.0')
41
- || isInNet(hostIP, '192.88.99.0', '255.255.255.0')
42
- || isInNet(hostIP, '192.168.0.0', '255.255.0.0')
43
- || isInNet(hostIP, '198.18.0.0', '255.254.0.0')
44
- || isInNet(hostIP, '224.0.0.0', '240.0.0.0')
45
- || isInNet(hostIP, '240.0.0.0', '240.0.0.0')) {
36
+ || isInNet(hostIP, '10.0.0.0', '255.0.0.0')
37
+ || isInNet(hostIP, '127.0.0.0', '255.0.0.0')
38
+ || isInNet(hostIP, '169.254.0.0', '255.255.0.0')
39
+ || isInNet(hostIP, '172.16.0.0', '255.240.0.0')
40
+ || isInNet(hostIP, '192.0.2.0', '255.255.255.0')
41
+ || isInNet(hostIP, '192.88.99.0', '255.255.255.0')
42
+ || isInNet(hostIP, '192.168.0.0', '255.255.0.0')
43
+ || isInNet(hostIP, '198.18.0.0', '255.254.0.0')
44
+ || isInNet(hostIP, '224.0.0.0', '240.0.0.0')
45
+ || isInNet(hostIP, '240.0.0.0', '240.0.0.0')) {
46
46
  return 'DIRECT';
47
47
  }
48
48
  }
49
49
 
50
50
  if (url.substring(0, 5) == 'http:' || url.substring(0, 6) == 'https:'
51
- || url.substring(0, 4) == 'ftp:') {
51
+ || url.substring(0, 4) == 'ftp:') {
52
52
  return 'PROXY proxy.example.com:8080';
53
53
  }
54
54
 
@@ -7,6 +7,7 @@ module Aspera
7
7
  # Base class for transfer agents
8
8
  class Base
9
9
  RUBY_EXT = '.rb'
10
+ private_constant :RUBY_EXT
10
11
  class << self
11
12
  def factory_create(agent, options)
12
13
  # Aspera.assert_values(agent, agent_list)
@@ -15,6 +16,7 @@ module Aspera
15
16
  end
16
17
 
17
18
  # discover available agents
19
+ # @return [Array] list of symbols of agents
18
20
  def agent_list
19
21
  base_class = File.basename(__FILE__)
20
22
  Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
@@ -1,7 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/agent/base'
4
+ require 'aspera/products/connect'
5
+ require 'aspera/products/other'
4
6
  require 'aspera/rest'
7
+ require 'aspera/environment'
5
8
  require 'securerandom'
6
9
 
7
10
  module Aspera
@@ -13,14 +16,14 @@ module Aspera
13
16
  SLEEP_SEC_BETWEEN_RETRY = 5
14
17
  private_constant :CONNECT_START_URIS, :SLEEP_SEC_BETWEEN_RETRY
15
18
  def initialize(**base_options)
16
- super(**base_options)
19
+ super
17
20
  @connect_settings = {
18
21
  'app_id' => SecureRandom.uuid
19
22
  }
20
23
  raise 'Using connect requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
21
24
  method_index = 0
22
25
  begin
23
- connect_url = Ascp::Products.connect_uri
26
+ connect_url = connect_api_url
24
27
  Log.log.debug{"found: #{connect_url}"}
25
28
  @connect_api = Rest.new(
26
29
  base_url: "#{connect_url}/v5/connect", # could use v6 also now
@@ -43,6 +46,21 @@ module Aspera
43
46
  end
44
47
  end
45
48
 
49
+ # @return the file path of local connect where API's URI can be read
50
+ def connect_api_url
51
+ connect_locations = Products::Other.find(Products::Connect.locations).first
52
+ raise "Product: #{name} not found, please install." if connect_locations.nil?
53
+ folder = File.join(connect_locations[:run_root], 'var', 'run')
54
+ ['', 's'].each do |ext|
55
+ uri_file = File.join(folder, "http#{ext}.uri")
56
+ Log.log.debug{"checking connect port file: #{uri_file}"}
57
+ if File.exist?(uri_file)
58
+ return File.open(uri_file, &:gets).strip
59
+ end
60
+ end
61
+ raise "no connect uri file found in #{folder}"
62
+ end
63
+
46
64
  def start_transfer(transfer_spec, token_regenerator: nil)
47
65
  if transfer_spec['direction'] == 'send'
48
66
  Log.log.warn{"Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red}
@@ -2,26 +2,25 @@
2
2
 
3
3
  require 'aspera/agent/base'
4
4
  require 'aspera/rest'
5
- require 'aspera/log'
6
- require 'aspera/json_rpc'
7
5
  require 'aspera/environment'
6
+ require 'aspera/json_rpc'
7
+ require 'aspera/products/desktop'
8
8
  require 'securerandom'
9
9
 
10
10
  module Aspera
11
11
  module Agent
12
- # Aspera Desktop Alpha Client
13
- class Alpha < Base
12
+ # Client: Aspera for Desktop
13
+ class Desktop < Base
14
14
  # try twice the main init url in sequence
15
15
  START_URIS = ['aspera://', 'aspera://', 'aspera://']
16
16
  # delay between each try to start the app
17
17
  SLEEP_SEC_BETWEEN_RETRY = 5
18
- APP_IDENTIFIER = 'com.ibm.software.aspera.desktop'
19
- APP_NAME = 'Aspera Desktop Alpha Client'
20
18
  private_constant :START_URIS, :SLEEP_SEC_BETWEEN_RETRY
19
+
21
20
  def initialize(**base_options)
22
21
  @application_id = SecureRandom.uuid
23
22
  @xfer_id = nil
24
- super(**base_options)
23
+ super
25
24
  raise 'Using client requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
26
25
  method_index = 0
27
26
  begin
@@ -34,24 +33,20 @@ module Aspera
34
33
  rescue Errno::ECONNREFUSED => e
35
34
  start_url = START_URIS[method_index]
36
35
  method_index += 1
37
- raise StandardError, "Unable to start #{APP_NAME} #{method_index} times" if start_url.nil?
38
- Log.log.warn{"#{APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
36
+ raise StandardError, "Unable to start #{Products::Desktop::APP_NAME} #{method_index} times" if start_url.nil?
37
+ Log.log.warn{"#{Products::Desktop::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
39
38
  if !Environment.open_uri_graphical(start_url)
40
39
  Environment.open_uri_graphical('https://www.ibm.com/aspera/connect/')
41
- raise StandardError, "#{APP_NAME} is not installed"
40
+ raise StandardError, "#{Products::Desktop::APP_NAME} is not installed"
42
41
  end
43
42
  sleep(SLEEP_SEC_BETWEEN_RETRY)
44
43
  retry
45
44
  end
46
45
  end
47
46
 
48
- def sdk_log_file
49
- File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER, 'ibm-aspera-desktop.log')
50
- end
51
-
52
47
  def aspera_client_api_url
53
- log_file = sdk_log_file
54
- url = nil
48
+ log_file = Products::Desktop.log_file
49
+ url = 'http://127.0.0.1:33024'
55
50
  File.open(log_file, 'r') do |file|
56
51
  file.each_line do |line|
57
52
  line = line.chomp
@@ -60,8 +55,7 @@ module Aspera
60
55
  end
61
56
  end
62
57
  end
63
- url = 'http://127.0.0.1:33024' if url.nil?
64
- raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
58
+ # raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
65
59
  return url
66
60
  end
67
61
 
@@ -72,7 +72,7 @@ module Aspera
72
72
  session = {
73
73
  id: nil, # SessionId from INIT message in mgt port
74
74
  job_id: SecureRandom.uuid, # job id (regroup sessions)
75
- ts: transfer_spec, # transfer spec
75
+ ts: transfer_spec, # global transfer spec
76
76
  thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
77
77
  error: nil, # exception if failed
78
78
  io: nil, # management port server socket
@@ -84,7 +84,7 @@ module Aspera
84
84
  if multi_session_info.nil?
85
85
  Log.log.debug('Starting single session thread')
86
86
  # single session for transfer : simple
87
- session[:thread] = Thread.new(session) {|session_info|transfer_thread_entry(session_info)}
87
+ session[:thread] = Thread.new {transfer_thread_entry(session)}
88
88
  @sessions.push(session)
89
89
  else
90
90
  Log.log.debug('Starting multi session threads')
@@ -101,7 +101,7 @@ module Aspera
101
101
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
102
102
  args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
103
103
  # finally start the thread
104
- this_session[:thread] = Thread.new(this_session) {|session_info|transfer_thread_entry(session_info)}
104
+ this_session[:thread] = Thread.new {transfer_thread_entry(this_session)}
105
105
  @sessions.push(this_session)
106
106
  end
107
107
  end
@@ -132,18 +132,18 @@ module Aspera
132
132
 
133
133
  # @return [Array] list of sessions for a job
134
134
  def sessions_by_job(job_id)
135
- @sessions.select{|session_info| session_info[:job_id].eql?(job_id)}
135
+ @sessions.select{|session| session[:job_id].eql?(job_id)}
136
136
  end
137
137
 
138
- # This is the low level method to start the transfer process
138
+ # This is the low level method to start the transfer process.
139
+ # Typically started in a thread.
139
140
  # Start process with management port.
140
- # returns
141
- # raises FaspError on error
142
141
  # @param session this session information, keys :io and :token_regenerator
143
- # @param env [Hash] environment variables
144
- # @param name [Symbol] name of executable: :ascp, :ascp4 or :async
145
- # @param args [Array] command line arguments
142
+ # @param env [Hash] environment variables (comes from ascp_args)
143
+ # @param name [Symbol] name of executable: :ascp, :ascp4 or :async (comes from ascp_args)
144
+ # @param args [Array] command line arguments (comes from ascp_args)
146
145
  # @return [nil] when process has exited
146
+ # @throw FaspError on error
147
147
  def start_and_monitor_process(
148
148
  session:,
149
149
  env:,
@@ -159,6 +159,8 @@ module Aspera
159
159
  mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
160
160
  # open any available (0) local TCP port for use as management port
161
161
  mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
162
+ # make port ready to accept connections, before starting ascp
163
+ mgt_server_socket.listen(1)
162
164
  # build arguments and add mgt port
163
165
  command_arguments = if name.eql?(:async)
164
166
  ["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
@@ -166,12 +168,15 @@ module Aspera
166
168
  ['-M', mgt_server_socket.local_address.ip_port.to_s]
167
169
  end
168
170
  command_arguments.concat(args)
171
+ # capture process stderr
172
+ stderr_r, stderr_w = IO.pipe
169
173
  # get location of command executable (ascp, async)
170
174
  command_path = Ascp::Installation.instance.path(name)
171
- command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments)
175
+ command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments, err: stderr_w)
176
+ stderr_w.close
172
177
  notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
173
- mgt_server_socket.listen(1)
174
178
  # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
179
+ # So we use select to detect that anything happens on the socket (connection)
175
180
  Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
176
181
  readable, _, _ = IO.select([mgt_server_socket], nil, nil, @spawn_timeout_sec)
177
182
  Log.log.debug('after select, before accept')
@@ -192,13 +197,19 @@ module Aspera
192
197
  next unless event
193
198
  # event is ready
194
199
  Log.log.trace1{Log.dump(:management_port, event)}
200
+ # store latest event by type
201
+ session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
195
202
  @management_cb&.call(event)
196
203
  process_progress(event)
197
- Log.log.error((event['Description']).to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
204
+ Log.log.error(event['Description'].to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
198
205
  end
199
206
  Log.log.debug('management io closed')
200
207
  # check that last status was received before process exit
201
208
  last_event = processor.last_event
209
+ # process stderr of ascp
210
+ stderr_r&.each_line do |line|
211
+ Log.log.error(line.chomp)
212
+ end
202
213
  raise Transfer::Error, "internal: no management event (#{last_event.class})" unless last_event.is_a?(Hash)
203
214
  case last_event['Type']
204
215
  when 'ERROR'
@@ -222,6 +233,7 @@ module Aspera
222
233
  raise Transfer::Error, 'transfer interrupted by user'
223
234
  ensure
224
235
  mgt_server_socket.close
236
+ stderr_r&.close
225
237
  # if command was successfully started, check its status
226
238
  unless command_pid.nil?
227
239
  # "wait" for process to avoid zombie
@@ -242,10 +254,13 @@ module Aspera
242
254
  nil
243
255
  end
244
256
 
257
+ attr_reader :sessions
258
+
245
259
  private
246
260
 
247
261
  # notify progress to callback
248
262
  # @param event management port event
263
+ # @param session sessin object
249
264
  def process_progress(event)
250
265
  session_id = event['SessionId']
251
266
  case event['Type']
@@ -282,14 +297,6 @@ module Aspera
282
297
  end
283
298
  end
284
299
 
285
- # @return [Hash] session information
286
- def session_by_id(id)
287
- matches = @sessions.select{|session_info| session_info[:id].eql?(id)}
288
- raise 'no such session' if matches.empty?
289
- raise 'more than one session' if matches.length > 1
290
- return matches.first
291
- end
292
-
293
300
  # send command to management port of command (used in `asession)
294
301
  # @param job_id identified transfer process
295
302
  # @param session_index index of session (for multi session)
@@ -297,18 +304,10 @@ module Aspera
297
304
  # {'type'=>'START','source'=>_path_,'destination'=>_path_}
298
305
  # {'type'=>'DONE'}
299
306
  def send_command(job_id, data)
300
- session = session_by_id(job_id)
307
+ session = @sessions.find{|session| session[:job_id].eql?(job_id)}
301
308
  Log.log.debug{"command: #{data}"}
302
- # build command
303
- command = data
304
- .keys
305
- .map{|k|"#{k.capitalize}: #{data[k]}"}
306
- .unshift(MGT_HEADER)
307
- .push('', '')
308
- .join("\n")
309
- session[:io].puts(command)
309
+ session[:io].puts(Ascp::Management.command_to_stream(data))
310
310
  end
311
- attr_reader :sessions
312
311
 
313
312
  # options for initialize (same as values in option transfer_info)
314
313
  # @param ascp_args [Array] additional arguments to ascp
@@ -50,14 +50,6 @@ module Aspera
50
50
  return @node_api
51
51
  end
52
52
 
53
- # use this to set the node_api end point before using the class.
54
- def node_api=(new_value)
55
- if !@node_api.nil? && !new_value.nil?
56
- Log.log.warn('overriding existing node api value')
57
- end
58
- @node_api = new_value
59
- end
60
-
61
53
  # generic method
62
54
  def start_transfer(transfer_spec, token_regenerator: nil)
63
55
  # add root id if access key
@@ -120,9 +112,7 @@ module Aspera
120
112
  # Bug in HSTS ? transfer is marked failed, but there is no reason
121
113
  break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
122
114
  raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
123
- else
124
- Log.log.warn{"transfer_data -> #{transfer_data}"}
125
- raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
115
+ else Aspera.error_unexpected_value(transfer_data['status']){"transfer_data -> #{transfer_data}"}
126
116
  end
127
117
  sleep(1.0)
128
118
  end
@@ -1,57 +1,39 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/environment'
3
4
  require 'aspera/agent/base'
4
- require 'aspera/ascp/installation'
5
+ require 'aspera/products/transferd'
5
6
  require 'aspera/temp_file_manager'
6
- require 'aspera/log'
7
- require 'aspera/assert'
8
7
  require 'json'
9
8
  require 'uri'
10
- require 'transfer_services_pb'
9
+ require 'transferd_services_pb'
11
10
 
12
11
  module Aspera
13
12
  module Agent
14
- class Trsdk < Base
15
- # see https://github.com/grpc/grpc/blob/master/doc/naming.md
13
+ class Transferd < Base
14
+ # https://github.com/grpc/grpc/blob/master/doc/naming.md
16
15
  # https://grpc.io/docs/guides/custom-name-resolution/
17
16
  LOCAL_SOCKET_ADDR = '127.0.0.1'
18
17
  PORT_SEP = ':'
19
18
  # port zero means select a random available high port
20
19
  AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
21
- class << self
22
- # Well, the port number is only in log file
23
- def daemon_port_from_log(log_file)
24
- result = nil
25
- # if port is zero, a dynamic port was created, get it
26
- File.open(log_file, 'r') do |file|
27
- file.each_line do |line|
28
- # Well, it's tricky to depend on log
29
- if (m = line.match(/Info: API Server: Listening on ([^:]+):(\d+) /))
30
- result = m[2].to_i
31
- # no "break" , need to read last matching log line
32
- end
33
- end
34
- end
35
- raise 'Port not found in daemon logs' if result.nil?
36
- Log.log.debug{"Got port #{result} from log"}
37
- return result
38
- end
39
- end
40
20
 
41
- # @param url [String] URL of the transfer manager daemon
42
- # @param external [Boolean] if true, expect that an external daemon is already running
43
- # @param keep [Boolean] if true, do not shutdown daemon on exit
44
- # @param base_options [Hash] base options
21
+ private_constant :LOCAL_SOCKET_ADDR, :PORT_SEP, :AUTO_LOCAL_TCP_PORT
22
+
23
+ # @param url [String] URL of the transfer manager daemon
24
+ # @param start [Bool] if false, expect that an external daemon is already running
25
+ # @param stop [Bool] if false, do not shutdown daemon on exit
26
+ # @param base [Hash] base class options
45
27
  def initialize(
46
- url: AUTO_LOCAL_TCP_PORT,
47
- external: false,
48
- keep: false,
49
- **base_options
28
+ url: AUTO_LOCAL_TCP_PORT,
29
+ start: true,
30
+ stop: true,
31
+ **base
50
32
  )
51
- super(**base_options)
52
- @keep = keep
33
+ super(**base)
34
+ @stop = stop
53
35
  is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
54
- raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@keep || external)
36
+ raise 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
55
37
  # keep PID for optional shutdown
56
38
  @daemon_pid = nil
57
39
  daemon_endpoint = url
@@ -61,18 +43,18 @@ module Aspera
61
43
  # no address: local bind
62
44
  daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{daemon_endpoint}" if daemon_endpoint.match?(/^#{PORT_SEP}[0-9]+$/o)
63
45
  # Create stub (without credentials)
64
- @transfer_client = Transfersdk::TransferService::Stub.new(daemon_endpoint, :this_channel_is_insecure)
46
+ @transfer_client = ::Transferd::Api::TransferService::Stub.new(daemon_endpoint, :this_channel_is_insecure)
65
47
  # Initiate actual connection
66
- get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
48
+ get_info_response = @transfer_client.get_info(::Transferd::Api::InstanceInfoRequest.new)
67
49
  Log.log.debug{"Daemon info: #{get_info_response}"}
68
- Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || external || @keep
50
+ Log.log.warn('Attached to existing daemon') unless @daemon_pid || !start || !@stop
69
51
  at_exit{shutdown}
70
52
  rescue GRPC::Unavailable => e
71
53
  # if transferd is external: do not start it, or other error
72
- raise if external || !e.message.include?('failed to connect')
54
+ raise if !start || !e.message.include?('failed to connect')
73
55
  # we already tried to start a daemon, but it failed
74
56
  Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
75
- Log.log.warn('no daemon present, starting daemon...') if external
57
+ Log.log.warn('no daemon present, starting daemon...') if !start
76
58
  # transferd only supports local ip and port
77
59
  daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
78
60
  Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
@@ -83,8 +65,8 @@ module Aspera
83
65
  fasp_runtime: {
84
66
  use_embedded: false,
85
67
  user_defined: {
86
- bin: Ascp::Installation.instance.sdk_folder,
87
- etc: Ascp::Installation.instance.sdk_folder
68
+ bin: Products::Transferd.sdk_directory,
69
+ etc: Products::Transferd.sdk_directory
88
70
  }
89
71
  }
90
72
  }
@@ -95,7 +77,11 @@ module Aspera
95
77
  log_stdout = "#{transferd_base_tmp}.out"
96
78
  log_stderr = "#{transferd_base_tmp}.err"
97
79
  File.write(conf_file, config.to_json)
98
- @daemon_pid = Process.spawn(Ascp::Installation.instance.path(:transferd), '--config', conf_file, out: log_stdout, err: log_stderr)
80
+ @daemon_pid = Environment.secure_spawn(
81
+ exec: Ascp::Installation.instance.path(:transferd),
82
+ args: ['--config', conf_file],
83
+ out: log_stdout,
84
+ err: log_stderr)
99
85
  begin
100
86
  # wait for process to initialize, max 2 seconds
101
87
  Timeout.timeout(2.0) do
@@ -107,10 +93,10 @@ module Aspera
107
93
  nil
108
94
  end
109
95
  Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
110
- Process.detach(@daemon_pid) if @keep
96
+ Process.detach(@daemon_pid) unless @stop
111
97
  at_exit {shutdown}
112
98
  # update port for next connection attempt (if auto high port was requested)
113
- daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{self.class.daemon_port_from_log(log_stdout)}" if is_local_auto_port
99
+ daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
114
100
  # local daemon started, try again
115
101
  retry
116
102
  end
@@ -118,9 +104,9 @@ module Aspera
118
104
 
119
105
  def start_transfer(transfer_spec, token_regenerator: nil)
120
106
  # create a transfer request
121
- transfer_request = Transfersdk::TransferRequest.new(
122
- transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
123
- config: Transfersdk::TransferConfig.new, # transfer configuration
107
+ transfer_request = ::Transferd::Api::TransferRequest.new(
108
+ transferType: ::Transferd::Api::TransferType::FILE_REGULAR, # transfer type (file/stream)
109
+ config: ::Transferd::Api::TransferConfig.new, # transfer configuration
124
110
  transferSpec: transfer_spec.to_json) # transfer definition
125
111
  # send start transfer request to the transfer manager daemon
126
112
  start_transfer_response = @transfer_client.start_transfer(transfer_request)
@@ -134,7 +120,7 @@ module Aspera
134
120
  session_started = false
135
121
  bytes_expected = nil
136
122
  # monitor transfer status
137
- @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
123
+ @transfer_client.monitor_transfers(::Transferd::Api::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
138
124
  Log.log.debug{Log.dump(:response, response.to_h)}
139
125
  # Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
140
126
  case response.status
@@ -167,7 +153,7 @@ module Aspera
167
153
  end
168
154
 
169
155
  def shutdown
170
- stop_daemon unless @keep
156
+ stop_daemon if @stop
171
157
  end
172
158
 
173
159
  def stop_daemon
@@ -7,7 +7,7 @@ module Aspera
7
7
  def initialize(entitlement_id, customer_id, api_domain: AoC::SAAS_DOMAIN_PROD, version: 'v1')
8
8
  super(
9
9
  base_url: "https://api.#{api_domain}/metering/#{version}",
10
- headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_token(entitlement_id, customer_id)}
10
+ headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_authorization(entitlement_id, customer_id)}
11
11
  )
12
12
  end
13
13
  end