aspera-cli 4.4.0 → 4.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1042 -787
  3. data/bin/ascli +1 -1
  4. data/bin/asession +3 -5
  5. data/docs/Makefile +4 -7
  6. data/docs/README.erb.md +988 -740
  7. data/examples/faspex4.rb +4 -6
  8. data/examples/transfer.rb +2 -2
  9. data/lib/aspera/aoc.rb +139 -118
  10. data/lib/aspera/cli/listener/progress_multi.rb +5 -5
  11. data/lib/aspera/cli/main.rb +64 -34
  12. data/lib/aspera/cli/manager.rb +19 -20
  13. data/lib/aspera/cli/plugin.rb +9 -1
  14. data/lib/aspera/cli/plugins/aoc.rb +156 -143
  15. data/lib/aspera/cli/plugins/ats.rb +11 -10
  16. data/lib/aspera/cli/plugins/bss.rb +2 -2
  17. data/lib/aspera/cli/plugins/config.rb +236 -112
  18. data/lib/aspera/cli/plugins/faspex.rb +29 -7
  19. data/lib/aspera/cli/plugins/faspex5.rb +21 -8
  20. data/lib/aspera/cli/plugins/node.rb +21 -9
  21. data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
  22. data/lib/aspera/cli/plugins/preview.rb +2 -2
  23. data/lib/aspera/cli/plugins/server.rb +3 -3
  24. data/lib/aspera/cli/plugins/shares.rb +17 -0
  25. data/lib/aspera/cli/transfer_agent.rb +47 -85
  26. data/lib/aspera/cli/version.rb +1 -1
  27. data/lib/aspera/environment.rb +4 -4
  28. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
  29. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
  30. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +14 -17
  31. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +4 -4
  32. data/lib/aspera/fasp/{node.rb → agent_node.rb} +25 -8
  33. data/lib/aspera/fasp/agent_trsdk.rb +106 -0
  34. data/lib/aspera/fasp/default.rb +17 -0
  35. data/lib/aspera/fasp/installation.rb +64 -48
  36. data/lib/aspera/fasp/parameters.rb +7 -3
  37. data/lib/aspera/faspex_gw.rb +6 -6
  38. data/lib/aspera/keychain/encrypted_hash.rb +120 -0
  39. data/lib/aspera/keychain/macos_security.rb +94 -0
  40. data/lib/aspera/log.rb +45 -32
  41. data/lib/aspera/node.rb +3 -6
  42. data/lib/aspera/rest.rb +65 -49
  43. metadata +68 -27
  44. data/lib/aspera/api_detector.rb +0 -60
  45. data/lib/aspera/secrets.rb +0 -20
@@ -1,4 +1,4 @@
1
- require 'aspera/fasp/manager'
1
+ require 'aspera/fasp/agent_base'
2
2
  require 'aspera/rest'
3
3
  require 'aspera/open_application'
4
4
  require 'securerandom'
@@ -6,19 +6,15 @@ require 'tty-spinner'
6
6
 
7
7
  module Aspera
8
8
  module Fasp
9
- class Connect < Manager
9
+ class AgentConnect < AgentBase
10
10
  MAX_CONNECT_START_RETRY=3
11
11
  SLEEP_SEC_BETWEEN_RETRY=2
12
12
  private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
13
- def initialize
14
- super
13
+ def initialize(options)
14
+ super()
15
15
  @connect_settings={
16
16
  'app_id' => SecureRandom.uuid
17
17
  }
18
- # TODO: start here and create monitor
19
- end
20
-
21
- def start_transfer(transfer_spec,options=nil)
22
18
  raise 'Using connect requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
23
19
  trynumber=0
24
20
  begin
@@ -26,6 +22,7 @@ module Aspera
26
22
  Log.log.debug("found: #{connect_url}")
27
23
  @connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
28
24
  cinfo=@connect_api.read('info/version')[:data]
25
+ Log.dump(:connect_version,cinfo)
29
26
  rescue => e # Errno::ECONNREFUSED
30
27
  raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
31
28
  Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
@@ -37,6 +34,9 @@ module Aspera
37
34
  sleep(SLEEP_SEC_BETWEEN_RETRY)
38
35
  retry
39
36
  end
37
+ end
38
+
39
+ def start_transfer(transfer_spec,options=nil)
40
40
  if transfer_spec['direction'] == 'send'
41
41
  Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
42
42
  transfer_spec.delete('paths')
@@ -47,7 +47,6 @@ module Aspera
47
47
  # if there is a token, we ask connect client to use well known ssh private keys
48
48
  # instead of asking password
49
49
  transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
50
- connect_settings=
51
50
  connect_transfer_args={
52
51
  'aspera_connect_settings'=>@connect_settings.merge({
53
52
  'request_id' =>@request_id,
@@ -65,42 +64,50 @@ module Aspera
65
64
  connect_activity_args={'aspera_connect_settings'=>@connect_settings}
66
65
  started=false
67
66
  spinner=nil
68
- loop do
69
- tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
70
- if tr_info['transfer_info'].is_a?(Hash)
71
- trdata=tr_info['transfer_info']
72
- if trdata.nil?
73
- Log.log.warn("no session in Connect")
74
- break
75
- end
76
- # TODO: get session id
77
- case trdata['status']
78
- when 'completed'
79
- notify_end(@connect_settings['app_id'])
80
- break
81
- when 'initiating','queued'
82
- if spinner.nil?
83
- spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
84
- spinner.start
67
+ begin
68
+ loop do
69
+ tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
70
+ if tr_info['transfer_info'].is_a?(Hash)
71
+ trdata=tr_info['transfer_info']
72
+ if trdata.nil?
73
+ Log.log.warn("no session in Connect")
74
+ break
85
75
  end
86
- spinner.update(title: trdata['status'])
87
- spinner.spin
88
- when 'running'
89
- #puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
90
- if !started and trdata['bytes_expected'] != 0
91
- notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
92
- started=true
76
+ # TODO: get session id
77
+ case trdata['status']
78
+ when 'completed'
79
+ notify_end(@connect_settings['app_id'])
80
+ break
81
+ when 'initiating','queued'
82
+ if spinner.nil?
83
+ spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
84
+ spinner.start
85
+ end
86
+ spinner.update(title: trdata['status'])
87
+ spinner.spin
88
+ when 'running'
89
+ #puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
90
+ if !started and trdata['bytes_expected'] != 0
91
+ spinner.success unless spinner.nil?
92
+ notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
93
+ started=true
94
+ else
95
+ notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
96
+ end
97
+ when 'failed'
98
+ spinner.error unless spinner.nil?
99
+ raise Fasp::Error.new(trdata['error_desc'])
93
100
  else
94
- notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
101
+ raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
95
102
  end
96
- else
97
- raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
98
103
  end
104
+ sleep 1
99
105
  end
100
- sleep 1
106
+ rescue => e
107
+ return [e]
101
108
  end
102
- return [] #TODO
109
+ return [:success]
103
110
  end # wait
104
- end # Connect
111
+ end # AgentConnect
105
112
  end
106
113
  end
@@ -5,11 +5,12 @@
5
5
  # Laurent Martin
6
6
  #
7
7
  ##############################################################################
8
- require 'aspera/fasp/manager'
8
+ require 'aspera/fasp/agent_base'
9
9
  require 'aspera/fasp/error'
10
10
  require 'aspera/fasp/parameters'
11
11
  require 'aspera/fasp/installation'
12
12
  require 'aspera/fasp/resume_policy'
13
+ require 'aspera/fasp/default'
13
14
  require 'aspera/log'
14
15
  require 'socket'
15
16
  require 'timeout'
@@ -18,19 +19,17 @@ require 'securerandom'
18
19
  module Aspera
19
20
  module Fasp
20
21
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
21
- class Local < Manager
22
+ class AgentDirect < AgentBase
22
23
  # options for initialize (same as values in option transfer_info)
23
24
  DEFAULT_OPTIONS = {
24
- :spawn_timeout_sec => 3,
25
- :spawn_delay_sec => 2,
26
- :wss => false,
27
- :multi_incr_udp => true,
28
- :resume => {}
25
+ spawn_timeout_sec: 3,
26
+ spawn_delay_sec: 2,
27
+ wss: false,
28
+ multi_incr_udp: true,
29
+ resume: {},
30
+ quiet: true # by default no interactive progress bar
29
31
  }
30
- DEFAULT_UDP_PORT=33001
31
32
  private_constant :DEFAULT_OPTIONS
32
- # set to false to keep ascp progress bar display ("true" adds ascp's option -q)
33
- attr_accessor :quiet
34
33
 
35
34
  # start ascp transfer (non blocking), single or multi-session
36
35
  # job information added to @jobs
@@ -76,13 +75,13 @@ module Aspera
76
75
  Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
77
76
  multi_session_info = nil
78
77
  elsif multi_session_info[:count].eql?(0)
79
- Log.log.debug("multi_session count is zero: no multisession")
78
+ Log.log.debug("multi_session count is zero: no multisession")
80
79
  multi_session_info = nil
81
80
  else # multi_session_info[:count] > 0
82
81
  # if option not true: keep default udp port for all sessions
83
82
  if @options[:multi_incr_udp]
84
83
  # override if specified, else use default value
85
- multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : DEFAULT_UDP_PORT
84
+ multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : Default::UDP_PORT
86
85
  # delete from original transfer spec, as we will increment values
87
86
  transfer_spec.delete('fasp_port')
88
87
  end
@@ -98,7 +97,7 @@ module Aspera
98
97
  env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
99
98
  end
100
99
 
101
- env_args[:args].unshift('-q') if @quiet
100
+ env_args[:args].unshift('-q') if @options[:quiet]
102
101
 
103
102
  # transfer job can be multi session
104
103
  xfer_job={
@@ -240,7 +239,7 @@ module Aspera
240
239
  when ''
241
240
  # empty line is separator to end event information
242
241
  raise 'unexpected empty line' if current_event_data.nil?
243
- current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
242
+ current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
244
243
  notify_listeners(current_event_text,current_event_data)
245
244
  case current_event_data['Type']
246
245
  when 'INIT'
@@ -333,8 +332,6 @@ module Aspera
333
332
  # @param options : keys(symbol): see DEFAULT_OPTIONS
334
333
  def initialize(options=nil)
335
334
  super()
336
- # by default no interactive progress bar
337
- @quiet=true
338
335
  # all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
339
336
  @jobs={}
340
337
  # mutex protects global data accessed by threads
@@ -374,6 +371,6 @@ module Aspera
374
371
  Log.log.debug("EXIT (#{Thread.current[:name]})")
375
372
  end
376
373
 
377
- end # Local
374
+ end # AgentDirect
378
375
  end
379
376
  end
@@ -1,5 +1,5 @@
1
1
  #!/bin/echo this is a ruby class:
2
- require 'aspera/fasp/manager'
2
+ require 'aspera/fasp/agent_base'
3
3
  require 'aspera/log'
4
4
  require 'aspera/rest'
5
5
  require 'websocket-client-simple'
@@ -12,7 +12,7 @@ require 'json'
12
12
  module Aspera
13
13
  module Fasp
14
14
  # start a transfer using Aspera HTTP Gateway, using web socket session
15
- class HttpGW < Manager
15
+ class AgentHttpgw < AgentBase
16
16
  # message returned by HTTP GW in case of success
17
17
  OK_MESSAGE='end upload'
18
18
  # refresh rate for progress
@@ -179,7 +179,7 @@ module Aspera
179
179
  when 'receive'
180
180
  download(transfer_spec)
181
181
  else
182
- raise "error"
182
+ raise "unexpected direction: [#{transfer_spec['direction']}]"
183
183
  end
184
184
  end # start_transfer
185
185
 
@@ -209,6 +209,6 @@ module Aspera
209
209
  @upload_chunksize=128000 # TODO: configurable ?
210
210
  end
211
211
 
212
- end # HttpGW
212
+ end # AgentHttpgw
213
213
  end
214
214
  end
@@ -1,4 +1,4 @@
1
- require 'aspera/fasp/manager'
1
+ require 'aspera/fasp/agent_base'
2
2
  require 'aspera/log'
3
3
  require 'tty-spinner'
4
4
 
@@ -6,13 +6,30 @@ module Aspera
6
6
  module Fasp
7
7
  # this singleton class is used by the CLI to provide a common interface to start a transfer
8
8
  # before using it, the use must set the `node_api` member.
9
- class Node < Manager
9
+ class AgentNode < AgentBase
10
10
  # option include: root_id if the node is an access key
11
11
  attr_writer :options
12
- def initialize(node_api,options={})
12
+ def initialize(options)
13
+ raise "node specification must be Hash" unless options.is_a?(Hash)
14
+ [:url,:username,:password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
13
15
  super()
14
- @node_api=node_api
15
- @options=options
16
+ # root id is required for access key
17
+ @root_id=options[:root_id]
18
+ rest_params={ base_url: options[:url]}
19
+ if options[:password].match(/^Bearer /)
20
+ rest_params[:headers]={
21
+ 'X-Aspera-AccessKey'=>options[:username],
22
+ 'Authorization' =>options[:password]
23
+ }
24
+ raise "root_id is required for access key" if @root_id.nil?
25
+ else
26
+ rest_params[:auth]={
27
+ type: :basic,
28
+ username: options[:username],
29
+ password: options[:password]
30
+ }
31
+ end
32
+ @node_api=Rest.new(rest_params)
16
33
  # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
17
34
  @transfer_id=nil
18
35
  end
@@ -36,10 +53,10 @@ module Aspera
36
53
  # generic method
37
54
  def start_transfer(transfer_spec,options=nil)
38
55
  # add root id if access key
39
- if @options.has_key?(:root_id)
56
+ if ! @root_id.nil?
40
57
  case transfer_spec['direction']
41
- when 'send';transfer_spec['source_root_id']=@options[:root_id]
42
- when 'receive';transfer_spec['destination_root_id']=@options[:root_id]
58
+ when 'send';transfer_spec['source_root_id']=@root_id
59
+ when 'receive';transfer_spec['destination_root_id']=@root_id
43
60
  else raise "unexpected direction in ts: #{transfer_spec['direction']}"
44
61
  end
45
62
  end
@@ -0,0 +1,106 @@
1
+ require 'aspera/fasp/agent_base'
2
+ require 'aspera/fasp/installation'
3
+ require 'json'
4
+
5
+ module Aspera
6
+ module Fasp
7
+ class AgentTrsdk < AgentBase
8
+ DEFAULT_OPTIONS = {
9
+ address: '127.0.0.1',
10
+ port: 55002,
11
+ }
12
+ private_constant :DEFAULT_OPTIONS
13
+
14
+ # options come from transfer_info
15
+ def initialize(user_opts)
16
+ raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? or user_opts.is_a?(Hash)
17
+ # set default options and override if specified
18
+ options=DEFAULT_OPTIONS.clone
19
+ if !user_opts.nil?
20
+ user_opts.each do |k,v|
21
+ if DEFAULT_OPTIONS.has_key?(k)
22
+ options[k]=v
23
+ else
24
+ raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
25
+ end
26
+ end
27
+ end
28
+ Log.log.debug("options= #{options}")
29
+ super()
30
+ # load and create SDK stub
31
+ $LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
32
+ require 'transfer_services_pb'
33
+ @transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}",:this_channel_is_insecure)
34
+ begin
35
+ get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
36
+ Log.log.debug("daemon info: #{get_info_response}")
37
+ rescue GRPC::Unavailable => e
38
+ Log.log.warn("no daemon present, starting daemon...")
39
+ # location of daemon binary
40
+ bin_folder=File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
41
+ # config file and logs are created in same folder
42
+ conf_file = File.join(bin_folder,'sdk.conf')
43
+ log_base = File.join(bin_folder,'transferd')
44
+ # create a config file for daemon
45
+ config = {
46
+ address: options[:address],
47
+ port: options[:port],
48
+ fasp_runtime: {
49
+ use_embedded: false,
50
+ user_defined: {
51
+ bin: bin_folder,
52
+ etc: bin_folder,
53
+ }
54
+ }
55
+ }
56
+ File.write(conf_file,config.to_json)
57
+ trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config' , conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
58
+ Process.detach(trd_pid)
59
+ sleep(2.0)
60
+ retry
61
+ end
62
+ end
63
+
64
+ def start_transfer(transfer_spec,options=nil)
65
+ # create a transfer request
66
+ transfer_request = Transfersdk::TransferRequest.new(
67
+ transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
68
+ config: Transfersdk::TransferConfig.new, # transfer configuration
69
+ transferSpec: transfer_spec.to_json) # transfer definition
70
+ # send start transfer request to the transfer manager daemon
71
+ start_transfer_response = @transfer_client.start_transfer(transfer_request)
72
+ Log.log.debug("start transfer response #{start_transfer_response}")
73
+ @transfer_id = start_transfer_response.transferId
74
+ Log.log.debug("transfer started with id #{@transfer_id}")
75
+ end
76
+
77
+ def wait_for_transfers_completion
78
+ started=false
79
+ # monitor transfer status
80
+ @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
81
+ Log.dump(:response, response.to_h)
82
+ #Log.log.debug("#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}")
83
+ case response.status
84
+ when :RUNNING
85
+ if !started and !response.sessionInfo.preTransferBytes.eql?(0)
86
+ notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
87
+ started=true
88
+ elsif started
89
+ notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
90
+ end
91
+ when :FAILED, :COMPLETED, :CANCELED
92
+ notify_end(@transfer_id)
93
+ raise Fasp::Error.new(JSON.parse(response.message)['Description']) unless :COMPLETED.eql?(response.status)
94
+ break
95
+ when :QUEUED,:UNKNOWN_STATUS,:PAUSED,:ORPHANED
96
+ # ignore
97
+ else
98
+ Log.log.error("unknown status#{response.status}")
99
+ end
100
+ end
101
+ # TODO return status
102
+ return []
103
+ end
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,17 @@
1
+ module Aspera
2
+ module Fasp
3
+ # default parameters
4
+ class Default
5
+ # (public) default transfer username for access key based transfers
6
+ ACCESS_KEY_TRANSFER_USER='xfer'
7
+ SSH_PORT=33001
8
+ UDP_PORT=33001
9
+ AK_TSPEC_BASE={
10
+ 'remote_user' => ACCESS_KEY_TRANSFER_USER,
11
+ 'ssh_port' => SSH_PORT,
12
+ 'fasp_port' => UDP_PORT
13
+ }
14
+ end
15
+ end
16
+ end
17
+
@@ -10,7 +10,7 @@ require 'fileutils'
10
10
  module Aspera
11
11
  module Fasp
12
12
  # Singleton that tells where to find ascp and other local resources (keys..) , using the "path(symb)" method.
13
- # It is used by object : Fasp::Local to find necessary resources
13
+ # It is used by object : AgentDirect to find necessary resources
14
14
  # By default it takes the first Aspera product found specified in product_locations
15
15
  # but the user can specify ascp location by calling:
16
16
  # Installation.instance.use_ascp_from_product(product_name)
@@ -22,6 +22,9 @@ module Aspera
22
22
  PRODUCT_CLI_V1='Aspera CLI'
23
23
  PRODUCT_DRIVE='Aspera Drive'
24
24
  PRODUCT_ENTSRV='Enterprise Server'
25
+ # protobuf generated files from sdk
26
+ EXT_RUBY_PROTOBUF='_pb.rb'
27
+ RB_SDK_FOLDER='lib'
25
28
  MAX_REDIRECT_SDK=2
26
29
  private_constant :MAX_REDIRECT_SDK
27
30
  # set ascp executable path
@@ -29,9 +32,10 @@ module Aspera
29
32
  @path_to_ascp=v
30
33
  end
31
34
 
32
- # filename for ascp with optional extension (Windows)
33
- def ascp_filename
34
- return 'ascp'+Environment.exe_extension
35
+ def sdk_ruby_folder
36
+ ruby_pb_folder=File.join(folder_path,RB_SDK_FOLDER)
37
+ FileUtils.mkdir_p(ruby_pb_folder) unless Dir.exist?(ruby_pb_folder)
38
+ return ruby_pb_folder
35
39
  end
36
40
 
37
41
  # location of SDK files
@@ -100,6 +104,8 @@ module Aspera
100
104
  file=@path_to_ascp
101
105
  # note that there might be a .exe at the end
102
106
  file=file.gsub('ascp','ascp4') if k.eql?(:ascp4)
107
+ when :transferd
108
+ file=transferd_filepath
103
109
  when :ssh_bypass_key_dsa
104
110
  file=File.join(folder_path,'aspera_bypass_dsa.pem')
105
111
  File.write(file,get_key('dsa',1)) unless File.exist?(file)
@@ -182,16 +188,22 @@ module Aspera
182
188
  return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
183
189
  end
184
190
 
191
+ # use in plugin `config`
192
+ def get_ascp_version(exe_path)
193
+ return get_exe_version(exe_path,'-A')
194
+ end
195
+
185
196
  # Check that specified path is ascp and get version
186
- def get_ascp_version(ascp_path)
187
- raise "File basename of #{ascp_path} must be #{ascp_filename}" unless File.basename(ascp_path).eql?(ascp_filename)
188
- ascp_version='n/a'
189
- raise "error in sdk: no ascp included" if ascp_path.nil?
190
- cmd_out=%x{"#{ascp_path}" -A}
197
+ def get_exe_version(exe_path,vers_arg)
198
+ raise "ERROR: nil arg" if exe_path.nil?
199
+ return nil unless File.exist?(exe_path)
200
+ exe_version=nil
201
+ cmd_out=%x{"#{exe_path}" #{vers_arg}}
191
202
  raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $? == 0
192
203
  # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
193
- m=cmd_out.match(/ascp version (.*)/)
194
- ascp_version=m[1] unless m.nil?
204
+ m=cmd_out.match(/ version ([0-9\.]+)/)
205
+ exe_version=m[1] unless m.nil?
206
+ return exe_version
195
207
  end
196
208
 
197
209
  # download aspera SDK or use local file
@@ -205,54 +217,49 @@ module Aspera
205
217
  raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
206
218
  sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
207
219
  else
208
- redirect_remain=MAX_REDIRECT_SDK
209
- begin
210
- Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
211
- rescue Aspera::RestCallError => e
212
- if e.response.is_a?(Net::HTTPRedirection)
213
- if redirect_remain > 0
214
- redirect_remain-=1
215
- sdk_url=e.response['location']
216
- retry
217
- else
218
- raise "Too many redirect"
219
- end
220
- else
221
- raise e
222
- end
223
- end
220
+ Aspera::Rest.new(base_url: sdk_url, redirect_max: 3).call(operation: 'GET',save_to_file: sdk_zip_path)
224
221
  end
225
- # SDK is organized by architecture
226
- filter="/#{Environment.architecture}/"
227
- ascp_path=nil
228
- sdk_path=folder_path
229
222
  # rename old install
230
- if File.exist?(File.join(sdk_path,ascp_filename))
231
- Log.log.warn("Previous install exists, renaming.")
232
- File.rename(sdk_path,"#{sdk_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
223
+ if ! Dir.empty?(folder_path)
224
+ Log.log.warn("Previous install exists, renaming folder.")
225
+ File.rename(folder_path,"#{folder_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
226
+ # TODO: delete old archives ?
233
227
  end
234
- # first ensure license file is here so that ascp invokation for version works
235
- self.path(:aspera_license)
236
- self.path(:aspera_conf)
228
+ # SDK is organized by architecture
229
+ arch_filter="#{Environment.architecture}/"
230
+ # extract files from archive
237
231
  Zip::File.open(sdk_zip_path) do |zip_file|
238
232
  zip_file.each do |entry|
239
- # get only specified arch, but not folder, only files
240
- if entry.name.include?(filter) and !entry.name.end_with?('/')
241
- archive_file=File.join(sdk_path,File.basename(entry.name))
242
- File.open(archive_file, 'wb') do |output_stream|
233
+ # skip folder entries
234
+ next if entry.name.end_with?('/')
235
+ dest_folder=nil
236
+ # binaries
237
+ dest_folder=folder_path if entry.name.include?(arch_filter)
238
+ # ruby adapters
239
+ dest_folder=sdk_ruby_folder if entry.name.end_with?(EXT_RUBY_PROTOBUF)
240
+ if !dest_folder.nil?
241
+ File.open(File.join(dest_folder,File.basename(entry.name)), 'wb') do |output_stream|
243
242
  IO.copy_stream(entry.get_input_stream, output_stream)
244
243
  end
245
- if File.basename(entry.name).eql?(ascp_filename)
246
- FileUtils.chmod(0755,archive_file)
247
- ascp_path=archive_file
248
- end
249
244
  end
250
245
  end
251
246
  end
252
247
  File.unlink(sdk_zip_path) rescue nil # Windows may give error
253
- ascp_version=get_ascp_version(ascp_path)
254
- File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
255
- return ascp_version
248
+ # ensure license file are generated so that ascp invokation for version works
249
+ self.path(:aspera_license)
250
+ self.path(:aspera_conf)
251
+ ascp_path=File.join(folder_path,ascp_filename)
252
+ raise "No #{ascp_filename} found in SDK archive" unless File.exist?(ascp_path)
253
+ FileUtils.chmod(0755,ascp_path)
254
+ FileUtils.chmod(0755,ascp_path.gsub('ascp','ascp4'))
255
+ ascp_version=get_ascp_version(File.join(folder_path,ascp_filename))
256
+ trd_path=transferd_filepath
257
+ Log.log.warn("No #{trd_path} in SDK archive") unless File.exist?(trd_path)
258
+ FileUtils.chmod(0755,trd_path) if File.exist?(trd_path)
259
+ transferd_version=get_exe_version(trd_path,'version')
260
+ sdk_version = transferd_version||ascp_version
261
+ File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{sdk_version}</version></product>")
262
+ return sdk_version
256
263
  end
257
264
 
258
265
  private
@@ -288,6 +295,15 @@ module Aspera
288
295
  @sdk_folder
289
296
  end
290
297
 
298
+ # filename for ascp with optional extension (Windows)
299
+ def ascp_filename
300
+ return 'ascp'+Environment.exe_extension
301
+ end
302
+
303
+ def transferd_filepath
304
+ return File.join(folder_path,'asperatransferd'+Environment.exe_extension)
305
+ end
306
+
291
307
  # @return product folders depending on OS fields
292
308
  # :expected M app name is taken from the manifest if present, else defaults to this value
293
309
  # :app_root M main folder for the application
@@ -7,6 +7,7 @@ require 'json'
7
7
  require 'yaml'
8
8
  require 'securerandom'
9
9
  require 'fileutils'
10
+ require 'openssl'
10
11
 
11
12
  module Aspera
12
13
  module Fasp
@@ -26,7 +27,7 @@ module Aspera
26
27
  Aspera::CommandLineBuilder.normalize_description(@@param_description_cache)
27
28
  end
28
29
 
29
- # Agents shown in manual
30
+ # Agents shown in manual for parameters (sub list)
30
31
  SUPPORTED_AGENTS=[:direct,:node,:connect]
31
32
  # Short names of columns in manual
32
33
  SUPPORTED_AGENTS_SHORT=SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
@@ -111,6 +112,9 @@ module Aspera
111
112
  @job_spec.delete('fasp_port')
112
113
  @job_spec.delete('EX_ssh_key_paths')
113
114
  @job_spec.delete('sshfp')
115
+ # set default location for CA bundle, see env var SSL_CERT_FILE / SSL_CERT_DIR
116
+ @job_spec['EX_ssh_key_paths']=[OpenSSL::X509::DEFAULT_CERT_FILE]
117
+ Log.log.debug("CA certs: EX_ssh_key_paths <- DEFAULT_CERT_FILE from openssl")
114
118
  else
115
119
  # remove unused parameter (avoid warning)
116
120
  @job_spec.delete('wss_port')
@@ -163,8 +167,8 @@ module Aspera
163
167
  lines=paths_array.map{|i|i['source']}
164
168
  end
165
169
  file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(@@file_list_folder)
166
- File.open(file_list_file, 'w+'){|f|f.puts(lines)}
167
- Log.log.debug("#{option}=\n#{File.read(file_list_file)}".red)
170
+ File.open(file_list_file, 'w+'){|f|f.write(lines.join("\n"))}
171
+ Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
168
172
  end
169
173
  end
170
174
  @builder.add_command_line_options(["#{option}=#{file_list_file}"])