aspera-cli 4.7.0 → 4.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +1 -0
  3. data/README.md +844 -861
  4. data/bin/ascli +20 -1
  5. data/bin/asession +37 -34
  6. data/docs/test_env.conf +11 -3
  7. data/examples/aoc.rb +13 -12
  8. data/examples/dascli +26 -0
  9. data/examples/faspex4.rb +34 -29
  10. data/examples/transfer.rb +30 -29
  11. data/lib/aspera/aoc.rb +151 -143
  12. data/lib/aspera/ascmd.rb +56 -45
  13. data/lib/aspera/ats_api.rb +6 -5
  14. data/lib/aspera/cli/basic_auth_plugin.rb +18 -16
  15. data/lib/aspera/cli/extended_value.rb +32 -30
  16. data/lib/aspera/cli/formater.rb +103 -111
  17. data/lib/aspera/cli/info.rb +2 -1
  18. data/lib/aspera/cli/listener/line_dump.rb +1 -0
  19. data/lib/aspera/cli/listener/logger.rb +1 -0
  20. data/lib/aspera/cli/listener/progress.rb +13 -12
  21. data/lib/aspera/cli/listener/progress_multi.rb +21 -20
  22. data/lib/aspera/cli/main.rb +106 -89
  23. data/lib/aspera/cli/manager.rb +96 -85
  24. data/lib/aspera/cli/plugin.rb +50 -32
  25. data/lib/aspera/cli/plugins/alee.rb +6 -5
  26. data/lib/aspera/cli/plugins/aoc.rb +521 -426
  27. data/lib/aspera/cli/plugins/ats.rb +84 -83
  28. data/lib/aspera/cli/plugins/bss.rb +30 -27
  29. data/lib/aspera/cli/plugins/config.rb +483 -397
  30. data/lib/aspera/cli/plugins/console.rb +17 -15
  31. data/lib/aspera/cli/plugins/cos.rb +26 -35
  32. data/lib/aspera/cli/plugins/faspex.rb +201 -168
  33. data/lib/aspera/cli/plugins/faspex5.rb +109 -74
  34. data/lib/aspera/cli/plugins/node.rb +378 -189
  35. data/lib/aspera/cli/plugins/orchestrator.rb +71 -65
  36. data/lib/aspera/cli/plugins/preview.rb +131 -122
  37. data/lib/aspera/cli/plugins/server.rb +94 -93
  38. data/lib/aspera/cli/plugins/shares.rb +42 -28
  39. data/lib/aspera/cli/plugins/sync.rb +15 -14
  40. data/lib/aspera/cli/transfer_agent.rb +56 -52
  41. data/lib/aspera/cli/version.rb +2 -1
  42. data/lib/aspera/colors.rb +29 -28
  43. data/lib/aspera/command_line_builder.rb +50 -43
  44. data/lib/aspera/cos_node.rb +64 -38
  45. data/lib/aspera/data_repository.rb +1 -0
  46. data/lib/aspera/environment.rb +18 -8
  47. data/lib/aspera/fasp/agent_base.rb +26 -23
  48. data/lib/aspera/fasp/agent_connect.rb +35 -30
  49. data/lib/aspera/fasp/agent_direct.rb +68 -60
  50. data/lib/aspera/fasp/agent_httpgw.rb +71 -64
  51. data/lib/aspera/fasp/agent_node.rb +24 -23
  52. data/lib/aspera/fasp/agent_trsdk.rb +19 -20
  53. data/lib/aspera/fasp/error.rb +2 -1
  54. data/lib/aspera/fasp/error_info.rb +79 -68
  55. data/lib/aspera/fasp/installation.rb +122 -114
  56. data/lib/aspera/fasp/listener.rb +1 -0
  57. data/lib/aspera/fasp/parameters.rb +44 -41
  58. data/lib/aspera/fasp/resume_policy.rb +14 -11
  59. data/lib/aspera/fasp/transfer_spec.rb +6 -5
  60. data/lib/aspera/fasp/uri.rb +25 -24
  61. data/lib/aspera/faspex_gw.rb +83 -72
  62. data/lib/aspera/hash_ext.rb +10 -12
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +60 -45
  65. data/lib/aspera/keychain/macos_security.rb +26 -24
  66. data/lib/aspera/log.rb +34 -38
  67. data/lib/aspera/nagios.rb +14 -13
  68. data/lib/aspera/node.rb +19 -19
  69. data/lib/aspera/oauth.rb +121 -101
  70. data/lib/aspera/open_application.rb +6 -5
  71. data/lib/aspera/persistency_action_once.rb +9 -8
  72. data/lib/aspera/persistency_folder.rb +10 -9
  73. data/lib/aspera/preview/file_types.rb +261 -266
  74. data/lib/aspera/preview/generator.rb +74 -73
  75. data/lib/aspera/preview/image_error.png +0 -0
  76. data/lib/aspera/preview/options.rb +7 -6
  77. data/lib/aspera/preview/utils.rb +30 -33
  78. data/lib/aspera/preview/video_error.png +0 -0
  79. data/lib/aspera/proxy_auto_config.rb +25 -23
  80. data/lib/aspera/rest.rb +73 -74
  81. data/lib/aspera/rest_call_error.rb +1 -0
  82. data/lib/aspera/rest_error_analyzer.rb +11 -9
  83. data/lib/aspera/rest_errors_aspera.rb +5 -4
  84. data/lib/aspera/secret_hider.rb +68 -0
  85. data/lib/aspera/ssh.rb +12 -10
  86. data/lib/aspera/sync.rb +49 -47
  87. data/lib/aspera/temp_file_manager.rb +7 -5
  88. data/lib/aspera/timer_limiter.rb +9 -8
  89. data/lib/aspera/uri_reader.rb +11 -14
  90. data/lib/aspera/web_auth.rb +17 -15
  91. data.tar.gz.sig +0 -0
  92. metadata +117 -34
  93. metadata.gz.sig +2 -0
  94. data/bin/dascli +0 -13
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'aspera/log'
3
4
  require 'rbconfig'
4
5
 
@@ -9,14 +10,14 @@ module Aspera
9
10
  OS_X = :osx
10
11
  OS_LINUX = :linux
11
12
  OS_AIX = :aix
12
- OS_LIST=[OS_WINDOWS,OS_X,OS_LINUX,OS_AIX].freeze
13
- CPU_X86_64=:x86_64 # rubocop:disable Naming/VariableNumber
14
- CPU_PPC64=:ppc64
15
- CPU_PPC64LE=:ppc64le
16
- CPU_S390=:s390
17
- CPU_LIST=[CPU_X86_64,CPU_PPC64,CPU_PPC64LE,CPU_S390].freeze
13
+ OS_LIST = [OS_WINDOWS,OS_X,OS_LINUX,OS_AIX].freeze
14
+ CPU_X86_64 = :x86_64
15
+ CPU_PPC64 = :ppc64
16
+ CPU_PPC64LE = :ppc64le
17
+ CPU_S390 = :s390
18
+ CPU_LIST = [CPU_X86_64,CPU_PPC64,CPU_PPC64LE,CPU_S390].freeze
18
19
 
19
- class<<self
20
+ class << self
20
21
  def ruby_version
21
22
  return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
22
23
  end
@@ -63,9 +64,18 @@ module Aspera
63
64
  # so, tell Ruby the right way
64
65
  def fix_home
65
66
  return unless os.eql?(OS_WINDOWS) && ENV.has_key?('USERPROFILE') && Dir.exist?(ENV['USERPROFILE'])
66
- ENV['HOME']=ENV['USERPROFILE']
67
+ ENV['HOME'] = ENV['USERPROFILE']
67
68
  Log.log.debug("Windows: set home to USERPROFILE: #{ENV['HOME']}")
68
69
  end
70
+
71
+ def empty_binding
72
+ return Kernel.binding
73
+ end
74
+
75
+ # secure execution of Ruby code
76
+ def secure_eval(code)
77
+ Kernel.send('lave'.reverse,code,empty_binding, __FILE__, __LINE__)
78
+ end
69
79
  end
70
80
  end
71
81
  end
@@ -1,15 +1,18 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module Aspera
3
4
  module Fasp
4
5
  # Base class for FASP transfer agents
5
6
  # sub classes shall implement start_transfer and shutdown
6
7
  class AgentBase
7
8
  # fields description for JSON generation
8
- INTEGER_FIELDS=%w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy DatagramSize XoptFlags VLinkVersion
9
- PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec ArgScansAttempted ArgScansCompleted PathScansAttempted FileScansCompleted TransfersAttempted TransfersPassed Delay].freeze
10
- BOOLEAN_FIELDS=%w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled MoveRange Keepalive TestLogin UseProxy
11
- Precalc RTTAutocorrect].freeze
12
- EXPECTED_METHODS=%i[text struct enhanced].freeze
9
+ INTEGER_FIELDS = %w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy
10
+ DatagramSize XoptFlags VLinkVersion PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize
11
+ ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec ArgScansAttempted
12
+ ArgScansCompleted PathScansAttempted FileScansCompleted TransfersAttempted TransfersPassed Delay].freeze
13
+ BOOLEAN_FIELDS = %w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled
14
+ MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect].freeze
15
+ EXPECTED_METHODS = %i[text struct enhanced].freeze
13
16
  private_constant :INTEGER_FIELDS,:BOOLEAN_FIELDS,:EXPECTED_METHODS
14
17
 
15
18
  private
@@ -18,55 +21,55 @@ module Aspera
18
21
  def enhanced_event_format(event)
19
22
  return event.keys.each_with_object({}) do |e,h|
20
23
  # capital_to_snake_case
21
- new_name=e.
22
- gsub(/([a-z\d])([A-Z])/,'\1_\2').
23
- gsub(/([a-z\d])(usec)$/,'\1_\2').
24
- gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
25
- downcase
26
- value=event[e]
27
- value=value.to_i if INTEGER_FIELDS.include?(e)
28
- value=value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
29
- h[new_name]=value
24
+ new_name = e.
25
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
26
+ gsub(/([a-z\d])(usec)$/,'\1_\2').
27
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
28
+ downcase
29
+ value = event[e]
30
+ value = value.to_i if INTEGER_FIELDS.include?(e)
31
+ value = value.eql?('Yes') if BOOLEAN_FIELDS.include?(e)
32
+ h[new_name] = value
30
33
  end
31
34
  end
32
35
 
33
36
  def initialize
34
- @listeners=[]
37
+ @listeners = []
35
38
  end
36
39
 
37
40
  def notify_listeners(current_event_text,current_event_data)
38
41
  Log.log.debug('send event to listeners')
39
- enhanced_event=nil
42
+ enhanced_event = nil
40
43
  @listeners.each do |listener|
41
44
  listener.event_text(current_event_text) if listener.respond_to?(:event_text)
42
45
  listener.event_struct(current_event_data) if listener.respond_to?(:event_struct)
43
46
  if listener.respond_to?(:event_enhanced)
44
- enhanced_event=enhanced_event_format(current_event_data) if enhanced_event.nil?
47
+ enhanced_event = enhanced_event_format(current_event_data) if enhanced_event.nil?
45
48
  listener.event_enhanced(enhanced_event)
46
49
  end
47
50
  end
48
51
  end # notify_listeners
49
52
 
50
53
  def notify_begin(id,size)
51
- notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'NOTIFICATION','PreTransferBytes'=>size})
54
+ notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'NOTIFICATION','PreTransferBytes' => size})
52
55
  end
53
56
 
54
57
  def notify_progress(id,size)
55
- notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'STATS','Bytescont'=>size})
58
+ notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'STATS','Bytescont' => size})
56
59
  end
57
60
 
58
61
  def notify_end(id)
59
- notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'DONE'})
62
+ notify_listeners('emulated',{LISTENER_SESSION_ID_B => id,'Type' => 'DONE'})
60
63
  end
61
64
 
62
65
  public
63
66
 
64
- LISTENER_SESSION_ID_B='ListenerSessionId'
65
- LISTENER_SESSION_ID_S='listener_session_id'
67
+ LISTENER_SESSION_ID_B = 'ListenerSessionId'
68
+ LISTENER_SESSION_ID_S = 'listener_session_id'
66
69
 
67
70
  # listener receives events
68
71
  def add_listener(listener)
69
- raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m,e|m+=listener.respond_to?("event_#{e}")?1:0;m}.eql?(0)
72
+ raise "expect one of #{EXPECTED_METHODS}" if EXPECTED_METHODS.inject(0){|m,e|m += listener.respond_to?("event_#{e}") ? 1 : 0;m}.eql?(0)
70
73
  @listeners.push(listener)
71
74
  self
72
75
  end
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'aspera/fasp/agent_base'
3
4
  require 'aspera/rest'
4
5
  require 'aspera/open_application'
@@ -8,26 +9,26 @@ require 'tty-spinner'
8
9
  module Aspera
9
10
  module Fasp
10
11
  class AgentConnect < AgentBase
11
- MAX_CONNECT_START_RETRY=3
12
- SLEEP_SEC_BETWEEN_RETRY=2
12
+ MAX_CONNECT_START_RETRY = 3
13
+ SLEEP_SEC_BETWEEN_RETRY = 2
13
14
  private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
14
15
  def initialize(_options)
15
16
  super()
16
- @connect_settings={
17
+ @connect_settings = {
17
18
  'app_id' => SecureRandom.uuid
18
19
  }
19
20
  raise 'Using connect requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
20
- trynumber=0
21
+ trynumber = 0
21
22
  begin
22
- connect_url=Installation.instance.connect_uri
23
+ connect_url = Installation.instance.connect_uri
23
24
  Log.log.debug("found: #{connect_url}")
24
- @connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
25
- cinfo=@connect_api.read('info/version')[:data]
25
+ @connect_api = Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin' => Rest.user_agent}}) # could use v6 also now
26
+ cinfo = @connect_api.read('info/version')[:data]
26
27
  Log.dump(:connect_version,cinfo)
27
28
  rescue StandardError => e # Errno::ECONNREFUSED
28
29
  raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
29
30
  Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
30
- trynumber+=1
31
+ trynumber += 1
31
32
  if !OpenApplication.uri_graphical('fasp://initialize')
32
33
  OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
33
34
  raise StandardError,'Connect is not installed'
@@ -41,36 +42,40 @@ module Aspera
41
42
  if transfer_spec['direction'] == 'send'
42
43
  Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
43
44
  transfer_spec.delete('paths')
44
- resdata=@connect_api.create('windows/select-open-file-dialog/',
45
- {'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
46
- transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
45
+ resdata = @connect_api.create('windows/select-open-file-dialog/',{
46
+ 'aspera_connect_settings' => @connect_settings,
47
+ 'title' => 'Select Files',
48
+ 'suggestedName' => '',
49
+ 'allowMultipleSelection' => true,
50
+ 'allowedFileTypes' => ''})[:data]
51
+ transfer_spec['paths'] = resdata['dataTransfer']['files'].map { |i| {'source' => i['name']}}
47
52
  end
48
- @request_id=SecureRandom.uuid
53
+ @request_id = SecureRandom.uuid
49
54
  # if there is a token, we ask connect client to use well known ssh private keys
50
55
  # instead of asking password
51
- transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
52
- connect_transfer_args={
53
- 'aspera_connect_settings'=>@connect_settings.merge({
54
- 'request_id' =>@request_id,
55
- 'allow_dialogs' =>true
56
+ transfer_spec['authentication'] = 'token' if transfer_spec.has_key?('token')
57
+ connect_transfer_args = {
58
+ 'aspera_connect_settings' => @connect_settings.merge({
59
+ 'request_id' => @request_id,
60
+ 'allow_dialogs' => true
56
61
  }),
57
- 'transfer_specs' =>[{
58
- 'transfer_spec' =>transfer_spec
62
+ 'transfer_specs' => [{
63
+ 'transfer_spec' => transfer_spec
59
64
  }]}
60
65
  # asynchronous anyway
61
- res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
62
- @xfer_id=res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
66
+ res = @connect_api.create('transfers/start',connect_transfer_args)[:data]
67
+ @xfer_id = res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
63
68
  end
64
69
 
65
70
  def wait_for_transfers_completion
66
- connect_activity_args={'aspera_connect_settings'=>@connect_settings}
67
- started=false
68
- spinner=nil
71
+ connect_activity_args = {'aspera_connect_settings' => @connect_settings}
72
+ started = false
73
+ spinner = nil
69
74
  begin
70
75
  loop do
71
- tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
76
+ tr_info = @connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
72
77
  if tr_info['transfer_info'].is_a?(Hash)
73
- trdata=tr_info['transfer_info']
78
+ trdata = tr_info['transfer_info']
74
79
  if trdata.nil?
75
80
  Log.log.warn('no session in Connect')
76
81
  break
@@ -90,20 +95,20 @@ module Aspera
90
95
  when 'running'
91
96
  #puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
92
97
  if !started && (trdata['bytes_expected'] != 0)
93
- spinner.success unless spinner.nil?
98
+ spinner&.success
94
99
  notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
95
- started=true
100
+ started = true
96
101
  else
97
102
  notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
98
103
  end
99
104
  when 'failed'
100
- spinner.error unless spinner.nil?
105
+ spinner&.error
101
106
  raise Fasp::Error, trdata['error_desc']
102
107
  else
103
108
  raise Fasp::Error, "unknown status: #{trdata['status']}: #{trdata['error_desc']}"
104
109
  end
105
110
  end
106
- sleep 1
111
+ sleep(1)
107
112
  end
108
113
  rescue StandardError => e
109
114
  return [e]
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'English'
3
4
  require 'aspera/fasp/agent_base'
4
5
  require 'aspera/fasp/error'
@@ -24,7 +25,7 @@ module Aspera
24
25
  multi_incr_udp: true,
25
26
  resume: {},
26
27
  quiet: true # by default no interactive progress bar
27
- }
28
+ }.freeze
28
29
  private_constant :DEFAULT_OPTIONS
29
30
 
30
31
  # start ascp transfer (non blocking), single or multi-session
@@ -37,17 +38,17 @@ module Aspera
37
38
  job_options[:resumer] ||= @resume_policy
38
39
  job_options[:job_id] ||= SecureRandom.uuid
39
40
  # clone transfer spec because we modify it (first level keys)
40
- transfer_spec=transfer_spec.clone
41
+ transfer_spec = transfer_spec.clone
41
42
  # if there is aspera tags
42
43
  if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
43
44
  # TODO: what is this for ? only on local ascp ?
44
45
  # NOTE: important: transfer id must be unique: generate random id
45
46
  # using a non unique id results in discard of tags in AoC, and a package is never finalized
46
47
  # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
47
- transfer_spec['tags']['aspera']['xfer_id']||=SecureRandom.uuid
48
+ transfer_spec['tags']['aspera']['xfer_id'] ||= SecureRandom.uuid
48
49
  Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
49
50
  # TODO: useful ? node only ?
50
- transfer_spec['tags']['aspera']['xfer_retry']||=3600
51
+ transfer_spec['tags']['aspera']['xfer_retry'] ||= 3600
51
52
  end
52
53
  Log.dump('ts',transfer_spec)
53
54
 
@@ -61,9 +62,9 @@ module Aspera
61
62
 
62
63
  # Compute this before using transfer spec because it potentially modifies the transfer spec
63
64
  # (even if the var is not used in single session)
64
- multi_session_info=nil
65
+ multi_session_info = nil
65
66
  if transfer_spec.has_key?('multi_session')
66
- multi_session_info={
67
+ multi_session_info = {
67
68
  count: transfer_spec['multi_session'].to_i
68
69
  }
69
70
  # Managed by multi-session, so delete from transfer spec
@@ -76,7 +77,7 @@ module Aspera
76
77
  multi_session_info = nil
77
78
  elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
78
79
  # if option not true: keep default udp port for all sessions
79
- multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
80
+ multi_session_info[:udp_base] = transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
80
81
  # delete from original transfer spec, as we will increment values
81
82
  transfer_spec.delete('fasp_port')
82
83
  # override if specified, else use default value
@@ -84,10 +85,10 @@ module Aspera
84
85
  end
85
86
 
86
87
  # compute known args
87
- env_args=Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
88
+ env_args = Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
88
89
 
89
90
  # add fallback cert and key as arguments if needed
90
- if ['1','force'].include?(transfer_spec['http_fallback'])
91
+ if %w[1 force].include?(transfer_spec['http_fallback'])
91
92
  env_args[:args].unshift('-Y',Installation.instance.path(:fallback_key))
92
93
  env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
93
94
  end
@@ -95,19 +96,19 @@ module Aspera
95
96
  env_args[:args].unshift('-q') if @options[:quiet]
96
97
 
97
98
  # transfer job can be multi session
98
- xfer_job={
99
- id: job_options[:job_id],
100
- sessions: [] # all sessions as below
99
+ xfer_job = {
100
+ id: job_options[:job_id],
101
+ sessions: [] # all sessions as below
101
102
  }
102
103
 
103
104
  # generic session information
104
- session={
105
- thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
106
- error: nil, # exception if failed
107
- io: nil, # management port server socket
108
- id: nil, # SessionId from INIT message in mgt port
109
- env_args: env_args, # env vars and args to ascp (from transfer spec)
110
- options: job_options # [Hash]
105
+ session = {
106
+ thread: nil, # Thread object monitoring management port, not nil when pushed to :sessions
107
+ error: nil, # exception if failed
108
+ io: nil, # management port server socket
109
+ id: nil, # SessionId from INIT message in mgt port
110
+ env_args: env_args, # env vars and args to ascp (from transfer spec)
111
+ options: job_options # [Hash]
111
112
  }
112
113
 
113
114
  if multi_session_info.nil?
@@ -121,12 +122,12 @@ module Aspera
121
122
  # do not delay the first session
122
123
  sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
123
124
  # do deep copy (each thread has its own copy because it is modified here below and in thread)
124
- this_session=session.clone()
125
- this_session[:env_args]=this_session[:env_args].clone()
126
- this_session[:env_args][:args]=this_session[:env_args][:args].clone()
125
+ this_session = session.clone
126
+ this_session[:env_args] = this_session[:env_args].clone
127
+ this_session[:env_args][:args] = this_session[:env_args][:args].clone
127
128
  this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
128
129
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
129
- this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base]+i-1).to_s) if @options[:multi_incr_udp]
130
+ this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base] + i - 1).to_s) if @options[:multi_incr_udp]
130
131
  this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
131
132
  xfer_job[:sessions].push(this_session)
132
133
  end
@@ -134,7 +135,7 @@ module Aspera
134
135
  Log.log.debug('started session thread(s)')
135
136
 
136
137
  # add job to list of jobs
137
- @jobs[job_options[:job_id]]=xfer_job
138
+ @jobs[job_options[:job_id]] = xfer_job
138
139
  Log.log.debug("jobs: #{@jobs.keys.count}")
139
140
 
140
141
  return job_options[:job_id]
@@ -145,7 +146,7 @@ module Aspera
145
146
  def wait_for_transfers_completion
146
147
  Log.log.debug('wait_for_transfers_completion')
147
148
  # set to non-nil to exit loop
148
- result=[]
149
+ result = []
149
150
  @jobs.each do |_id,job|
150
151
  job[:sessions].each do |session|
151
152
  Log.log.debug("join #{session[:thread]}")
@@ -176,11 +177,11 @@ module Aspera
176
177
  raise 'env_args must be Hash' unless env_args.is_a?(Hash)
177
178
  raise 'session must be Hash' unless session.is_a?(Hash)
178
179
  # by default we assume an exception will be raised (for ensure block)
179
- exception_raised=true
180
+ exception_raised = true
180
181
  begin
181
182
  Log.log.debug("env_args=#{env_args.inspect}")
182
183
  # get location of ascp executable
183
- ascp_path=@mutex.synchronize do
184
+ ascp_path = @mutex.synchronize do
184
185
  Fasp::Installation.instance.path(env_args[:ascp_version])
185
186
  end
186
187
  # (optional) check it exists
@@ -188,17 +189,24 @@ module Aspera
188
189
  # open random local TCP port for listening for ascp management
189
190
  mgt_sock = TCPServer.new('127.0.0.1',0)
190
191
  # clone arguments as we eed to modify with mgt port
191
- ascp_arguments=env_args[:args].clone
192
+ ascp_arguments = env_args[:args].clone
192
193
  # add management port
193
194
  ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
194
195
  # start ascp in sub process
195
- Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')} #{Shellwords.shellescape(ascp_path)} #{ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')}")
196
+ Log.log.debug do
197
+ 'execute: '+
198
+ env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')+
199
+ ' '+
200
+ Shellwords.shellescape(ascp_path)+
201
+ ' '+
202
+ ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')
203
+ end
196
204
  # start process
197
205
  ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
198
206
  # in parent, wait for connection to socket max 3 seconds
199
207
  Log.log.debug("before accept for pid (#{ascp_pid})")
200
208
  # init management socket
201
- ascp_mgt_io=nil
209
+ ascp_mgt_io = nil
202
210
  Timeout.timeout(@options[:spawn_timeout_sec]) do
203
211
  ascp_mgt_io = mgt_sock.accept
204
212
  # management messages include file names which may be utf8
@@ -207,20 +215,20 @@ module Aspera
207
215
  ascp_mgt_io.set_encoding(Encoding::UTF_8)
208
216
  end
209
217
  Log.log.debug("after accept (#{ascp_mgt_io})")
210
- session[:io]=ascp_mgt_io
218
+ session[:io] = ascp_mgt_io
211
219
  # exact text for event, with \n
212
- current_event_text=''
220
+ current_event_text = ''
213
221
  # parsed event (hash)
214
- current_event_data=nil
222
+ current_event_data = nil
215
223
  # this is the last full status
216
- last_status_event=nil
224
+ last_status_event = nil
217
225
  # read management port
218
226
  loop do
219
227
  # TODO: timeout here ?
220
228
  line = ascp_mgt_io.gets
221
229
  # nil when ascp process exits
222
230
  break if line.nil?
223
- current_event_text+=line
231
+ current_event_text += line
224
232
  line.chomp!
225
233
  Log.log.debug("line=[#{line}]")
226
234
  case line
@@ -234,11 +242,11 @@ module Aspera
234
242
  when ''
235
243
  # empty line is separator to end event information
236
244
  raise 'unexpected empty line' if current_event_data.nil?
237
- current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
245
+ current_event_data[AgentBase::LISTENER_SESSION_ID_B] = ascp_pid
238
246
  notify_listeners(current_event_text,current_event_data)
239
247
  case current_event_data['Type']
240
248
  when 'INIT'
241
- session[:id]=current_event_data['SessionId']
249
+ session[:id] = current_event_data['SessionId']
242
250
  Log.log.debug("session id: #{session[:id]}")
243
251
  when 'DONE','ERROR'
244
252
  # TODO: check if this is always the last event
@@ -253,15 +261,15 @@ module Aspera
253
261
  case last_status_event['Type']
254
262
  when 'DONE'
255
263
  # all went well
256
- exception_raised=false
264
+ exception_raised = false
257
265
  when 'ERROR'
258
266
  Log.log.error("code: #{last_status_event['Code']}")
259
- if last_status_event['Description'] =~ /bearer token/i
267
+ if /bearer token/i.match?(last_status_event['Description'])
260
268
  Log.log.error('need to regenerate token'.red)
261
269
  if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
262
270
  # regenerate token here, expired, or error on it
263
271
  # Note: in multi-session, each session will have a different one.
264
- env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
272
+ env_args[:env]['ASPERA_SCP_TOKEN'] = session[:options][:regenerate_token].call(true)
265
273
  end
266
274
  end
267
275
  raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
@@ -269,7 +277,7 @@ module Aspera
269
277
  raise "unexpected last event type: #{last_status_event['Type']}"
270
278
  end
271
279
  else
272
- exception_raised=false
280
+ exception_raised = false
273
281
  Log.log.debug('no status read from ascp mgt port')
274
282
  end
275
283
  rescue SystemCallError => e
@@ -284,11 +292,11 @@ module Aspera
284
292
  unless ascp_pid.nil?
285
293
  # "wait" for process to avoid zombie
286
294
  Process.wait(ascp_pid)
287
- status=$CHILD_STATUS
288
- ascp_pid=nil
295
+ status = $CHILD_STATUS
296
+ ascp_pid = nil
289
297
  session.delete(:io)
290
298
  if !status.success?
291
- message="ascp failed with code #{status.exitstatus}"
299
+ message = "ascp failed with code #{status.exitstatus}"
292
300
  # raise error only if there was not already an exception
293
301
  raise Fasp::Error, message unless exception_raised
294
302
  # else just debug, as main exception is already here
@@ -305,18 +313,18 @@ module Aspera
305
313
  # {'type'=>'START','source'=>_path_,'destination'=>_path_}
306
314
  # {'type'=>'DONE'}
307
315
  def send_command(job_id,session_index,data)
308
- job=@jobs[job_id]
316
+ job = @jobs[job_id]
309
317
  raise 'no such job' if job.nil?
310
- session=job[:sessions][session_index]
318
+ session = job[:sessions][session_index]
311
319
  raise 'no such session' if session.nil?
312
320
  Log.log.debug("command: #{data}")
313
321
  # build command
314
- command=data.
315
- keys.
316
- map{|k|"#{k.capitalize}: #{data[k]}"}.
317
- unshift('FASPMGR 2').
318
- push('','').
319
- join("\n")
322
+ command = data.
323
+ keys.
324
+ map{|k|"#{k.capitalize}: #{data[k]}"}.
325
+ unshift('FASPMGR 2').
326
+ push('','').
327
+ join("\n")
320
328
  session[:io].puts(command)
321
329
  end
322
330
 
@@ -326,20 +334,20 @@ module Aspera
326
334
  def initialize(options=nil)
327
335
  super()
328
336
  # all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
329
- @jobs={}
337
+ @jobs = {}
330
338
  # mutex protects global data accessed by threads
331
- @mutex=Mutex.new
339
+ @mutex = Mutex.new
332
340
  # set default options and override if specified
333
- @options=DEFAULT_OPTIONS.clone
341
+ @options = DEFAULT_OPTIONS.dup
334
342
  if !options.nil?
335
343
  raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
336
344
  options.each do |k,v|
337
345
  raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
338
- @options[k]=v
346
+ @options[k] = v
339
347
  end
340
348
  end
341
349
  Log.log.debug("local options= #{options}")
342
- @resume_policy=ResumePolicy.new(@options[:resume].symbolize_keys)
350
+ @resume_policy = ResumePolicy.new(@options[:resume].symbolize_keys)
343
351
  end
344
352
 
345
353
  # transfer thread entry
@@ -347,15 +355,15 @@ module Aspera
347
355
  def transfer_thread_entry(session)
348
356
  begin
349
357
  # set name for logging
350
- Thread.current[:name]='transfer'
358
+ Thread.current[:name] = 'transfer'
351
359
  Log.log.debug("ENTER (#{Thread.current[:name]})")
352
360
  # start transfer with selected resumer policy
353
- session[:options][:resumer].process do
361
+ session[:options][:resumer].execute_with_resume do
354
362
  start_transfer_with_args_env(session[:env_args],session)
355
363
  end
356
364
  Log.log.debug('transfer ok'.bg_green)
357
365
  rescue StandardError => e
358
- session[:error]=e
366
+ session[:error] = e
359
367
  Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
360
368
  end
361
369
  Log.log.debug("EXIT (#{Thread.current[:name]})")