aspera-cli 4.10.0 → 4.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +20 -0
  4. data/CHANGELOG.md +509 -0
  5. data/CONTRIBUTING.md +118 -0
  6. data/README.md +621 -378
  7. data/bin/ascli +4 -4
  8. data/bin/asession +11 -11
  9. data/docs/test_env.conf +28 -19
  10. data/examples/aoc.rb +4 -4
  11. data/examples/dascli +11 -9
  12. data/examples/faspex4.rb +8 -8
  13. data/examples/node.rb +11 -11
  14. data/examples/server.rb +9 -9
  15. data/lib/aspera/aoc.rb +273 -266
  16. data/lib/aspera/ascmd.rb +56 -54
  17. data/lib/aspera/ats_api.rb +4 -4
  18. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  19. data/lib/aspera/cli/extended_value.rb +5 -5
  20. data/lib/aspera/cli/formater.rb +64 -64
  21. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  22. data/lib/aspera/cli/listener/logger.rb +1 -1
  23. data/lib/aspera/cli/listener/progress.rb +5 -6
  24. data/lib/aspera/cli/listener/progress_multi.rb +14 -19
  25. data/lib/aspera/cli/main.rb +66 -67
  26. data/lib/aspera/cli/manager.rb +110 -110
  27. data/lib/aspera/cli/plugin.rb +54 -37
  28. data/lib/aspera/cli/plugins/alee.rb +4 -4
  29. data/lib/aspera/cli/plugins/aoc.rb +308 -669
  30. data/lib/aspera/cli/plugins/ats.rb +44 -46
  31. data/lib/aspera/cli/plugins/bss.rb +10 -10
  32. data/lib/aspera/cli/plugins/config.rb +447 -344
  33. data/lib/aspera/cli/plugins/console.rb +12 -12
  34. data/lib/aspera/cli/plugins/cos.rb +18 -20
  35. data/lib/aspera/cli/plugins/faspex.rb +110 -112
  36. data/lib/aspera/cli/plugins/faspex5.rb +67 -46
  37. data/lib/aspera/cli/plugins/node.rb +364 -288
  38. data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
  39. data/lib/aspera/cli/plugins/preview.rb +122 -114
  40. data/lib/aspera/cli/plugins/server.rb +137 -83
  41. data/lib/aspera/cli/plugins/shares.rb +30 -29
  42. data/lib/aspera/cli/plugins/sync.rb +13 -33
  43. data/lib/aspera/cli/transfer_agent.rb +57 -57
  44. data/lib/aspera/cli/version.rb +1 -1
  45. data/lib/aspera/colors.rb +3 -3
  46. data/lib/aspera/command_line_builder.rb +27 -27
  47. data/lib/aspera/cos_node.rb +22 -20
  48. data/lib/aspera/data_repository.rb +1 -1
  49. data/lib/aspera/environment.rb +30 -28
  50. data/lib/aspera/fasp/agent_base.rb +15 -15
  51. data/lib/aspera/fasp/agent_connect.rb +23 -21
  52. data/lib/aspera/fasp/agent_direct.rb +65 -67
  53. data/lib/aspera/fasp/agent_httpgw.rb +72 -68
  54. data/lib/aspera/fasp/agent_node.rb +23 -21
  55. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  56. data/lib/aspera/fasp/error.rb +3 -2
  57. data/lib/aspera/fasp/error_info.rb +11 -8
  58. data/lib/aspera/fasp/installation.rb +78 -78
  59. data/lib/aspera/fasp/listener.rb +1 -1
  60. data/lib/aspera/fasp/parameters.rb +75 -72
  61. data/lib/aspera/fasp/parameters.yaml +2 -2
  62. data/lib/aspera/fasp/resume_policy.rb +8 -8
  63. data/lib/aspera/fasp/transfer_spec.rb +35 -2
  64. data/lib/aspera/fasp/uri.rb +7 -7
  65. data/lib/aspera/faspex_gw.rb +7 -5
  66. data/lib/aspera/hash_ext.rb +3 -3
  67. data/lib/aspera/id_generator.rb +5 -5
  68. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  69. data/lib/aspera/keychain/macos_security.rb +21 -20
  70. data/lib/aspera/log.rb +7 -7
  71. data/lib/aspera/nagios.rb +19 -18
  72. data/lib/aspera/node.rb +209 -35
  73. data/lib/aspera/oauth.rb +37 -36
  74. data/lib/aspera/open_application.rb +19 -11
  75. data/lib/aspera/persistency_action_once.rb +4 -4
  76. data/lib/aspera/persistency_folder.rb +13 -13
  77. data/lib/aspera/preview/file_types.rb +8 -8
  78. data/lib/aspera/preview/generator.rb +67 -67
  79. data/lib/aspera/preview/utils.rb +27 -27
  80. data/lib/aspera/proxy_auto_config.js +41 -41
  81. data/lib/aspera/proxy_auto_config.rb +16 -16
  82. data/lib/aspera/rest.rb +56 -60
  83. data/lib/aspera/rest_call_error.rb +2 -1
  84. data/lib/aspera/rest_error_analyzer.rb +18 -17
  85. data/lib/aspera/rest_errors_aspera.rb +16 -16
  86. data/lib/aspera/secret_hider.rb +15 -13
  87. data/lib/aspera/ssh.rb +11 -10
  88. data/lib/aspera/sync.rb +158 -44
  89. data/lib/aspera/temp_file_manager.rb +2 -2
  90. data/lib/aspera/uri_reader.rb +4 -4
  91. data/lib/aspera/web_auth.rb +14 -13
  92. data.tar.gz.sig +0 -0
  93. metadata +8 -5
  94. metadata.gz.sig +0 -0
@@ -14,7 +14,7 @@ require 'json'
14
14
  module Aspera
15
15
  module Fasp
16
16
  # start a transfer using Aspera HTTP Gateway, using web socket session for uploads
17
- class AgentHttpgw < AgentBase
17
+ class AgentHttpgw < Aspera::Fasp::AgentBase
18
18
  # message returned by HTTP GW in case of success
19
19
  MSG_END_UPLOAD = 'end upload'
20
20
  MSG_END_SLICE = 'end_slice_upload'
@@ -23,15 +23,15 @@ module Aspera
23
23
  upload_chunksize: 64_000,
24
24
  upload_bar_refresh_sec: 0.5
25
25
  }.freeze
26
- DEFAULT_BASE_PATH='/aspera/http-gwy'
26
+ DEFAULT_BASE_PATH = '/aspera/http-gwy'
27
27
  # upload endpoints
28
- V1_UPLOAD='/v1/upload'
29
- V2_UPLOAD='/v2/upload'
30
- private_constant :DEFAULT_OPTIONS,:MSG_END_UPLOAD,:MSG_END_SLICE,:V1_UPLOAD,:V2_UPLOAD
28
+ V1_UPLOAD = '/v1/upload'
29
+ V2_UPLOAD = '/v2/upload'
30
+ private_constant :DEFAULT_OPTIONS, :MSG_END_UPLOAD, :MSG_END_SLICE, :V1_UPLOAD, :V2_UPLOAD
31
31
 
32
32
  # send message on http gw web socket
33
33
  def ws_snd_json(data)
34
- @slice_uploads += 1 if data.has_key?(:slice_upload)
34
+ @slice_uploads += 1 if data.key?(:slice_upload)
35
35
  Log.log.debug{JSON.generate(data)}
36
36
  ws_send(JSON.generate(data))
37
37
  end
@@ -47,7 +47,7 @@ module Aspera
47
47
  # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
48
48
  source_paths = []
49
49
  # get source root or nil
50
- source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
50
+ source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
51
51
  # source root is ignored by GW, used only here
52
52
  transfer_spec.delete('source_root')
53
53
  # compute total size of files to upload (for progress)
@@ -56,7 +56,7 @@ module Aspera
56
56
  # save actual file location to be able read contents later
57
57
  full_src_filepath = item['source']
58
58
  # add source root if needed
59
- full_src_filepath = File.join(source_root,full_src_filepath) unless source_root.nil?
59
+ full_src_filepath = File.join(source_root, full_src_filepath) unless source_root.nil?
60
60
  # GW expects a simple file name in 'source' but if user wants to change the name, we take it
61
61
  item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
62
62
  item['file_size'] = File.size(full_src_filepath)
@@ -66,18 +66,18 @@ module Aspera
66
66
  end
67
67
  # identify this session uniquely
68
68
  session_id = SecureRandom.uuid
69
- @slice_uploads=0
69
+ @slice_uploads = 0
70
70
  # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
71
71
  upload_api_version = V2_UPLOAD
72
72
  # is the latest supported? else revert to old api
73
- upload_api_version=V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
73
+ upload_api_version = V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
74
74
  Log.log.debug{"api version: #{upload_api_version}"}
75
- url=File.join(@gw_api.params[:base_url],upload_api_version)
76
- #uri = URI.parse(url)
75
+ url = File.join(@gw_api.params[:base_url], upload_api_version)
76
+ # uri = URI.parse(url)
77
77
  # open web socket to end point (equivalent to Net::HTTP.start)
78
78
  http_socket = Rest.start_http_session(url)
79
79
  @ws_io = http_socket.instance_variable_get(:@socket)
80
- #@ws_io.debug_output = Log.log
80
+ # @ws_io.debug_output = Log.log
81
81
  @ws_handshake = ::WebSocket::Handshake::Client.new(url: url, headers: {})
82
82
  @ws_io.write(@ws_handshake.to_s)
83
83
  sleep(0.1)
@@ -85,21 +85,21 @@ module Aspera
85
85
  raise 'Error in websocket handshake' unless @ws_handshake.finished?
86
86
  Log.log.debug('ws: handshake success')
87
87
  # data shared between main thread and read thread
88
- shared_info={
88
+ shared_info = {
89
89
  read_exception: nil, # error message if any in callback
90
90
  end_uploads: 0 # number of files totally sent
91
- #mutex: Mutex.new
92
- #cond_var: ConditionVariable.new
91
+ # mutex: Mutex.new
92
+ # cond_var: ConditionVariable.new
93
93
  }
94
94
  # start read thread
95
95
  ws_read_thread = Thread.new do
96
96
  Log.log.debug('ws: thread: started')
97
97
  frame = ::WebSocket::Frame::Incoming::Client.new
98
98
  loop do
99
- begin
99
+ begin # rubocop:disable Style/RedundantBegin
100
100
  frame << @ws_io.readuntil("\n")
101
101
  while (msg = frame.next)
102
- Log.log.debug("ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}")
102
+ Log.log.debug{"ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}"}
103
103
  message = msg.data
104
104
  if message.eql?(MSG_END_UPLOAD)
105
105
  shared_info[:end_uploads] += 1
@@ -122,12 +122,12 @@ module Aspera
122
122
  break
123
123
  end
124
124
  end
125
- Log.log.debug("ws: thread: stopping #{shared_info[:read_exception]} #{shared_info[:read_exception].class}")
125
+ Log.log.debug{"ws: thread: stopping (exc=#{shared_info[:read_exception]},cls=#{shared_info[:read_exception].class})"}
126
126
  end
127
127
  # notify progress bar
128
- notify_begin(session_id,total_size)
128
+ notify_begin(session_id, total_size)
129
129
  # first step send transfer spec
130
- Log.dump(:ws_spec,transfer_spec)
130
+ Log.dump(:ws_spec, transfer_spec)
131
131
  ws_snd_json(transfer_spec: transfer_spec)
132
132
  # current file index
133
133
  file_index = 0
@@ -135,58 +135,63 @@ module Aspera
135
135
  sent_bytes = 0
136
136
  # last progress event
137
137
  last_progress_time = nil
138
- begin
139
- transfer_spec['paths'].each do |item|
140
- # TODO: get mime type?
141
- file_mime_type = ''
142
- file_size = item['file_size']
143
- file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
144
- # compute total number of slices
145
- numslices = 1 + ((file_size - 1) / @options[:upload_chunksize])
146
- File.open(source_paths[file_index]) do |file|
147
- # current slice index
148
- slicenum = 0
149
- while !file.eof?
150
- # interrupt main thread if read thread failed
151
- raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
152
- data = file.read(@options[:upload_chunksize])
153
- slice_data = {
154
- name: file_name,
155
- type: file_mime_type,
156
- size: file_size,
157
- slice: slicenum,
158
- total_slices: numslices,
159
- fileIndex: file_index
160
- }
161
- #Log.dump(:slice_data,slice_data) #if slicenum.eql?(0)
138
+
139
+ transfer_spec['paths'].each do |item|
140
+ # TODO: get mime type?
141
+ file_mime_type = ''
142
+ file_size = item['file_size']
143
+ file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
144
+ # compute total number of slices
145
+ numslices = ((file_size - 1) / @options[:upload_chunksize]) + 1
146
+ File.open(source_paths[file_index]) do |file|
147
+ # current slice index
148
+ slicenum = 0
149
+ until file.eof?
150
+ data = file.read(@options[:upload_chunksize])
151
+ slice_data = {
152
+ name: file_name,
153
+ type: file_mime_type,
154
+ size: file_size,
155
+ slice: slicenum,
156
+ total_slices: numslices,
157
+ fileIndex: file_index
158
+ }
159
+ # Log.dump(:slice_data,slice_data) #if slicenum.eql?(0)
160
+ # interrupt main thread if read thread failed
161
+ raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
162
+ begin
162
163
  if upload_api_version.eql?(V1_UPLOAD)
163
164
  slice_data[:data] = Base64.strict_encode64(data)
164
165
  ws_snd_json(slice_upload: slice_data)
165
166
  else
166
167
  ws_snd_json(slice_upload: slice_data) if slicenum.eql?(0)
167
- ws_send(data,type: :binary)
168
+ ws_send(data, type: :binary)
168
169
  Log.log.debug{"ws: sent buffer: #{file_index} / #{slicenum}"}
169
- ws_snd_json(slice_upload: slice_data) if slicenum.eql?(numslices-1)
170
- end
171
- sent_bytes += data.length
172
- currenttime = Time.now
173
- if last_progress_time.nil? || ((currenttime - last_progress_time) > @options[:upload_bar_refresh_sec])
174
- notify_progress(session_id,sent_bytes)
175
- last_progress_time = currenttime
170
+ ws_snd_json(slice_upload: slice_data) if slicenum.eql?(numslices - 1)
176
171
  end
177
- slicenum += 1
172
+ rescue Errno::EPIPE => e
173
+ raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
174
+ raise e
175
+ end
176
+ sent_bytes += data.length
177
+ currenttime = Time.now
178
+ if last_progress_time.nil? || ((currenttime - last_progress_time) > @options[:upload_bar_refresh_sec])
179
+ notify_progress(session_id, sent_bytes)
180
+ last_progress_time = currenttime
178
181
  end
182
+ slicenum += 1
179
183
  end
180
- file_index += 1
181
184
  end
185
+ file_index += 1
182
186
  end
187
+
183
188
  Log.log.debug('Finished upload')
184
189
  ws_read_thread.join
185
190
  Log.log.debug{"result: #{shared_info[:end_uploads]} / #{@slice_uploads}"}
186
191
  ws_send(nil, type: :close) unless @ws_io.nil?
187
192
  @ws_io = nil
188
193
  http_socket&.finish
189
- notify_progress(session_id,sent_bytes)
194
+ notify_progress(session_id, sent_bytes)
190
195
  notify_end(session_id)
191
196
  end
192
197
 
@@ -194,18 +199,18 @@ module Aspera
194
199
  transfer_spec['zip_required'] ||= false
195
200
  transfer_spec['source_root'] ||= '/'
196
201
  # is normally provided by application, like package name
197
- if !transfer_spec.has_key?('download_name')
202
+ if !transfer_spec.key?('download_name')
198
203
  # by default it is the name of first file
199
204
  dname = File.basename(transfer_spec['paths'].first['source'])
200
205
  # we remove extension
201
- dname = dname.gsub(/\.@gw_api.*$/,'')
206
+ dname = dname.gsub(/\.@gw_api.*$/, '')
202
207
  # ands add indication of number of files if there is more than one
203
208
  if transfer_spec['paths'].length > 1
204
209
  dname += " #{transfer_spec['paths'].length} Files"
205
210
  end
206
211
  transfer_spec['download_name'] = dname
207
212
  end
208
- creation = @gw_api.create('v1/download',{'transfer_spec' => transfer_spec})[:data]
213
+ creation = @gw_api.create('v1/download', {'transfer_spec' => transfer_spec})[:data]
209
214
  transfer_uuid = creation['url'].split('/').last
210
215
  file_dest =
211
216
  if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
@@ -215,19 +220,18 @@ module Aspera
215
220
  # it is a plain file if we don't require zip and there is only one file
216
221
  File.basename(transfer_spec['paths'].first['source'])
217
222
  end
218
- file_dest = File.join(transfer_spec['destination_root'],file_dest)
219
- @gw_api.call({operation: 'GET',subpath: "v1/download/#{transfer_uuid}",save_to_file: file_dest})
223
+ file_dest = File.join(transfer_spec['destination_root'], file_dest)
224
+ @gw_api.call({operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_dest})
220
225
  end
221
226
 
222
227
  # start FASP transfer based on transfer spec (hash table)
223
228
  # note that it is asynchronous
224
229
  # HTTP download only supports file list
225
- def start_transfer(transfer_spec,options={})
230
+ def start_transfer(transfer_spec)
226
231
  raise 'GW URL must be set' if @gw_api.nil?
227
- raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
228
232
  raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
229
233
  raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
230
- Log.dump(:user_spec,transfer_spec)
234
+ Log.dump(:user_spec, transfer_spec)
231
235
  transfer_spec['authentication'] ||= 'token'
232
236
  case transfer_spec['direction']
233
237
  when Fasp::TransferSpec::DIRECTION_SEND
@@ -253,17 +257,17 @@ module Aspera
253
257
  private
254
258
 
255
259
  def initialize(opts)
256
- Log.log.debug("local options= #{opts}")
260
+ Log.log.debug{"local options= #{opts}"}
257
261
  # set default options and override if specified
258
262
  @options = DEFAULT_OPTIONS.dup
259
263
  raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
260
- opts.symbolize_keys.each do |k,v|
261
- raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
264
+ opts.symbolize_keys.each do |k, v|
265
+ raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
262
266
  @options[k] = v
263
267
  end
264
268
  raise 'missing param: url' if @options[:url].nil?
265
269
  # remove /v1 from end
266
- @options[:url].gsub(%r{/v1/*$},'')
270
+ @options[:url].gsub(%r{/v1/*$}, '')
267
271
  super()
268
272
  @gw_api = Rest.new({base_url: @options[:url]})
269
273
  @api_info = @gw_api.read('v1/info')[:data]
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'aspera/fasp/agent_base'
4
4
  require 'aspera/fasp/transfer_spec'
5
+ require 'aspera/node'
5
6
  require 'aspera/log'
6
7
  require 'tty-spinner'
7
8
 
@@ -9,20 +10,21 @@ module Aspera
9
10
  module Fasp
10
11
  # this singleton class is used by the CLI to provide a common interface to start a transfer
11
12
  # before using it, the use must set the `node_api` member.
12
- class AgentNode < AgentBase
13
+ class AgentNode < Aspera::Fasp::AgentBase
13
14
  # option include: root_id if the node is an access key
14
15
  attr_writer :options
16
+
15
17
  def initialize(options)
16
18
  raise 'node specification must be Hash' unless options.is_a?(Hash)
17
- %i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
19
+ %i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.key?(k) }
18
20
  super()
19
21
  # root id is required for access key
20
22
  @root_id = options[:root_id]
21
23
  rest_params = { base_url: options[:url]}
22
24
  if /^Bearer /.match?(options[:password])
23
25
  rest_params[:headers] = {
24
- 'X-Aspera-AccessKey' => options[:username],
25
- 'Authorization' => options[:password]
26
+ Aspera::Node::X_ASPERA_ACCESSKEY => options[:username],
27
+ 'Authorization' => options[:password]
26
28
  }
27
29
  raise 'root_id is required for access key' if @root_id.nil?
28
30
  else
@@ -39,7 +41,7 @@ module Aspera
39
41
 
40
42
  # used internally to ensure node api is set before using.
41
43
  def node_api_
42
- raise StandardError,'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
44
+ raise StandardError, 'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
43
45
  return @node_api
44
46
  end
45
47
  # use this to read the node_api end point.
@@ -54,7 +56,7 @@ module Aspera
54
56
  end
55
57
 
56
58
  # generic method
57
- def start_transfer(transfer_spec,_options=nil)
59
+ def start_transfer(transfer_spec)
58
60
  # add root id if access key
59
61
  if !@root_id.nil?
60
62
  case transfer_spec['direction']
@@ -64,9 +66,9 @@ module Aspera
64
66
  end
65
67
  end
66
68
  # manage special additional parameter
67
- if transfer_spec.has_key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
69
+ if transfer_spec.key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
68
70
  # not standard, so place standard field
69
- if transfer_spec.has_key?('ssh_private_key')
71
+ if transfer_spec.key?('ssh_private_key')
70
72
  Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
71
73
  else
72
74
  Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
@@ -77,13 +79,13 @@ module Aspera
77
79
  if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
78
80
  transfer_spec['tags']['aspera']['xfer_retry'] ||= 150
79
81
  end
80
- # optimisation in case of sending to the same node
81
- if transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
82
- transfer_spec['remote_host'] = 'localhost'
82
+ # Optimisation in case of sending to the same node (TODO: probably remove this, as /etc/hosts shall be used for that)
83
+ if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
84
+ transfer_spec['remote_host'] = '127.0.0.1'
83
85
  end
84
- resp = node_api_.create('ops/transfers',transfer_spec)[:data]
86
+ resp = node_api_.create('ops/transfers', transfer_spec)[:data]
85
87
  @transfer_id = resp['id']
86
- Log.log.debug("tr_id=#{@transfer_id}")
88
+ Log.log.debug{"tr_id=#{@transfer_id}"}
87
89
  return @transfer_id
88
90
  end
89
91
 
@@ -99,30 +101,30 @@ module Aspera
99
101
  when 'completed'
100
102
  notify_end(@transfer_id)
101
103
  break
102
- when 'waiting','partially_completed','unknown','waiting(read error)'
104
+ when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
103
105
  if spinner.nil?
104
106
  spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
105
107
  spinner.start
106
108
  end
107
109
  spinner.update(title: trdata['status'])
108
110
  spinner.spin
109
- #puts trdata
111
+ # puts trdata
110
112
  when 'running'
111
- #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
113
+ # puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
112
114
  if !started && trdata['precalc'].is_a?(Hash) &&
113
- trdata['precalc']['status'].eql?('ready')
114
- notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
115
+ trdata['precalc']['status'].eql?('ready')
116
+ notify_begin(@transfer_id, trdata['precalc']['bytes_expected'])
115
117
  started = true
116
118
  else
117
- notify_progress(@transfer_id,trdata['bytes_transferred'])
119
+ notify_progress(@transfer_id, trdata['bytes_transferred'])
118
120
  end
119
121
  else
120
- Log.log.warn("trdata -> #{trdata}")
122
+ Log.log.warn{"trdata -> #{trdata}"}
121
123
  raise Fasp::Error, "#{trdata['status']}: #{trdata['error_desc']}"
122
124
  end
123
125
  sleep(1)
124
126
  end
125
- #TODO get status of sessions
127
+ # TODO: get status of sessions
126
128
  return []
127
129
  end
128
130
  end
@@ -6,7 +6,7 @@ require 'json'
6
6
 
7
7
  module Aspera
8
8
  module Fasp
9
- class AgentTrsdk < AgentBase
9
+ class AgentTrsdk < Aspera::Fasp::AgentBase
10
10
  DEFAULT_OPTIONS = {
11
11
  address: '127.0.0.1',
12
12
  port: 55_002
@@ -18,26 +18,26 @@ module Aspera
18
18
  raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? || user_opts.is_a?(Hash)
19
19
  # set default options and override if specified
20
20
  options = DEFAULT_OPTIONS.dup
21
- user_opts&.each do |k,v|
22
- raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
21
+ user_opts&.each do |k, v|
22
+ raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
23
23
  options[k] = v
24
24
  end
25
- Log.log.debug("options= #{options}")
25
+ Log.log.debug{"options= #{options}"}
26
26
  super()
27
27
  # load and create SDK stub
28
28
  $LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
29
29
  require 'transfer_services_pb'
30
- @transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}",:this_channel_is_insecure)
30
+ @transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}", :this_channel_is_insecure)
31
31
  begin
32
32
  get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
33
- Log.log.debug("daemon info: #{get_info_response}")
33
+ Log.log.debug{"daemon info: #{get_info_response}"}
34
34
  rescue GRPC::Unavailable
35
35
  Log.log.warn('no daemon present, starting daemon...')
36
36
  # location of daemon binary
37
- bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
37
+ bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder, '..'))
38
38
  # config file and logs are created in same folder
39
- conf_file = File.join(bin_folder,'sdk.conf')
40
- log_base = File.join(bin_folder,'transferd')
39
+ conf_file = File.join(bin_folder, 'sdk.conf')
40
+ log_base = File.join(bin_folder, 'transferd')
41
41
  # create a config file for daemon
42
42
  config = {
43
43
  address: options[:address],
@@ -50,15 +50,15 @@ module Aspera
50
50
  }
51
51
  }
52
52
  }
53
- File.write(conf_file,config.to_json)
54
- trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
53
+ File.write(conf_file, config.to_json)
54
+ trd_pid = Process.spawn(Installation.instance.path(:transferd), '--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
55
55
  Process.detach(trd_pid)
56
56
  sleep(2.0)
57
57
  retry
58
58
  end
59
59
  end
60
60
 
61
- def start_transfer(transfer_spec,_options=nil)
61
+ def start_transfer(transfer_spec)
62
62
  # create a transfer request
63
63
  transfer_request = Transfersdk::TransferRequest.new(
64
64
  transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
@@ -66,9 +66,9 @@ module Aspera
66
66
  transferSpec: transfer_spec.to_json) # transfer definition
67
67
  # send start transfer request to the transfer manager daemon
68
68
  start_transfer_response = @transfer_client.start_transfer(transfer_request)
69
- Log.log.debug("start transfer response #{start_transfer_response}")
69
+ Log.log.debug{"start transfer response #{start_transfer_response}"}
70
70
  @transfer_id = start_transfer_response.transferId
71
- Log.log.debug("transfer started with id #{@transfer_id}")
71
+ Log.log.debug{"transfer started with id #{@transfer_id}"}
72
72
  end
73
73
 
74
74
  def wait_for_transfers_completion
@@ -76,26 +76,26 @@ module Aspera
76
76
  # monitor transfer status
77
77
  @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
78
78
  Log.dump(:response, response.to_h)
79
- #Log.log.debug("#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}")
79
+ # Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
80
80
  case response.status
81
81
  when :RUNNING
82
82
  if !started && !response.sessionInfo.preTransferBytes.eql?(0)
83
- notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
83
+ notify_begin(@transfer_id, response.sessionInfo.preTransferBytes)
84
84
  started = true
85
85
  elsif started
86
- notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
86
+ notify_progress(@transfer_id, response.transferInfo.bytesTransferred)
87
87
  end
88
88
  when :FAILED, :COMPLETED, :CANCELED
89
89
  notify_end(@transfer_id)
90
90
  raise Fasp::Error, JSON.parse(response.message)['Description'] unless :COMPLETED.eql?(response.status)
91
91
  break
92
- when :QUEUED,:UNKNOWN_STATUS,:PAUSED,:ORPHANED
92
+ when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
93
93
  # ignore
94
94
  else
95
- Log.log.error("unknown status#{response.status}")
95
+ Log.log.error{"unknown status#{response.status}"}
96
96
  end
97
97
  end
98
- # TODO return status
98
+ # TODO: return status
99
99
  return []
100
100
  end
101
101
  end
@@ -7,7 +7,8 @@ module Aspera
7
7
  # error raised if transfer fails
8
8
  class Error < StandardError
9
9
  attr_reader :err_code
10
- def initialize(message,err_code=nil)
10
+
11
+ def initialize(message, err_code=nil)
11
12
  super(message)
12
13
  @err_code = err_code
13
14
  end
@@ -17,7 +18,7 @@ module Aspera
17
18
  return r.merge({i: @err_code})
18
19
  end
19
20
 
20
- def retryable?; info[:r];end
21
+ def retryable?; info[:r]; end
21
22
  end
22
23
  end
23
24
  end
@@ -4,6 +4,8 @@ module Aspera
4
4
  module Fasp
5
5
  # from https://www.google.com/search?q=FASP+error+codes
6
6
  # Note that the fact that an error is retryable is not internally defined by protocol, it's client-side responsibility
7
+ # rubocop:disable Layout/MultilineHashKeyLineBreaks
8
+ # rubocop:disable Layout/FirstHashElementLineBreak
7
9
  ERROR_INFO = {
8
10
  # id retryable mnemo message additional info
9
11
  1 => { r: false, c: 'FASP_PROTO', m: 'Generic fasp(tm) protocol error', a: 'fasp(tm) error'},
@@ -18,13 +20,13 @@ module Aspera
18
20
  10 => { r: false, c: 'LIC_RATE_EXCEEDED', m: 'Rate exceeds the cap imposed by license', a: 'Rate exceeds cap imposed by license'},
19
21
  11 => { r: false, c: 'INTERNAL_ERROR', m: 'Internal error (unexpected error)', a: 'Internal error'},
20
22
  12 => { r: true, c: 'TRANSFER_ERROR', m: 'Error establishing control connection',
21
- a: 'Error establishing SSH connection (check SSH port and firewall)'},
23
+ a: 'Error establishing SSH connection (check SSH port and firewall)'},
22
24
  13 => { r: true, c: 'TRANSFER_TIMEOUT', m: 'Timeout establishing control connection',
23
- a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
25
+ a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
24
26
  14 => { r: true, c: 'CONNECTION_ERROR', m: 'Error establishing data connection',
25
- a: 'Error establishing UDP connection (check UDP port and firewall)'},
27
+ a: 'Error establishing UDP connection (check UDP port and firewall)'},
26
28
  15 => { r: true, c: 'CONNECTION_TIMEOUT', m: 'Timeout establishing data connection',
27
- a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
29
+ a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
28
30
  16 => { r: true, c: 'CONNECTION_LOST', m: 'Connection lost', a: 'Connection lost'},
29
31
  17 => { r: true, c: 'RCVR_SEND_ERROR', m: 'Receiver fails to send feedback', a: 'Network failure (receiver can\'t send feedback)'},
30
32
  18 => { r: true, c: 'RCVR_RECV_ERROR', m: 'Receiver fails to receive data packets', a: 'Network failure (receiver can\'t receive UDP data)'},
@@ -68,20 +70,21 @@ module Aspera
68
70
  54 => { r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed', a: 'File threshold validation failed'},
69
71
  55 => { r: false, c: 'FILEPATH_TOO_LONG', m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
70
72
  56 => { r: false, c: 'ILLEGAL_CHARS_IN_PATH', m: 'Windows path contains illegal characters',
71
- a: 'Path being written to Windows file system contains illegal characters'},
73
+ a: 'Path being written to Windows file system contains illegal characters'},
72
74
  57 => { r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT', m: 'Chunk size/start must be aligned with storage', a: 'Chunk size/start must be aligned with storage'},
73
75
  58 => { r: false, c: 'VALIDATION_SESSION_ABORT', m: 'Session aborted to due to validation error', a: 'Session aborted to due validation error'},
74
76
  59 => { r: false, c: 'REMOTE_STORAGE_ERROR', m: 'Remote storage errored', a: 'Remote storage errored'},
75
77
  60 => { r: false, c: 'LUA_SCRIPT_ABORTED_SESSION', m: 'Session aborted due to Lua script abort', a: 'Session aborted due to Lua script abort'},
76
78
  61 => { r: true, c: 'SSEAR_RETRYABLE', m: 'Transfer failed because of a retryable Encryption at Rest error',
77
- a: 'Transfer failed because of a retryable Encryption at Rest error'},
79
+ a: 'Transfer failed because of a retryable Encryption at Rest error'},
78
80
  62 => { r: false, c: 'SSEAR_FATAL', m: 'Transfer failed because of a fatal Encryption at Rest error',
79
- a: 'Transfer failed because of a fatal Encryption at Rest error'},
81
+ a: 'Transfer failed because of a fatal Encryption at Rest error'},
80
82
  63 => { r: false, c: 'LINK_LOOP', m: 'Path refers to a symbolic link loop', a: 'Path refers to a symbolic link loop'},
81
83
  64 => { r: false, c: 'CANNOT_RENAME_PARTIAL_FILES', m: 'Can\'t rename a partial file', a: 'Can\'t rename a partial file.'},
82
84
  65 => { r: false, c: 'CIPHER_NON_COMPAT_FIPS', m: 'Can\'t use this cipher with FIPS mode enabled', a: 'Can\'t use this cipher with FIPS mode enabled'},
83
85
  66 => { r: false, c: 'PEER_REQUIRES_FIPS', m: 'Peer rejects cipher due to FIPS mode enabled on peer',
84
- a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
86
+ a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
85
87
  }.freeze
88
+ # rubocop:enable Layout/MultilineHashKeyLineBreaks
86
89
  end
87
90
  end