aspera-cli 4.10.0 → 4.11.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +20 -0
  4. data/CHANGELOG.md +509 -0
  5. data/CONTRIBUTING.md +118 -0
  6. data/README.md +621 -378
  7. data/bin/ascli +4 -4
  8. data/bin/asession +11 -11
  9. data/docs/test_env.conf +28 -19
  10. data/examples/aoc.rb +4 -4
  11. data/examples/dascli +11 -9
  12. data/examples/faspex4.rb +8 -8
  13. data/examples/node.rb +11 -11
  14. data/examples/server.rb +9 -9
  15. data/lib/aspera/aoc.rb +273 -266
  16. data/lib/aspera/ascmd.rb +56 -54
  17. data/lib/aspera/ats_api.rb +4 -4
  18. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  19. data/lib/aspera/cli/extended_value.rb +5 -5
  20. data/lib/aspera/cli/formater.rb +64 -64
  21. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  22. data/lib/aspera/cli/listener/logger.rb +1 -1
  23. data/lib/aspera/cli/listener/progress.rb +5 -6
  24. data/lib/aspera/cli/listener/progress_multi.rb +14 -19
  25. data/lib/aspera/cli/main.rb +66 -67
  26. data/lib/aspera/cli/manager.rb +110 -110
  27. data/lib/aspera/cli/plugin.rb +54 -37
  28. data/lib/aspera/cli/plugins/alee.rb +4 -4
  29. data/lib/aspera/cli/plugins/aoc.rb +308 -669
  30. data/lib/aspera/cli/plugins/ats.rb +44 -46
  31. data/lib/aspera/cli/plugins/bss.rb +10 -10
  32. data/lib/aspera/cli/plugins/config.rb +447 -344
  33. data/lib/aspera/cli/plugins/console.rb +12 -12
  34. data/lib/aspera/cli/plugins/cos.rb +18 -20
  35. data/lib/aspera/cli/plugins/faspex.rb +110 -112
  36. data/lib/aspera/cli/plugins/faspex5.rb +67 -46
  37. data/lib/aspera/cli/plugins/node.rb +364 -288
  38. data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
  39. data/lib/aspera/cli/plugins/preview.rb +122 -114
  40. data/lib/aspera/cli/plugins/server.rb +137 -83
  41. data/lib/aspera/cli/plugins/shares.rb +30 -29
  42. data/lib/aspera/cli/plugins/sync.rb +13 -33
  43. data/lib/aspera/cli/transfer_agent.rb +57 -57
  44. data/lib/aspera/cli/version.rb +1 -1
  45. data/lib/aspera/colors.rb +3 -3
  46. data/lib/aspera/command_line_builder.rb +27 -27
  47. data/lib/aspera/cos_node.rb +22 -20
  48. data/lib/aspera/data_repository.rb +1 -1
  49. data/lib/aspera/environment.rb +30 -28
  50. data/lib/aspera/fasp/agent_base.rb +15 -15
  51. data/lib/aspera/fasp/agent_connect.rb +23 -21
  52. data/lib/aspera/fasp/agent_direct.rb +65 -67
  53. data/lib/aspera/fasp/agent_httpgw.rb +72 -68
  54. data/lib/aspera/fasp/agent_node.rb +23 -21
  55. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  56. data/lib/aspera/fasp/error.rb +3 -2
  57. data/lib/aspera/fasp/error_info.rb +11 -8
  58. data/lib/aspera/fasp/installation.rb +78 -78
  59. data/lib/aspera/fasp/listener.rb +1 -1
  60. data/lib/aspera/fasp/parameters.rb +75 -72
  61. data/lib/aspera/fasp/parameters.yaml +2 -2
  62. data/lib/aspera/fasp/resume_policy.rb +8 -8
  63. data/lib/aspera/fasp/transfer_spec.rb +35 -2
  64. data/lib/aspera/fasp/uri.rb +7 -7
  65. data/lib/aspera/faspex_gw.rb +7 -5
  66. data/lib/aspera/hash_ext.rb +3 -3
  67. data/lib/aspera/id_generator.rb +5 -5
  68. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  69. data/lib/aspera/keychain/macos_security.rb +21 -20
  70. data/lib/aspera/log.rb +7 -7
  71. data/lib/aspera/nagios.rb +19 -18
  72. data/lib/aspera/node.rb +209 -35
  73. data/lib/aspera/oauth.rb +37 -36
  74. data/lib/aspera/open_application.rb +19 -11
  75. data/lib/aspera/persistency_action_once.rb +4 -4
  76. data/lib/aspera/persistency_folder.rb +13 -13
  77. data/lib/aspera/preview/file_types.rb +8 -8
  78. data/lib/aspera/preview/generator.rb +67 -67
  79. data/lib/aspera/preview/utils.rb +27 -27
  80. data/lib/aspera/proxy_auto_config.js +41 -41
  81. data/lib/aspera/proxy_auto_config.rb +16 -16
  82. data/lib/aspera/rest.rb +56 -60
  83. data/lib/aspera/rest_call_error.rb +2 -1
  84. data/lib/aspera/rest_error_analyzer.rb +18 -17
  85. data/lib/aspera/rest_errors_aspera.rb +16 -16
  86. data/lib/aspera/secret_hider.rb +15 -13
  87. data/lib/aspera/ssh.rb +11 -10
  88. data/lib/aspera/sync.rb +158 -44
  89. data/lib/aspera/temp_file_manager.rb +2 -2
  90. data/lib/aspera/uri_reader.rb +4 -4
  91. data/lib/aspera/web_auth.rb +14 -13
  92. data.tar.gz.sig +0 -0
  93. metadata +8 -5
  94. metadata.gz.sig +0 -0
@@ -14,7 +14,7 @@ require 'json'
14
14
  module Aspera
15
15
  module Fasp
16
16
  # start a transfer using Aspera HTTP Gateway, using web socket session for uploads
17
- class AgentHttpgw < AgentBase
17
+ class AgentHttpgw < Aspera::Fasp::AgentBase
18
18
  # message returned by HTTP GW in case of success
19
19
  MSG_END_UPLOAD = 'end upload'
20
20
  MSG_END_SLICE = 'end_slice_upload'
@@ -23,15 +23,15 @@ module Aspera
23
23
  upload_chunksize: 64_000,
24
24
  upload_bar_refresh_sec: 0.5
25
25
  }.freeze
26
- DEFAULT_BASE_PATH='/aspera/http-gwy'
26
+ DEFAULT_BASE_PATH = '/aspera/http-gwy'
27
27
  # upload endpoints
28
- V1_UPLOAD='/v1/upload'
29
- V2_UPLOAD='/v2/upload'
30
- private_constant :DEFAULT_OPTIONS,:MSG_END_UPLOAD,:MSG_END_SLICE,:V1_UPLOAD,:V2_UPLOAD
28
+ V1_UPLOAD = '/v1/upload'
29
+ V2_UPLOAD = '/v2/upload'
30
+ private_constant :DEFAULT_OPTIONS, :MSG_END_UPLOAD, :MSG_END_SLICE, :V1_UPLOAD, :V2_UPLOAD
31
31
 
32
32
  # send message on http gw web socket
33
33
  def ws_snd_json(data)
34
- @slice_uploads += 1 if data.has_key?(:slice_upload)
34
+ @slice_uploads += 1 if data.key?(:slice_upload)
35
35
  Log.log.debug{JSON.generate(data)}
36
36
  ws_send(JSON.generate(data))
37
37
  end
@@ -47,7 +47,7 @@ module Aspera
47
47
  # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
48
48
  source_paths = []
49
49
  # get source root or nil
50
- source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
50
+ source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
51
51
  # source root is ignored by GW, used only here
52
52
  transfer_spec.delete('source_root')
53
53
  # compute total size of files to upload (for progress)
@@ -56,7 +56,7 @@ module Aspera
56
56
  # save actual file location to be able read contents later
57
57
  full_src_filepath = item['source']
58
58
  # add source root if needed
59
- full_src_filepath = File.join(source_root,full_src_filepath) unless source_root.nil?
59
+ full_src_filepath = File.join(source_root, full_src_filepath) unless source_root.nil?
60
60
  # GW expects a simple file name in 'source' but if user wants to change the name, we take it
61
61
  item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
62
62
  item['file_size'] = File.size(full_src_filepath)
@@ -66,18 +66,18 @@ module Aspera
66
66
  end
67
67
  # identify this session uniquely
68
68
  session_id = SecureRandom.uuid
69
- @slice_uploads=0
69
+ @slice_uploads = 0
70
70
  # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
71
71
  upload_api_version = V2_UPLOAD
72
72
  # is the latest supported? else revert to old api
73
- upload_api_version=V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
73
+ upload_api_version = V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
74
74
  Log.log.debug{"api version: #{upload_api_version}"}
75
- url=File.join(@gw_api.params[:base_url],upload_api_version)
76
- #uri = URI.parse(url)
75
+ url = File.join(@gw_api.params[:base_url], upload_api_version)
76
+ # uri = URI.parse(url)
77
77
  # open web socket to end point (equivalent to Net::HTTP.start)
78
78
  http_socket = Rest.start_http_session(url)
79
79
  @ws_io = http_socket.instance_variable_get(:@socket)
80
- #@ws_io.debug_output = Log.log
80
+ # @ws_io.debug_output = Log.log
81
81
  @ws_handshake = ::WebSocket::Handshake::Client.new(url: url, headers: {})
82
82
  @ws_io.write(@ws_handshake.to_s)
83
83
  sleep(0.1)
@@ -85,21 +85,21 @@ module Aspera
85
85
  raise 'Error in websocket handshake' unless @ws_handshake.finished?
86
86
  Log.log.debug('ws: handshake success')
87
87
  # data shared between main thread and read thread
88
- shared_info={
88
+ shared_info = {
89
89
  read_exception: nil, # error message if any in callback
90
90
  end_uploads: 0 # number of files totally sent
91
- #mutex: Mutex.new
92
- #cond_var: ConditionVariable.new
91
+ # mutex: Mutex.new
92
+ # cond_var: ConditionVariable.new
93
93
  }
94
94
  # start read thread
95
95
  ws_read_thread = Thread.new do
96
96
  Log.log.debug('ws: thread: started')
97
97
  frame = ::WebSocket::Frame::Incoming::Client.new
98
98
  loop do
99
- begin
99
+ begin # rubocop:disable Style/RedundantBegin
100
100
  frame << @ws_io.readuntil("\n")
101
101
  while (msg = frame.next)
102
- Log.log.debug("ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}")
102
+ Log.log.debug{"ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}"}
103
103
  message = msg.data
104
104
  if message.eql?(MSG_END_UPLOAD)
105
105
  shared_info[:end_uploads] += 1
@@ -122,12 +122,12 @@ module Aspera
122
122
  break
123
123
  end
124
124
  end
125
- Log.log.debug("ws: thread: stopping #{shared_info[:read_exception]} #{shared_info[:read_exception].class}")
125
+ Log.log.debug{"ws: thread: stopping (exc=#{shared_info[:read_exception]},cls=#{shared_info[:read_exception].class})"}
126
126
  end
127
127
  # notify progress bar
128
- notify_begin(session_id,total_size)
128
+ notify_begin(session_id, total_size)
129
129
  # first step send transfer spec
130
- Log.dump(:ws_spec,transfer_spec)
130
+ Log.dump(:ws_spec, transfer_spec)
131
131
  ws_snd_json(transfer_spec: transfer_spec)
132
132
  # current file index
133
133
  file_index = 0
@@ -135,58 +135,63 @@ module Aspera
135
135
  sent_bytes = 0
136
136
  # last progress event
137
137
  last_progress_time = nil
138
- begin
139
- transfer_spec['paths'].each do |item|
140
- # TODO: get mime type?
141
- file_mime_type = ''
142
- file_size = item['file_size']
143
- file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
144
- # compute total number of slices
145
- numslices = 1 + ((file_size - 1) / @options[:upload_chunksize])
146
- File.open(source_paths[file_index]) do |file|
147
- # current slice index
148
- slicenum = 0
149
- while !file.eof?
150
- # interrupt main thread if read thread failed
151
- raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
152
- data = file.read(@options[:upload_chunksize])
153
- slice_data = {
154
- name: file_name,
155
- type: file_mime_type,
156
- size: file_size,
157
- slice: slicenum,
158
- total_slices: numslices,
159
- fileIndex: file_index
160
- }
161
- #Log.dump(:slice_data,slice_data) #if slicenum.eql?(0)
138
+
139
+ transfer_spec['paths'].each do |item|
140
+ # TODO: get mime type?
141
+ file_mime_type = ''
142
+ file_size = item['file_size']
143
+ file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
144
+ # compute total number of slices
145
+ numslices = ((file_size - 1) / @options[:upload_chunksize]) + 1
146
+ File.open(source_paths[file_index]) do |file|
147
+ # current slice index
148
+ slicenum = 0
149
+ until file.eof?
150
+ data = file.read(@options[:upload_chunksize])
151
+ slice_data = {
152
+ name: file_name,
153
+ type: file_mime_type,
154
+ size: file_size,
155
+ slice: slicenum,
156
+ total_slices: numslices,
157
+ fileIndex: file_index
158
+ }
159
+ # Log.dump(:slice_data,slice_data) #if slicenum.eql?(0)
160
+ # interrupt main thread if read thread failed
161
+ raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
162
+ begin
162
163
  if upload_api_version.eql?(V1_UPLOAD)
163
164
  slice_data[:data] = Base64.strict_encode64(data)
164
165
  ws_snd_json(slice_upload: slice_data)
165
166
  else
166
167
  ws_snd_json(slice_upload: slice_data) if slicenum.eql?(0)
167
- ws_send(data,type: :binary)
168
+ ws_send(data, type: :binary)
168
169
  Log.log.debug{"ws: sent buffer: #{file_index} / #{slicenum}"}
169
- ws_snd_json(slice_upload: slice_data) if slicenum.eql?(numslices-1)
170
- end
171
- sent_bytes += data.length
172
- currenttime = Time.now
173
- if last_progress_time.nil? || ((currenttime - last_progress_time) > @options[:upload_bar_refresh_sec])
174
- notify_progress(session_id,sent_bytes)
175
- last_progress_time = currenttime
170
+ ws_snd_json(slice_upload: slice_data) if slicenum.eql?(numslices - 1)
176
171
  end
177
- slicenum += 1
172
+ rescue Errno::EPIPE => e
173
+ raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
174
+ raise e
175
+ end
176
+ sent_bytes += data.length
177
+ currenttime = Time.now
178
+ if last_progress_time.nil? || ((currenttime - last_progress_time) > @options[:upload_bar_refresh_sec])
179
+ notify_progress(session_id, sent_bytes)
180
+ last_progress_time = currenttime
178
181
  end
182
+ slicenum += 1
179
183
  end
180
- file_index += 1
181
184
  end
185
+ file_index += 1
182
186
  end
187
+
183
188
  Log.log.debug('Finished upload')
184
189
  ws_read_thread.join
185
190
  Log.log.debug{"result: #{shared_info[:end_uploads]} / #{@slice_uploads}"}
186
191
  ws_send(nil, type: :close) unless @ws_io.nil?
187
192
  @ws_io = nil
188
193
  http_socket&.finish
189
- notify_progress(session_id,sent_bytes)
194
+ notify_progress(session_id, sent_bytes)
190
195
  notify_end(session_id)
191
196
  end
192
197
 
@@ -194,18 +199,18 @@ module Aspera
194
199
  transfer_spec['zip_required'] ||= false
195
200
  transfer_spec['source_root'] ||= '/'
196
201
  # is normally provided by application, like package name
197
- if !transfer_spec.has_key?('download_name')
202
+ if !transfer_spec.key?('download_name')
198
203
  # by default it is the name of first file
199
204
  dname = File.basename(transfer_spec['paths'].first['source'])
200
205
  # we remove extension
201
- dname = dname.gsub(/\.@gw_api.*$/,'')
206
+ dname = dname.gsub(/\.@gw_api.*$/, '')
202
207
  # ands add indication of number of files if there is more than one
203
208
  if transfer_spec['paths'].length > 1
204
209
  dname += " #{transfer_spec['paths'].length} Files"
205
210
  end
206
211
  transfer_spec['download_name'] = dname
207
212
  end
208
- creation = @gw_api.create('v1/download',{'transfer_spec' => transfer_spec})[:data]
213
+ creation = @gw_api.create('v1/download', {'transfer_spec' => transfer_spec})[:data]
209
214
  transfer_uuid = creation['url'].split('/').last
210
215
  file_dest =
211
216
  if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
@@ -215,19 +220,18 @@ module Aspera
215
220
  # it is a plain file if we don't require zip and there is only one file
216
221
  File.basename(transfer_spec['paths'].first['source'])
217
222
  end
218
- file_dest = File.join(transfer_spec['destination_root'],file_dest)
219
- @gw_api.call({operation: 'GET',subpath: "v1/download/#{transfer_uuid}",save_to_file: file_dest})
223
+ file_dest = File.join(transfer_spec['destination_root'], file_dest)
224
+ @gw_api.call({operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_dest})
220
225
  end
221
226
 
222
227
  # start FASP transfer based on transfer spec (hash table)
223
228
  # note that it is asynchronous
224
229
  # HTTP download only supports file list
225
- def start_transfer(transfer_spec,options={})
230
+ def start_transfer(transfer_spec)
226
231
  raise 'GW URL must be set' if @gw_api.nil?
227
- raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
228
232
  raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
229
233
  raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
230
- Log.dump(:user_spec,transfer_spec)
234
+ Log.dump(:user_spec, transfer_spec)
231
235
  transfer_spec['authentication'] ||= 'token'
232
236
  case transfer_spec['direction']
233
237
  when Fasp::TransferSpec::DIRECTION_SEND
@@ -253,17 +257,17 @@ module Aspera
253
257
  private
254
258
 
255
259
  def initialize(opts)
256
- Log.log.debug("local options= #{opts}")
260
+ Log.log.debug{"local options= #{opts}"}
257
261
  # set default options and override if specified
258
262
  @options = DEFAULT_OPTIONS.dup
259
263
  raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
260
- opts.symbolize_keys.each do |k,v|
261
- raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
264
+ opts.symbolize_keys.each do |k, v|
265
+ raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
262
266
  @options[k] = v
263
267
  end
264
268
  raise 'missing param: url' if @options[:url].nil?
265
269
  # remove /v1 from end
266
- @options[:url].gsub(%r{/v1/*$},'')
270
+ @options[:url].gsub(%r{/v1/*$}, '')
267
271
  super()
268
272
  @gw_api = Rest.new({base_url: @options[:url]})
269
273
  @api_info = @gw_api.read('v1/info')[:data]
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'aspera/fasp/agent_base'
4
4
  require 'aspera/fasp/transfer_spec'
5
+ require 'aspera/node'
5
6
  require 'aspera/log'
6
7
  require 'tty-spinner'
7
8
 
@@ -9,20 +10,21 @@ module Aspera
9
10
  module Fasp
10
11
  # this singleton class is used by the CLI to provide a common interface to start a transfer
11
12
  # before using it, the use must set the `node_api` member.
12
- class AgentNode < AgentBase
13
+ class AgentNode < Aspera::Fasp::AgentBase
13
14
  # option include: root_id if the node is an access key
14
15
  attr_writer :options
16
+
15
17
  def initialize(options)
16
18
  raise 'node specification must be Hash' unless options.is_a?(Hash)
17
- %i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
19
+ %i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.key?(k) }
18
20
  super()
19
21
  # root id is required for access key
20
22
  @root_id = options[:root_id]
21
23
  rest_params = { base_url: options[:url]}
22
24
  if /^Bearer /.match?(options[:password])
23
25
  rest_params[:headers] = {
24
- 'X-Aspera-AccessKey' => options[:username],
25
- 'Authorization' => options[:password]
26
+ Aspera::Node::X_ASPERA_ACCESSKEY => options[:username],
27
+ 'Authorization' => options[:password]
26
28
  }
27
29
  raise 'root_id is required for access key' if @root_id.nil?
28
30
  else
@@ -39,7 +41,7 @@ module Aspera
39
41
 
40
42
  # used internally to ensure node api is set before using.
41
43
  def node_api_
42
- raise StandardError,'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
44
+ raise StandardError, 'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
43
45
  return @node_api
44
46
  end
45
47
  # use this to read the node_api end point.
@@ -54,7 +56,7 @@ module Aspera
54
56
  end
55
57
 
56
58
  # generic method
57
- def start_transfer(transfer_spec,_options=nil)
59
+ def start_transfer(transfer_spec)
58
60
  # add root id if access key
59
61
  if !@root_id.nil?
60
62
  case transfer_spec['direction']
@@ -64,9 +66,9 @@ module Aspera
64
66
  end
65
67
  end
66
68
  # manage special additional parameter
67
- if transfer_spec.has_key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
69
+ if transfer_spec.key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
68
70
  # not standard, so place standard field
69
- if transfer_spec.has_key?('ssh_private_key')
71
+ if transfer_spec.key?('ssh_private_key')
70
72
  Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
71
73
  else
72
74
  Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
@@ -77,13 +79,13 @@ module Aspera
77
79
  if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
78
80
  transfer_spec['tags']['aspera']['xfer_retry'] ||= 150
79
81
  end
80
- # optimisation in case of sending to the same node
81
- if transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
82
- transfer_spec['remote_host'] = 'localhost'
82
+ # Optimisation in case of sending to the same node (TODO: probably remove this, as /etc/hosts shall be used for that)
83
+ if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
84
+ transfer_spec['remote_host'] = '127.0.0.1'
83
85
  end
84
- resp = node_api_.create('ops/transfers',transfer_spec)[:data]
86
+ resp = node_api_.create('ops/transfers', transfer_spec)[:data]
85
87
  @transfer_id = resp['id']
86
- Log.log.debug("tr_id=#{@transfer_id}")
88
+ Log.log.debug{"tr_id=#{@transfer_id}"}
87
89
  return @transfer_id
88
90
  end
89
91
 
@@ -99,30 +101,30 @@ module Aspera
99
101
  when 'completed'
100
102
  notify_end(@transfer_id)
101
103
  break
102
- when 'waiting','partially_completed','unknown','waiting(read error)'
104
+ when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
103
105
  if spinner.nil?
104
106
  spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
105
107
  spinner.start
106
108
  end
107
109
  spinner.update(title: trdata['status'])
108
110
  spinner.spin
109
- #puts trdata
111
+ # puts trdata
110
112
  when 'running'
111
- #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
113
+ # puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
112
114
  if !started && trdata['precalc'].is_a?(Hash) &&
113
- trdata['precalc']['status'].eql?('ready')
114
- notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
115
+ trdata['precalc']['status'].eql?('ready')
116
+ notify_begin(@transfer_id, trdata['precalc']['bytes_expected'])
115
117
  started = true
116
118
  else
117
- notify_progress(@transfer_id,trdata['bytes_transferred'])
119
+ notify_progress(@transfer_id, trdata['bytes_transferred'])
118
120
  end
119
121
  else
120
- Log.log.warn("trdata -> #{trdata}")
122
+ Log.log.warn{"trdata -> #{trdata}"}
121
123
  raise Fasp::Error, "#{trdata['status']}: #{trdata['error_desc']}"
122
124
  end
123
125
  sleep(1)
124
126
  end
125
- #TODO get status of sessions
127
+ # TODO: get status of sessions
126
128
  return []
127
129
  end
128
130
  end
@@ -6,7 +6,7 @@ require 'json'
6
6
 
7
7
  module Aspera
8
8
  module Fasp
9
- class AgentTrsdk < AgentBase
9
+ class AgentTrsdk < Aspera::Fasp::AgentBase
10
10
  DEFAULT_OPTIONS = {
11
11
  address: '127.0.0.1',
12
12
  port: 55_002
@@ -18,26 +18,26 @@ module Aspera
18
18
  raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? || user_opts.is_a?(Hash)
19
19
  # set default options and override if specified
20
20
  options = DEFAULT_OPTIONS.dup
21
- user_opts&.each do |k,v|
22
- raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
21
+ user_opts&.each do |k, v|
22
+ raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
23
23
  options[k] = v
24
24
  end
25
- Log.log.debug("options= #{options}")
25
+ Log.log.debug{"options= #{options}"}
26
26
  super()
27
27
  # load and create SDK stub
28
28
  $LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
29
29
  require 'transfer_services_pb'
30
- @transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}",:this_channel_is_insecure)
30
+ @transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}", :this_channel_is_insecure)
31
31
  begin
32
32
  get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
33
- Log.log.debug("daemon info: #{get_info_response}")
33
+ Log.log.debug{"daemon info: #{get_info_response}"}
34
34
  rescue GRPC::Unavailable
35
35
  Log.log.warn('no daemon present, starting daemon...')
36
36
  # location of daemon binary
37
- bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
37
+ bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder, '..'))
38
38
  # config file and logs are created in same folder
39
- conf_file = File.join(bin_folder,'sdk.conf')
40
- log_base = File.join(bin_folder,'transferd')
39
+ conf_file = File.join(bin_folder, 'sdk.conf')
40
+ log_base = File.join(bin_folder, 'transferd')
41
41
  # create a config file for daemon
42
42
  config = {
43
43
  address: options[:address],
@@ -50,15 +50,15 @@ module Aspera
50
50
  }
51
51
  }
52
52
  }
53
- File.write(conf_file,config.to_json)
54
- trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
53
+ File.write(conf_file, config.to_json)
54
+ trd_pid = Process.spawn(Installation.instance.path(:transferd), '--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
55
55
  Process.detach(trd_pid)
56
56
  sleep(2.0)
57
57
  retry
58
58
  end
59
59
  end
60
60
 
61
- def start_transfer(transfer_spec,_options=nil)
61
+ def start_transfer(transfer_spec)
62
62
  # create a transfer request
63
63
  transfer_request = Transfersdk::TransferRequest.new(
64
64
  transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
@@ -66,9 +66,9 @@ module Aspera
66
66
  transferSpec: transfer_spec.to_json) # transfer definition
67
67
  # send start transfer request to the transfer manager daemon
68
68
  start_transfer_response = @transfer_client.start_transfer(transfer_request)
69
- Log.log.debug("start transfer response #{start_transfer_response}")
69
+ Log.log.debug{"start transfer response #{start_transfer_response}"}
70
70
  @transfer_id = start_transfer_response.transferId
71
- Log.log.debug("transfer started with id #{@transfer_id}")
71
+ Log.log.debug{"transfer started with id #{@transfer_id}"}
72
72
  end
73
73
 
74
74
  def wait_for_transfers_completion
@@ -76,26 +76,26 @@ module Aspera
76
76
  # monitor transfer status
77
77
  @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
78
78
  Log.dump(:response, response.to_h)
79
- #Log.log.debug("#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}")
79
+ # Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
80
80
  case response.status
81
81
  when :RUNNING
82
82
  if !started && !response.sessionInfo.preTransferBytes.eql?(0)
83
- notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
83
+ notify_begin(@transfer_id, response.sessionInfo.preTransferBytes)
84
84
  started = true
85
85
  elsif started
86
- notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
86
+ notify_progress(@transfer_id, response.transferInfo.bytesTransferred)
87
87
  end
88
88
  when :FAILED, :COMPLETED, :CANCELED
89
89
  notify_end(@transfer_id)
90
90
  raise Fasp::Error, JSON.parse(response.message)['Description'] unless :COMPLETED.eql?(response.status)
91
91
  break
92
- when :QUEUED,:UNKNOWN_STATUS,:PAUSED,:ORPHANED
92
+ when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
93
93
  # ignore
94
94
  else
95
- Log.log.error("unknown status#{response.status}")
95
+ Log.log.error{"unknown status#{response.status}"}
96
96
  end
97
97
  end
98
- # TODO return status
98
+ # TODO: return status
99
99
  return []
100
100
  end
101
101
  end
@@ -7,7 +7,8 @@ module Aspera
7
7
  # error raised if transfer fails
8
8
  class Error < StandardError
9
9
  attr_reader :err_code
10
- def initialize(message,err_code=nil)
10
+
11
+ def initialize(message, err_code=nil)
11
12
  super(message)
12
13
  @err_code = err_code
13
14
  end
@@ -17,7 +18,7 @@ module Aspera
17
18
  return r.merge({i: @err_code})
18
19
  end
19
20
 
20
- def retryable?; info[:r];end
21
+ def retryable?; info[:r]; end
21
22
  end
22
23
  end
23
24
  end
@@ -4,6 +4,8 @@ module Aspera
4
4
  module Fasp
5
5
  # from https://www.google.com/search?q=FASP+error+codes
6
6
  # Note that the fact that an error is retryable is not internally defined by protocol, it's client-side responsibility
7
+ # rubocop:disable Layout/MultilineHashKeyLineBreaks
8
+ # rubocop:disable Layout/FirstHashElementLineBreak
7
9
  ERROR_INFO = {
8
10
  # id retryable mnemo message additional info
9
11
  1 => { r: false, c: 'FASP_PROTO', m: 'Generic fasp(tm) protocol error', a: 'fasp(tm) error'},
@@ -18,13 +20,13 @@ module Aspera
18
20
  10 => { r: false, c: 'LIC_RATE_EXCEEDED', m: 'Rate exceeds the cap imposed by license', a: 'Rate exceeds cap imposed by license'},
19
21
  11 => { r: false, c: 'INTERNAL_ERROR', m: 'Internal error (unexpected error)', a: 'Internal error'},
20
22
  12 => { r: true, c: 'TRANSFER_ERROR', m: 'Error establishing control connection',
21
- a: 'Error establishing SSH connection (check SSH port and firewall)'},
23
+ a: 'Error establishing SSH connection (check SSH port and firewall)'},
22
24
  13 => { r: true, c: 'TRANSFER_TIMEOUT', m: 'Timeout establishing control connection',
23
- a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
25
+ a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
24
26
  14 => { r: true, c: 'CONNECTION_ERROR', m: 'Error establishing data connection',
25
- a: 'Error establishing UDP connection (check UDP port and firewall)'},
27
+ a: 'Error establishing UDP connection (check UDP port and firewall)'},
26
28
  15 => { r: true, c: 'CONNECTION_TIMEOUT', m: 'Timeout establishing data connection',
27
- a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
29
+ a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
28
30
  16 => { r: true, c: 'CONNECTION_LOST', m: 'Connection lost', a: 'Connection lost'},
29
31
  17 => { r: true, c: 'RCVR_SEND_ERROR', m: 'Receiver fails to send feedback', a: 'Network failure (receiver can\'t send feedback)'},
30
32
  18 => { r: true, c: 'RCVR_RECV_ERROR', m: 'Receiver fails to receive data packets', a: 'Network failure (receiver can\'t receive UDP data)'},
@@ -68,20 +70,21 @@ module Aspera
68
70
  54 => { r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed', a: 'File threshold validation failed'},
69
71
  55 => { r: false, c: 'FILEPATH_TOO_LONG', m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
70
72
  56 => { r: false, c: 'ILLEGAL_CHARS_IN_PATH', m: 'Windows path contains illegal characters',
71
- a: 'Path being written to Windows file system contains illegal characters'},
73
+ a: 'Path being written to Windows file system contains illegal characters'},
72
74
  57 => { r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT', m: 'Chunk size/start must be aligned with storage', a: 'Chunk size/start must be aligned with storage'},
73
75
  58 => { r: false, c: 'VALIDATION_SESSION_ABORT', m: 'Session aborted to due to validation error', a: 'Session aborted to due validation error'},
74
76
  59 => { r: false, c: 'REMOTE_STORAGE_ERROR', m: 'Remote storage errored', a: 'Remote storage errored'},
75
77
  60 => { r: false, c: 'LUA_SCRIPT_ABORTED_SESSION', m: 'Session aborted due to Lua script abort', a: 'Session aborted due to Lua script abort'},
76
78
  61 => { r: true, c: 'SSEAR_RETRYABLE', m: 'Transfer failed because of a retryable Encryption at Rest error',
77
- a: 'Transfer failed because of a retryable Encryption at Rest error'},
79
+ a: 'Transfer failed because of a retryable Encryption at Rest error'},
78
80
  62 => { r: false, c: 'SSEAR_FATAL', m: 'Transfer failed because of a fatal Encryption at Rest error',
79
- a: 'Transfer failed because of a fatal Encryption at Rest error'},
81
+ a: 'Transfer failed because of a fatal Encryption at Rest error'},
80
82
  63 => { r: false, c: 'LINK_LOOP', m: 'Path refers to a symbolic link loop', a: 'Path refers to a symbolic link loop'},
81
83
  64 => { r: false, c: 'CANNOT_RENAME_PARTIAL_FILES', m: 'Can\'t rename a partial file', a: 'Can\'t rename a partial file.'},
82
84
  65 => { r: false, c: 'CIPHER_NON_COMPAT_FIPS', m: 'Can\'t use this cipher with FIPS mode enabled', a: 'Can\'t use this cipher with FIPS mode enabled'},
83
85
  66 => { r: false, c: 'PEER_REQUIRES_FIPS', m: 'Peer rejects cipher due to FIPS mode enabled on peer',
84
- a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
86
+ a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
85
87
  }.freeze
88
+ # rubocop:enable Layout/MultilineHashKeyLineBreaks
86
89
  end
87
90
  end