aspera-cli 4.13.0 → 4.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +28 -5
  4. data/CONTRIBUTING.md +17 -1
  5. data/README.md +782 -401
  6. data/examples/dascli +1 -1
  7. data/examples/rubyc +24 -0
  8. data/lib/aspera/aoc.rb +21 -32
  9. data/lib/aspera/ascmd.rb +1 -0
  10. data/lib/aspera/cli/basic_auth_plugin.rb +6 -6
  11. data/lib/aspera/cli/formatter.rb +17 -25
  12. data/lib/aspera/cli/main.rb +21 -27
  13. data/lib/aspera/cli/manager.rb +128 -114
  14. data/lib/aspera/cli/plugin.rb +87 -38
  15. data/lib/aspera/cli/plugins/alee.rb +2 -2
  16. data/lib/aspera/cli/plugins/aoc.rb +216 -102
  17. data/lib/aspera/cli/plugins/ats.rb +16 -18
  18. data/lib/aspera/cli/plugins/bss.rb +3 -3
  19. data/lib/aspera/cli/plugins/config.rb +177 -367
  20. data/lib/aspera/cli/plugins/console.rb +4 -6
  21. data/lib/aspera/cli/plugins/cos.rb +12 -13
  22. data/lib/aspera/cli/plugins/faspex.rb +17 -18
  23. data/lib/aspera/cli/plugins/faspex5.rb +332 -216
  24. data/lib/aspera/cli/plugins/node.rb +171 -142
  25. data/lib/aspera/cli/plugins/orchestrator.rb +15 -18
  26. data/lib/aspera/cli/plugins/preview.rb +38 -60
  27. data/lib/aspera/cli/plugins/server.rb +22 -15
  28. data/lib/aspera/cli/plugins/shares.rb +24 -33
  29. data/lib/aspera/cli/plugins/sync.rb +3 -3
  30. data/lib/aspera/cli/transfer_agent.rb +29 -26
  31. data/lib/aspera/cli/version.rb +1 -1
  32. data/lib/aspera/colors.rb +9 -7
  33. data/lib/aspera/data/6 +0 -0
  34. data/lib/aspera/environment.rb +7 -3
  35. data/lib/aspera/fasp/agent_connect.rb +5 -0
  36. data/lib/aspera/fasp/agent_direct.rb +5 -5
  37. data/lib/aspera/fasp/agent_httpgw.rb +138 -60
  38. data/lib/aspera/fasp/agent_trsdk.rb +2 -0
  39. data/lib/aspera/fasp/error_info.rb +2 -0
  40. data/lib/aspera/fasp/installation.rb +18 -19
  41. data/lib/aspera/fasp/parameters.rb +18 -17
  42. data/lib/aspera/fasp/parameters.yaml +2 -1
  43. data/lib/aspera/fasp/resume_policy.rb +3 -3
  44. data/lib/aspera/fasp/transfer_spec.rb +6 -5
  45. data/lib/aspera/fasp/uri.rb +23 -21
  46. data/lib/aspera/faspex_postproc.rb +1 -1
  47. data/lib/aspera/hash_ext.rb +12 -2
  48. data/lib/aspera/keychain/macos_security.rb +13 -13
  49. data/lib/aspera/log.rb +1 -0
  50. data/lib/aspera/node.rb +62 -80
  51. data/lib/aspera/oauth.rb +1 -1
  52. data/lib/aspera/persistency_action_once.rb +1 -1
  53. data/lib/aspera/preview/terminal.rb +61 -15
  54. data/lib/aspera/preview/utils.rb +3 -3
  55. data/lib/aspera/proxy_auto_config.js +2 -2
  56. data/lib/aspera/rest.rb +37 -0
  57. data/lib/aspera/secret_hider.rb +6 -1
  58. data/lib/aspera/ssh.rb +1 -1
  59. data/lib/aspera/sync.rb +2 -0
  60. data.tar.gz.sig +0 -0
  61. metadata +3 -4
  62. metadata.gz.sig +0 -0
  63. data/docs/test_env.conf +0 -186
  64. data/lib/aspera/data/7 +0 -0
@@ -25,7 +25,7 @@ module Aspera
25
25
  multi_incr_udp: true,
26
26
  resume: {},
27
27
  ascp_args: [],
28
- quiet: true # by default no interactive progress bar
28
+ quiet: true # by default no native ascp progress bar
29
29
  }.freeze
30
30
  private_constant :DEFAULT_OPTIONS
31
31
 
@@ -82,12 +82,12 @@ module Aspera
82
82
  end
83
83
 
84
84
  # compute known args
85
- env_args = Parameters.ts_to_env_args(transfer_spec, wss: @options[:wss], ascp_args: @options[:ascp_args])
85
+ env_args = Parameters.new(transfer_spec, @options).ascp_args
86
86
 
87
87
  # add fallback cert and key as arguments if needed
88
88
  if ['1', 1, true, 'force'].include?(transfer_spec['http_fallback'])
89
- env_args[:args].unshift('-Y', Installation.instance.path(:fallback_key))
90
- env_args[:args].unshift('-I', Installation.instance.path(:fallback_cert))
89
+ env_args[:args].unshift('-Y', Installation.instance.path(:fallback_cert_privkey))
90
+ env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
91
91
  end
92
92
 
93
93
  env_args[:args].unshift('-q') if @options[:quiet]
@@ -334,7 +334,7 @@ module Aspera
334
334
  # @param options : keys(symbol): see DEFAULT_OPTIONS
335
335
  def initialize(options=nil)
336
336
  super()
337
- # all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
337
+ # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
338
338
  @jobs = {}
339
339
  # mutex protects global data accessed by threads
340
340
  @mutex = Mutex.new
@@ -9,6 +9,17 @@ require 'websocket'
9
9
  require 'base64'
10
10
  require 'json'
11
11
 
12
+ # HTTP GW Upload protocol
13
+ # -----------------------
14
+ # v1
15
+ # 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
16
+ # 2.. - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end upload"
17
+ # v2
18
+ # 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
19
+ # 2 - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end_slice_upload"
20
+ # 3.. - MessageType: ByteArray (File Size) Chunks : acknowledged with "end upload"
21
+ # last - MessageType: String (Slice Upload end) JSON : type: slice_upload, acknowledged with "end_slice_upload"
22
+
12
23
  # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
13
24
  # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
14
25
  module Aspera
@@ -16,35 +27,66 @@ module Aspera
16
27
  # start a transfer using Aspera HTTP Gateway, using web socket session for uploads
17
28
  class AgentHttpgw < Aspera::Fasp::AgentBase
18
29
  # message returned by HTTP GW in case of success
19
- MSG_END_UPLOAD = 'end upload'
20
- MSG_END_SLICE = 'end_slice_upload'
30
+ MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
31
+ MSG_RECV_SLICE_UPLOAD_SIGNAL = 'end_slice_upload'
32
+ MSG_SEND_SLICE_UPLOAD = 'slice_upload'
33
+ MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
34
+ # upload API versions
35
+ API_V1 = 'v1'
36
+ API_V2 = 'v2'
21
37
  # options available in CLI (transfer_info)
22
38
  DEFAULT_OPTIONS = {
23
39
  url: nil,
24
40
  upload_chunk_size: 64_000,
25
- upload_bar_refresh_sec: 0.5
41
+ upload_bar_refresh_sec: 0.5,
42
+ api_version: API_V2,
43
+ synchronous: true
26
44
  }.freeze
27
45
  DEFAULT_BASE_PATH = '/aspera/http-gwy'
28
- # upload endpoints
29
- V1_UPLOAD = '/v1/upload'
30
- V2_UPLOAD = '/v2/upload'
31
- private_constant :DEFAULT_OPTIONS, :MSG_END_UPLOAD, :MSG_END_SLICE, :V1_UPLOAD, :V2_UPLOAD
46
+ LOG_WS_MAIN = 'ws: send: '.green
47
+ LOG_WS_THREAD = 'ws: ack: '.red
48
+ private_constant :DEFAULT_OPTIONS, :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL, :API_V1, :API_V2
32
49
 
33
50
  # send message on http gw web socket
34
- def ws_snd_json(data)
35
- @slice_uploads += 1 if data.key?(:slice_upload)
36
- Log.log.debug{JSON.generate(data)}
37
- ws_send(JSON.generate(data))
51
+ def ws_snd_json(msg_type, payload)
52
+ if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @options[:api_version].eql?(API_V2)
53
+ @shared_info[:count][:sent_v2_slice] += 1
54
+ else
55
+ @shared_info[:count][:sent_other] += 1
56
+ end
57
+ Log.log.debug do
58
+ log_data = payload.dup
59
+ log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
60
+ "send_txt: #{msg_type}: #{JSON.generate(log_data)}"
61
+ end
62
+ ws_send(JSON.generate({msg_type => payload}))
38
63
  end
39
64
 
40
- def ws_send(data, type: :text)
41
- frame = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: type, version: @ws_handshake.version)
65
+ def ws_send(data_to_send, type: :text)
66
+ Log.log.debug{"#{LOG_WS_MAIN}send low: type: #{type}"}
67
+ @shared_info[:count][:sent_other] += 1 if type.eql?(:binary)
68
+ Log.log.debug{"#{LOG_WS_MAIN}counts: #{@shared_info[:count]}"}
69
+ frame = ::WebSocket::Frame::Outgoing::Client.new(data: data_to_send, type: type, version: @ws_handshake.version)
42
70
  @ws_io.write(frame.to_s)
43
71
  end
44
72
 
73
+ # wait for all message sent to be acknowledged by HTTPGW server
74
+ def wait_for_sent_msg_ack_or_exception
75
+ return unless @options[:synchronous]
76
+ @shared_info[:mutex].synchronize do
77
+ while (@shared_info[:count][:received_data] != @shared_info[:count][:sent_other]) ||
78
+ (@shared_info[:count][:received_v2_slice] != @shared_info[:count][:sent_v2_slice])
79
+ Log.log.debug{"#{LOG_WS_MAIN}wait: counts: #{@shared_info[:count]}"}
80
+ @shared_info[:cond_var].wait(@shared_info[:mutex], 1.0)
81
+ raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
82
+ end
83
+ end
84
+ Log.log.debug{"#{LOG_WS_MAIN}sync ok: counts: #{@shared_info[:count]}"}
85
+ end
86
+
45
87
  def upload(transfer_spec)
46
88
  # total size of all files
47
- total_size = 0
89
+ total_bytes_to_transfer = 0
48
90
  # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
49
91
  source_paths = []
50
92
  # get source root or nil
@@ -61,50 +103,65 @@ module Aspera
61
103
  # GW expects a simple file name in 'source' but if user wants to change the name, we take it
62
104
  item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
63
105
  item['file_size'] = File.size(full_src_filepath)
64
- total_size += item['file_size']
106
+ total_bytes_to_transfer += item['file_size']
65
107
  # save so that we can actually read the file later
66
108
  source_paths.push(full_src_filepath)
67
109
  end
68
110
  # identify this session uniquely
69
111
  session_id = SecureRandom.uuid
70
- @slice_uploads = 0
71
- # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
72
- upload_api_version = V2_UPLOAD
73
- # is the latest supported? else revert to old api
74
- upload_api_version = V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
75
- Log.log.debug{"api version: #{upload_api_version}"}
76
- url = File.join(@gw_api.params[:base_url], upload_api_version)
77
- # uri = URI.parse(url)
112
+ upload_url = File.join(@gw_api.params[:base_url], @options[:api_version], 'upload')
113
+ # uri = URI.parse(upload_url)
78
114
  # open web socket to end point (equivalent to Net::HTTP.start)
79
- http_socket = Rest.start_http_session(url)
115
+ http_socket = Rest.start_http_session(upload_url)
116
+ # little hack to get the socket opened for HTTP, handy because HTTP debug will be available
80
117
  @ws_io = http_socket.instance_variable_get(:@socket)
81
118
  # @ws_io.debug_output = Log.log
82
- @ws_handshake = ::WebSocket::Handshake::Client.new(url: url, headers: {})
119
+ @ws_handshake = ::WebSocket::Handshake::Client.new(url: upload_url, headers: {})
83
120
  @ws_io.write(@ws_handshake.to_s)
84
121
  sleep(0.1)
85
122
  @ws_handshake << @ws_io.readuntil("\r\n\r\n")
86
123
  raise 'Error in websocket handshake' unless @ws_handshake.finished?
87
- Log.log.debug('ws: handshake success')
124
+ Log.log.debug{"#{LOG_WS_MAIN}handshake success"}
88
125
  # data shared between main thread and read thread
89
- shared_info = {
126
+ @shared_info = {
90
127
  read_exception: nil, # error message if any in callback
91
- end_uploads: 0 # number of files totally sent
92
- # mutex: Mutex.new
93
- # cond_var: ConditionVariable.new
128
+ count: {
129
+ received_data: 0, # number of files received on other side
130
+ received_v2_slice: 0, # number of slices received on other side
131
+ sent_other: 0,
132
+ sent_v2_slice: 0
133
+ },
134
+ mutex: Mutex.new,
135
+ cond_var: ConditionVariable.new
94
136
  }
95
137
  # start read thread
96
138
  ws_read_thread = Thread.new do
97
- Log.log.debug('ws: thread: started')
98
- frame = ::WebSocket::Frame::Incoming::Client.new
139
+ Log.log.debug{"#{LOG_WS_THREAD}read started"}
140
+ frame = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
99
141
  loop do
100
142
  begin # rubocop:disable Style/RedundantBegin
101
- frame << @ws_io.readuntil("\n")
143
+ # unless (recv_data = @ws_io.getc)
144
+ # sleep(0.1)
145
+ # next
146
+ # end
147
+ # frame << recv_data
148
+ # frame << @ws_io.readuntil("\n")
149
+ # frame << @ws_io.read_all
150
+ frame << @ws_io.read(1)
102
151
  while (msg = frame.next)
103
- Log.log.debug{"ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}"}
152
+ Log.log.debug{"#{LOG_WS_THREAD}type: #{msg.class}"}
104
153
  message = msg.data
105
- if message.eql?(MSG_END_UPLOAD)
106
- shared_info[:end_uploads] += 1
107
- elsif message.eql?(MSG_END_SLICE)
154
+ Log.log.debug{"#{LOG_WS_THREAD}message: [#{message}]"}
155
+ if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
156
+ @shared_info[:mutex].synchronize do
157
+ @shared_info[:count][:received_data] += 1
158
+ @shared_info[:cond_var].signal
159
+ end
160
+ elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
161
+ @shared_info[:mutex].synchronize do
162
+ @shared_info[:count][:received_v2_slice] += 1
163
+ @shared_info[:cond_var].signal
164
+ end
108
165
  else
109
166
  message.chomp!
110
167
  error_message =
@@ -117,19 +174,25 @@ module Aspera
117
174
  end
118
175
  raise error_message
119
176
  end
120
- end
177
+ Log.log.debug{"#{LOG_WS_THREAD}counts: #{@shared_info[:count]}"}
178
+ end # while
121
179
  rescue => e
122
- shared_info[:read_exception] = e unless e.is_a?(EOFError)
180
+ Log.log.debug{"#{LOG_WS_THREAD}Exception: #{e}"}
181
+ @shared_info[:mutex].synchronize do
182
+ @shared_info[:read_exception] = e unless e.is_a?(EOFError)
183
+ @shared_info[:cond_var].signal
184
+ end
123
185
  break
124
- end
125
- end
126
- Log.log.debug{"ws: thread: stopping (exc=#{shared_info[:read_exception]},cls=#{shared_info[:read_exception].class})"}
186
+ end # begin
187
+ end # loop
188
+ Log.log.debug{"#{LOG_WS_THREAD}stopping (exc=#{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"}
127
189
  end
128
190
  # notify progress bar
129
- notify_begin(session_id, total_size)
191
+ notify_begin(session_id, total_bytes_to_transfer)
130
192
  # first step send transfer spec
131
193
  Log.dump(:ws_spec, transfer_spec)
132
- ws_snd_json(transfer_spec: transfer_spec)
194
+ ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
195
+ wait_for_sent_msg_ack_or_exception
133
196
  # current file index
134
197
  file_index = 0
135
198
  # aggregate size sent
@@ -148,7 +211,7 @@ module Aspera
148
211
  # current slice index
149
212
  slice_index = 0
150
213
  until file.eof?
151
- data = file.read(@options[:upload_chunk_size])
214
+ file_bin_data = file.read(@options[:upload_chunk_size])
152
215
  slice_data = {
153
216
  name: file_name,
154
217
  type: file_mime_type,
@@ -159,22 +222,26 @@ module Aspera
159
222
  }
160
223
  # Log.dump(:slice_data,slice_data) #if slice_index.eql?(0)
161
224
  # interrupt main thread if read thread failed
162
- raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
225
+ raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
163
226
  begin
164
- if upload_api_version.eql?(V1_UPLOAD)
165
- slice_data[:data] = Base64.strict_encode64(data)
166
- ws_snd_json(slice_upload: slice_data)
227
+ if @options[:api_version].eql?(API_V1)
228
+ slice_data[:data] = Base64.strict_encode64(file_bin_data)
229
+ ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data)
167
230
  else
168
- ws_snd_json(slice_upload: slice_data) if slice_index.eql?(0)
169
- ws_send(data, type: :binary)
170
- Log.log.debug{"ws: sent buffer: #{file_index} / #{slice_index}"}
171
- ws_snd_json(slice_upload: slice_data) if slice_index.eql?(slice_total - 1)
231
+ ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(0)
232
+ ws_send(file_bin_data, type: :binary)
233
+ Log.log.debug{"#{LOG_WS_MAIN}sent bin buffer: #{file_index} / #{slice_index}"}
234
+ ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(slice_total - 1)
172
235
  end
236
+ wait_for_sent_msg_ack_or_exception
173
237
  rescue Errno::EPIPE => e
174
- raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
238
+ raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
239
+ raise e
240
+ rescue Net::ReadTimeout => e
241
+ Log.log.warn{'A timeout condition using HTTPGW may signal a permission problem on destination. Check ascp logs on httpgw.'}
175
242
  raise e
176
243
  end
177
- sent_bytes += data.length
244
+ sent_bytes += file_bin_data.length
178
245
  current_time = Time.now
179
246
  if last_progress_time.nil? || ((current_time - last_progress_time) > @options[:upload_bar_refresh_sec])
180
247
  notify_progress(session_id, sent_bytes)
@@ -186,9 +253,9 @@ module Aspera
186
253
  file_index += 1
187
254
  end
188
255
 
189
- Log.log.debug('Finished upload')
256
+ Log.log.debug('Finished upload, waiting for end of read thread.')
190
257
  ws_read_thread.join
191
- Log.log.debug{"result: #{shared_info[:end_uploads]} / #{@slice_uploads}"}
258
+ Log.log.debug{"Read thread joined, result: #{@shared_info[:count][:received_data]} / #{@shared_info[:count][:sent_other]}"}
192
259
  ws_send(nil, type: :close) unless @ws_io.nil?
193
260
  @ws_io = nil
194
261
  http_socket&.finish
@@ -258,7 +325,7 @@ module Aspera
258
325
  private
259
326
 
260
327
  def initialize(opts)
261
- Log.log.debug{"local options= #{opts}"}
328
+ Log.dump(:in_options, opts)
262
329
  # set default options and override if specified
263
330
  @options = DEFAULT_OPTIONS.dup
264
331
  raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
@@ -266,13 +333,24 @@ module Aspera
266
333
  raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
267
334
  @options[k] = v
268
335
  end
269
- raise 'missing param: url' if @options[:url].nil?
336
+ if @options[:url].nil?
337
+ available = DEFAULT_OPTIONS.map { |k, v| "#{k}(#{v})"}.join(', ')
338
+ raise "Missing mandatory parameter for HTTP GW in transfer_info: url. Allowed parameters: #{available}."
339
+ end
270
340
  # remove /v1 from end
271
341
  @options[:url].gsub(%r{/v1/*$}, '')
272
342
  super()
273
343
  @gw_api = Rest.new({base_url: @options[:url]})
274
344
  @api_info = @gw_api.read('v1/info')[:data]
275
- Log.log.info(@api_info.to_s)
345
+ Log.dump(:api_info, @api_info)
346
+ if @options[:api_version].nil?
347
+ # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
348
+ @options[:api_version] = API_V2
349
+ # is the latest supported? else revert to old api
350
+ @options[:api_version] = API_V1 unless @api_info['endpoints'].any?{|i|i.include?(@options[:api_version])}
351
+ end
352
+ @options.freeze
353
+ Log.dump(:final_options, @options)
276
354
  end
277
355
  end # AgentHttpgw
278
356
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # cspell:words Transfersdk
4
+
3
5
  require 'aspera/fasp/agent_base'
4
6
  require 'aspera/fasp/installation'
5
7
  require 'json'
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # cspell:words mnemo PROTO RCVR NOLIC PMTU BRTT VLINK BWMEAS SSEAR FIPS
4
+
3
5
  module Aspera
4
6
  module Fasp
5
7
  # from https://www.google.com/search?q=FASP+error+codes
@@ -126,7 +126,11 @@ module Aspera
126
126
  end
127
127
 
128
128
  # all ascp files (in SDK)
129
- FILES = %i[ascp ascp4 ssh_bypass_key_dsa ssh_bypass_key_rsa aspera_license aspera_conf fallback_cert fallback_key].freeze
129
+ FILES = %i[ascp ascp4 ssh_bypass_dsa_privkey ssh_bypass_rsa_privkey aspera_license aspera_conf fallback_certificate fallback_cert_privkey].freeze
130
+
131
+ def check_or_create_sdk_file(filename, force: false, &block)
132
+ return Environment.write_file_restricted(File.join(sdk_folder, filename), force: force, mode: 0o644, &block)
133
+ end
130
134
 
131
135
  # get path of one resource file of currently activated product
132
136
  # keys and certs are generated locally... (they are well known values, arch. independent)
@@ -139,23 +143,18 @@ module Aspera
139
143
  file = file.gsub('ascp', 'ascp4') if k.eql?(:ascp4)
140
144
  when :transferd
141
145
  file = transferd_filepath
142
- when :ssh_bypass_key_dsa
143
- file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera_bypass_dsa.pem')) {get_key('dsa', 1)}
144
- when :ssh_bypass_key_rsa
145
- file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera_bypass_rsa.pem')) {get_key('rsa', 2)}
146
+ when :ssh_bypass_dsa_privkey
147
+ file = check_or_create_sdk_file('aspera_bypass_dsa.pem') {get_key('dsa', 1)}
148
+ when :ssh_bypass_rsa_privkey
149
+ file = check_or_create_sdk_file('aspera_bypass_rsa.pem') {get_key('rsa', 2)}
146
150
  when :aspera_license
147
- file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera-license')) do
148
- clear = [
149
- Zlib::Inflate.inflate(DataRepository.instance.data(6)),
150
- "==SIGNATURE==\n",
151
- Base64.strict_encode64(DataRepository.instance.data(7))
152
- ]
153
- Base64.strict_encode64(clear.join)
151
+ file = check_or_create_sdk_file('aspera-license') do
152
+ Zlib::Inflate.inflate(DataRepository.instance.data(6))
154
153
  end
155
154
  when :aspera_conf
156
- file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera.conf')) {DEFAULT_ASPERA_CONF}
157
- when :fallback_cert, :fallback_key
158
- file_key = File.join(sdk_folder, 'aspera_fallback_key.pem')
155
+ file = check_or_create_sdk_file('aspera.conf') {DEFAULT_ASPERA_CONF}
156
+ when :fallback_certificate, :fallback_cert_privkey
157
+ file_key = File.join(sdk_folder, 'aspera_fallback_cert_private_key.pem')
159
158
  file_cert = File.join(sdk_folder, 'aspera_fallback_cert.pem')
160
159
  if !File.exist?(file_key) || !File.exist?(file_cert)
161
160
  require 'openssl'
@@ -169,10 +168,10 @@ module Aspera
169
168
  cert.serial = 0x0
170
169
  cert.version = 2
171
170
  cert.sign(private_key, OpenSSL::Digest.new('SHA1'))
172
- Environment.write_file_restricted(file_key, force: true) {private_key.to_pem}
173
- Environment.write_file_restricted(file_cert, force: true) {cert.to_pem}
171
+ check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true) {private_key.to_pem}
172
+ check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
174
173
  end
175
- file = k.eql?(:fallback_cert) ? file_cert : file_key
174
+ file = k.eql?(:fallback_certificate) ? file_cert : file_key
176
175
  else
177
176
  raise "INTERNAL ERROR: #{k}"
178
177
  end
@@ -206,7 +205,7 @@ module Aspera
206
205
  end
207
206
 
208
207
  def bypass_keys
209
- return %i[ssh_bypass_key_dsa ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
208
+ return %i[ssh_bypass_dsa_privkey ssh_bypass_rsa_privkey].map{|i|Installation.instance.path(i)}
210
209
  end
211
210
 
212
211
  # use in plugin `config`
@@ -3,6 +3,7 @@
3
3
  require 'aspera/log'
4
4
  require 'aspera/command_line_builder'
5
5
  require 'aspera/temp_file_manager'
6
+ require 'aspera/fasp/error'
6
7
  require 'securerandom'
7
8
  require 'base64'
8
9
  require 'json'
@@ -19,6 +20,7 @@ module Aspera
19
20
  # Short names of columns in manual
20
21
  SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
21
22
  FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
23
+ SUPPORTED_OPTIONS = %i[ascp_args wss].freeze
22
24
 
23
25
  private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
24
26
 
@@ -103,10 +105,6 @@ module Aspera
103
105
  ts.key?('EX_file_pair_list')
104
106
  end
105
107
 
106
- def ts_to_env_args(transfer_spec, wss:, ascp_args:)
107
- return Parameters.new(transfer_spec, wss: wss, ascp_args: ascp_args).ascp_args
108
- end
109
-
110
108
  # temp file list files are created here
111
109
  def file_list_folder=(v)
112
110
  @file_list_folder = v
@@ -120,19 +118,20 @@ module Aspera
120
118
  end # self
121
119
 
122
120
  # @param options [Hash] key: :wss: bool, :ascp_args: array of strings
123
- def initialize(job_spec, wss:, ascp_args:)
121
+ def initialize(job_spec, options)
124
122
  @job_spec = job_spec
125
- @opt_wss = wss
126
- @opt_args = ascp_args
127
- Log.log.debug{"agent options: #{@opt_wss} #{@opt_args}"}
128
- raise 'ascp args must be an Array' unless @opt_args.is_a?(Array)
129
- raise 'ascp args must be an Array of String' if @opt_args.any?{|i|!i.is_a?(String)}
123
+ # check necessary options
124
+ raise 'Internal: missing options' unless (SUPPORTED_OPTIONS - options.keys).empty?
125
+ @options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
126
+ Log.dump(:options, @options)
127
+ raise 'ascp args must be an Array' unless @options[:ascp_args].is_a?(Array)
128
+ raise 'ascp args must be an Array of String' if @options[:ascp_args].any?{|i|!i.is_a?(String)}
130
129
  @builder = Aspera::CommandLineBuilder.new(@job_spec, self.class.description)
131
130
  end
132
131
 
133
132
  def process_file_list
134
133
  # is the file list provided through EX_ parameters?
135
- ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @opt_args)
134
+ ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @options[:ascp_args])
136
135
  # set if paths is mandatory in ts
137
136
  @builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided
138
137
  # get paths in transfer spec (after setting if it is mandatory)
@@ -154,12 +153,14 @@ module Aspera
154
153
  Log.log.debug('placing source file list on command line (no file list file)')
155
154
  @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
156
155
  else
156
+ raise "All elements of paths must have a 'source' key" unless ts_paths_array.all?{|i|i.key?('source')}
157
+ is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
158
+ raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
157
159
  # safer option: generate a file list file if there is storage defined for it
158
- # if there is destination in paths, then use file-pair-list
159
- # TODO: well, we test only the first one, but anyway it shall be consistent
160
- if ts_paths_array.first.key?('destination')
160
+ # if there is one destination in paths, then use file-pair-list
161
+ if is_pair_list
161
162
  option = '--file-pair-list'
162
- lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']); }
163
+ lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'] || e['source']); }
163
164
  else
164
165
  option = '--file-list'
165
166
  lines = ts_paths_array.map{|i|i['source']}
@@ -194,7 +195,7 @@ module Aspera
194
195
  @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
195
196
 
196
197
  # use web socket session initiation ?
197
- if @builder.read_param('wss_enabled') && (@opt_wss || !@job_spec.key?('fasp_port'))
198
+ if @builder.read_param('wss_enabled') && (@options[:wss] || !@job_spec.key?('fasp_port'))
198
199
  # by default use web socket session if available, unless removed by user
199
200
  @builder.add_command_line_options(['--ws-connect'])
200
201
  # TODO: option to give order ssh,ws (legacy http is implied bu ssh)
@@ -226,7 +227,7 @@ module Aspera
226
227
  process_file_list
227
228
  # optional args, at the end to override previous ones (to allow override)
228
229
  @builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
229
- @builder.add_command_line_options(@opt_args)
230
+ @builder.add_command_line_options(@options[:ascp_args])
230
231
  # process destination folder
231
232
  destination_folder = @builder.read_param('destination_root') || '/'
232
233
  # ascp4 does not support base64 encoding of destination
@@ -8,6 +8,7 @@
8
8
  # cli.switch : ascp: switch for ascp command line
9
9
  # cli.convert : ascp: transform value: either a Hash with conversion values, or name of class
10
10
  # cli.variable : ascp: name of env var
11
+ # cspell:words dgram
11
12
  ---
12
13
  cipher:
13
14
  :desc: "In transit encryption type."
@@ -569,7 +570,7 @@ EX_at_rest_password:
569
570
  EX_proxy_password:
570
571
  :desc: |-
571
572
  Password used for Aspera proxy server authentication.
572
- May be overridden by password in URL EX_fasp_proxy_url.
573
+ May be overridden by password in URL provided in parameter: proxy.
573
574
 
574
575
 
575
576
  :agents:
@@ -49,11 +49,11 @@ module Aspera
49
49
  # failure in ascp
50
50
  if e.retryable?
51
51
  # exit if we exceed the max number of retry
52
- raise Fasp::Error, 'Maximum number of retry reached' if remaining_resumes <= 0
52
+ raise Fasp::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
53
53
  else
54
54
  # give one chance only to non retryable errors
55
55
  unless remaining_resumes.eql?(@parameters[:iter_max])
56
- Log.log.error('non-retryable error')
56
+ Log.log.error('non-retryable error'.red.blink)
57
57
  raise e
58
58
  end
59
59
  end
@@ -61,7 +61,7 @@ module Aspera
61
61
 
62
62
  # take this retry in account
63
63
  remaining_resumes -= 1
64
- Log.log.warn{"resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
64
+ Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
65
65
 
66
66
  # wait a bit before retrying, maybe network condition will be better
67
67
  sleep(sleep_seconds)
@@ -17,11 +17,12 @@ module Aspera
17
17
  }.freeze
18
18
  # reserved tag for Aspera
19
19
  TAG_RESERVED = 'aspera'
20
- # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128
21
- Aspera::Fasp::Parameters.description.each do |k, v|
22
- next unless v[:enum].is_a?(Array)
23
- v[:enum].each do |enum|
24
- TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
20
+ # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
21
+ Aspera::Fasp::Parameters.description.each do |name, description|
22
+ next unless description[:enum].is_a?(Array)
23
+ TransferSpec.const_set("#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
24
+ description[:enum].each do |enum|
25
+ TransferSpec.const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
25
26
  end
26
27
  end
27
28
  class << self