aspera-cli 4.1.0 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -21,16 +21,24 @@ module Aspera
21
21
  ACCESS_KEY_TRANSFER_USER='xfer'
22
22
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
23
23
  class Local < Manager
24
- ASCP_SPAWN_TIMEOUT_SEC = 3
25
- private_constant :ASCP_SPAWN_TIMEOUT_SEC
26
- # set to false to keep ascp progress bar display (basically: removes ascp's option -q)
24
+ # options for initialize
25
+ DEFAULT_OPTIONS = {
26
+ :spawn_timeout_sec => 3,
27
+ :spawn_delay_sec => 2,
28
+ :wss => false,
29
+ :resume => {}
30
+ }
31
+ DEFAULT_UDP_PORT=33001
32
+ private_constant :DEFAULT_OPTIONS
33
+ # set to false to keep ascp progress bar display ("true" adds ascp's option -q)
27
34
  attr_accessor :quiet
35
+
28
36
  # start ascp transfer (non blocking), single or multi-session
29
37
  # job information added to @jobs
30
38
  # @param transfer_spec [Hash] aspera transfer specification
31
39
  # @param options [Hash] :resumer, :regenerate_token
32
40
  def start_transfer(transfer_spec,options={})
33
- raise "option: must be hash (or nil)" unless options.is_a?(Hash)
41
+ raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
34
42
  job_options = options.clone
35
43
  job_options[:resumer] ||= @resume_policy
36
44
  job_options[:job_id] ||= SecureRandom.uuid
@@ -58,21 +66,26 @@ module Aspera
58
66
 
59
67
  # TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
60
68
  # compute this before using transfer spec, even if the var is not used in single session
61
- multi_session_udp_port_base=33001
62
- multi_session_number=nil
69
+ multi_session_udp_port_base=DEFAULT_UDP_PORT
70
+ multi_session_number=0
63
71
  if transfer_spec.has_key?('multi_session')
64
72
  multi_session_number=transfer_spec['multi_session'].to_i
65
- raise "multi_session(#{transfer_spec['multi_session']}) shall be integer > 1" unless multi_session_number >= 1
66
- # managed here, so delete from transfer spec
67
- transfer_spec.delete('multi_session')
68
- if transfer_spec.has_key?('fasp_port')
69
- multi_session_udp_port_base=transfer_spec['fasp_port']
70
- transfer_spec.delete('fasp_port')
73
+ if multi_session_number < 0
74
+ Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
75
+ multi_session_number = 0
76
+ end
77
+ if multi_session_number > 0
78
+ # managed here, so delete from transfer spec
79
+ transfer_spec.delete('multi_session')
80
+ if transfer_spec.has_key?('fasp_port')
81
+ multi_session_udp_port_base=transfer_spec['fasp_port']
82
+ transfer_spec.delete('fasp_port')
83
+ end
71
84
  end
72
85
  end
73
86
 
74
87
  # compute known args
75
- env_args=Parameters.ts_to_env_args(transfer_spec,wss: @enable_wss)
88
+ env_args=Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
76
89
 
77
90
  # add fallback cert and key as arguments if needed
78
91
  if ['1','force'].include?(transfer_spec['http_fallback'])
@@ -98,25 +111,27 @@ module Aspera
98
111
  :options => job_options # [Hash]
99
112
  }
100
113
 
101
- Log.log.debug("starting session thread(s)")
102
- if !multi_session_number
114
+ if multi_session_number <= 1
115
+ Log.log.debug('Starting single session thread')
103
116
  # single session for transfer : simple
104
117
  session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
105
118
  xfer_job[:sessions].push(session)
106
119
  else
120
+ Log.log.debug('Starting multi session threads')
107
121
  1.upto(multi_session_number) do |i|
122
+ sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
108
123
  # do deep copy (each thread has its own copy because it is modified here below and in thread)
109
124
  this_session=session.clone()
110
125
  this_session[:env_args]=this_session[:env_args].clone()
111
126
  this_session[:env_args][:args]=this_session[:env_args][:args].clone()
112
127
  this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
113
128
  # necessary only if server is not linux, i.e. server does not support port re-use
114
- this_session[:env_args][:args].unshift("-O","#{multi_session_udp_port_base+i-1}")
129
+ this_session[:env_args][:args].unshift('-O',"#{multi_session_udp_port_base+i-1}")
115
130
  this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
116
131
  xfer_job[:sessions].push(this_session)
117
132
  end
118
133
  end
119
- Log.log.debug("started session thread(s)")
134
+ Log.log.debug('started session thread(s)')
120
135
 
121
136
  # add job to list of jobs
122
137
  @jobs[job_options[:job_id]]=xfer_job
@@ -128,7 +143,7 @@ module Aspera
128
143
  # wait for completion of all jobs started
129
144
  # @return list of :success or error message
130
145
  def wait_for_transfers_completion
131
- Log.log.debug("wait_for_transfers_completion")
146
+ Log.log.debug('wait_for_transfers_completion')
132
147
  # set to non-nil to exit loop
133
148
  result=[]
134
149
  @jobs.each do |id,job|
@@ -138,7 +153,7 @@ module Aspera
138
153
  result.push(session[:error] ? session[:error] : :success)
139
154
  end
140
155
  end
141
- Log.log.debug("all transfers joined")
156
+ Log.log.debug('all transfers joined')
142
157
  # since all are finished and we return the result, clear statuses
143
158
  @jobs.clear
144
159
  return result
@@ -146,7 +161,7 @@ module Aspera
146
161
 
147
162
  # used by asession (to be removed ?)
148
163
  def shutdown
149
- Log.log.debug("fasp local shutdown")
164
+ Log.log.debug('fasp local shutdown')
150
165
  end
151
166
 
152
167
  # This is the low level method to start the "ascp" process
@@ -158,8 +173,10 @@ module Aspera
158
173
  # @param session this session information
159
174
  # could be private method
160
175
  def start_transfer_with_args_env(env_args,session)
161
- raise "env_args must be Hash" unless env_args.is_a?(Hash)
162
- raise "session must be Hash" unless session.is_a?(Hash)
176
+ raise 'env_args must be Hash' unless env_args.is_a?(Hash)
177
+ raise 'session must be Hash' unless session.is_a?(Hash)
178
+ # by default we assume an exception will be raised (for ensure block)
179
+ exception_raised=true
163
180
  begin
164
181
  Log.log.debug("env_args=#{env_args.inspect}")
165
182
  # get location of ascp executable
@@ -182,7 +199,7 @@ module Aspera
182
199
  Log.log.debug("before accept for pid (#{ascp_pid})")
183
200
  # init management socket
184
201
  ascp_mgt_io=nil
185
- Timeout.timeout(ASCP_SPAWN_TIMEOUT_SEC) do
202
+ Timeout.timeout(@options[:spawn_timeout_sec]) do
186
203
  ascp_mgt_io = mgt_sock.accept
187
204
  # management messages include file names which may be utf8
188
205
  # by default socket is US-ASCII
@@ -216,7 +233,7 @@ module Aspera
216
233
  current_event_data[$1] = $2
217
234
  when ''
218
235
  # empty line is separator to end event information
219
- raise "unexpected empty line" if current_event_data.nil?
236
+ raise 'unexpected empty line' if current_event_data.nil?
220
237
  current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
221
238
  notify_listeners(current_event_text,current_event_data)
222
239
  case current_event_data['Type']
@@ -232,26 +249,27 @@ module Aspera
232
249
  end # case
233
250
  end # loop (process mgt port lines)
234
251
  # check that last status was received before process exit
235
- if last_status_event.nil?
236
- Log.log.warn("no status read from ascp mgt port")
237
- else
252
+ if last_status_event.is_a?(Hash)
238
253
  case last_status_event['Type']
239
254
  when 'DONE'
240
- # return method (or just don't do anything)
241
- return
255
+ # all went well
256
+ exception_raised=false
242
257
  when 'ERROR'
243
258
  Log.log.error("code: #{last_status_event['Code']}")
244
259
  if last_status_event['Description'] =~ /bearer token/i
245
- Log.log.error("need to regenerate token".red)
260
+ Log.log.error('need to regenerate token'.red)
246
261
  if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
247
262
  # regenerate token here, expired, or error on it
248
263
  env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
249
264
  end
250
265
  end
251
266
  raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
252
- else
267
+ else # case
253
268
  raise "unexpected last event type: #{last_status_event['Type']}"
254
269
  end
270
+ else
271
+ exception_raised=false
272
+ Log.log.debug('no status read from ascp mgt port')
255
273
  end
256
274
  rescue SystemCallError => e
257
275
  # Process.spawn
@@ -269,7 +287,13 @@ module Aspera
269
287
  ascp_pid=nil
270
288
  session.delete(:io)
271
289
  if !status.success?
272
- raise Fasp::Error.new("ascp failed with code #{status.exitstatus}")
290
+ message="ascp failed with code #{status.exitstatus}"
291
+ if exception_raised
292
+ # just debug, as main exception is already here
293
+ Log.log.debug(message)
294
+ else
295
+ raise Fasp::Error.new(message)
296
+ end
273
297
  end
274
298
  end
275
299
  end # begin-ensure
@@ -283,9 +307,9 @@ module Aspera
283
307
  # {'type'=>'DONE'}
284
308
  def send_command(job_id,session_index,data)
285
309
  job=@jobs[job_id]
286
- raise "no such job" if job.nil?
310
+ raise 'no such job' if job.nil?
287
311
  session=job[:sessions][session_index]
288
- raise "no such session" if session.nil?
312
+ raise 'no such session' if session.nil?
289
313
  Log.log.debug("command: #{data}")
290
314
  # build command
291
315
  command=data.
@@ -299,8 +323,8 @@ module Aspera
299
323
 
300
324
  private
301
325
 
302
- def initialize(agent_options=nil)
303
- agent_options||={}
326
+ # @param options : keys(symbol): wss, resume
327
+ def initialize(options=nil)
304
328
  super()
305
329
  # by default no interactive progress bar
306
330
  @quiet=true
@@ -308,9 +332,20 @@ module Aspera
308
332
  @jobs={}
309
333
  # mutex protects global data accessed by threads
310
334
  @mutex=Mutex.new
311
- @enable_wss = agent_options[:wss] || false
312
- agent_options.delete(:wss)
313
- @resume_policy=ResumePolicy.new(agent_options)
335
+ # manage options
336
+ @options=DEFAULT_OPTIONS.clone
337
+ if !options.nil?
338
+ raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
339
+ options.each do |k,v|
340
+ if DEFAULT_OPTIONS.has_key?(k)
341
+ @options[k]=v
342
+ else
343
+ raise "unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
344
+ end
345
+ end
346
+ end
347
+ Log.log.debug("local options= #{options}")
348
+ @resume_policy=ResumePolicy.new(@options[:resume].symbolize_keys)
314
349
  end
315
350
 
316
351
  # transfer thread entry
@@ -318,7 +353,7 @@ module Aspera
318
353
  def transfer_thread_entry(session)
319
354
  begin
320
355
  # set name for logging
321
- Thread.current[:name]="transfer"
356
+ Thread.current[:name]='transfer'
322
357
  Log.log.debug("ENTER (#{Thread.current[:name]})")
323
358
  # start transfer with selected resumer policy
324
359
  session[:options][:resumer].process do
@@ -59,7 +59,7 @@ module Aspera
59
59
  'exclude_older_than' => { :type => :opt_with_arg, :accepted_types=>Integer},
60
60
  'preserve_acls' => { :type => :opt_with_arg, :accepted_types=>String},
61
61
  'move_after_transfer' => { :type => :opt_with_arg, :accepted_types=>String},
62
- 'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>String},
62
+ 'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>Integer},
63
63
  # non standard parameters
64
64
  'EX_fasp_proxy_url' => { :type => :opt_with_arg, :option_switch=>'--proxy',:accepted_types=>String},
65
65
  'EX_http_proxy_url' => { :type => :opt_with_arg, :option_switch=>'-x',:accepted_types=>String},
@@ -78,11 +78,12 @@ module Aspera
78
78
  'lock_rate_policy' => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
79
79
  'lock_min_rate' => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
80
80
  'lock_target_rate' => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
81
- #'authentication' => { :type => :ignore, :accepted_types=>String}, # = token
81
+ 'authentication' => { :type => :ignore, :accepted_types=>String}, # value = token
82
82
  'https_fallback_port' => { :type => :ignore, :accepted_types=>Integer}, # same as http fallback, option -t ?
83
83
  'content_protection' => { :type => :ignore, :accepted_types=>String},
84
84
  'cipher_allowed' => { :type => :ignore, :accepted_types=>String},
85
85
  'multi_session' => { :type => :ignore, :accepted_types=>Integer}, # managed
86
+ 'obfuscate_file_names' => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
86
87
  # optional tags ( additional option to generate: {:space=>' ',:object_nl=>' ',:space_before=>'+',:array_nl=>'1'} )
87
88
  'tags' => { :type => :opt_with_arg, :option_switch=>'--tags64',:accepted_types=>Hash,:encode=>lambda{|tags|Base64.strict_encode64(JSON.generate(tags))}},
88
89
  # special processing @builder.process_param( called individually
@@ -14,17 +14,21 @@ module Aspera
14
14
  :sleep_max => 60
15
15
  }
16
16
 
17
- def initialize(params={})
17
+ # @param params see DEFAULTS
18
+ def initialize(params=nil)
18
19
  @parameters=DEFAULTS.clone
19
- return if params.nil?
20
- raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
21
- params.each do |k,v|
22
- if DEFAULTS.has_key?(k)
23
- @parameters[k]=v
24
- else
25
- raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
20
+ if !params.nil?
21
+ raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
22
+ params.each do |k,v|
23
+ if DEFAULTS.has_key?(k)
24
+ raise "#{k} must be Integer" unless v.is_a?(Integer)
25
+ @parameters[k]=v
26
+ else
27
+ raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
28
+ end
26
29
  end
27
30
  end
31
+ Log.log.debug("resume params=#{@parameters}")
28
32
  end
29
33
 
30
34
  # calls block a number of times (resumes) until success or limit reached
@@ -45,10 +49,7 @@ module Aspera
45
49
  # failure in ascp
46
50
  if e.retryable? then
47
51
  # exit if we exceed the max number of retry
48
- unless remaining_resumes > 0
49
- Log.log.error "Maximum number of retry reached"
50
- raise Fasp::Error,"max retry after: [#{status[:message]}]"
51
- end
52
+ raise Fasp::Error,'Maximum number of retry reached' if remaining_resumes <= 0
52
53
  else
53
54
  # give one chance only to non retryable errors
54
55
  unless remaining_resumes.eql?(@parameters[:iter_max])
data/lib/aspera/node.rb CHANGED
@@ -8,12 +8,25 @@ module Aspera
8
8
  class Node < Rest
9
9
  # permissions
10
10
  ACCESS_LEVELS=['delete','list','mkdir','preview','read','rename','write']
11
+ MATCH_EXEC_PREFIX='exec:'
11
12
 
12
13
  # for information only
13
14
  def self.decode_bearer_token(token)
14
15
  return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)
15
16
  end
16
17
 
18
+ # for access keys: provide expression to match entry in folder
19
+ # if no prefix: regex
20
+ # if prefix: ruby code
21
+ # if filder is nil, then always match
22
+ def self.file_matcher(match_expression)
23
+ match_expression||="#{MATCH_EXEC_PREFIX}true"
24
+ if match_expression.start_with?(MATCH_EXEC_PREFIX)
25
+ return eval "lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
26
+ end
27
+ return lambda{|f|f['name'].match(/#{match_expression}/)}
28
+ end
29
+
17
30
  def initialize(rest_params)
18
31
  super(rest_params)
19
32
  end
data/lib/aspera/oauth.rb CHANGED
@@ -225,6 +225,16 @@ module Aspera
225
225
  :nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET, # not before
226
226
  :exp => seconds_since_epoch+JWT_EXPIRY_OFFSET # expiration
227
227
  }
228
+ # Hum.. compliant ? TODO: remove when Faspex5 API is clarified
229
+ if @params[:jwt_is_f5]
230
+ payload[:jti] = SecureRandom.uuid
231
+ payload[:iat] = seconds_since_epoch
232
+ payload.delete(:nbf)
233
+ p_scope[:redirect_uri]="https://127.0.0.1:5000/token"
234
+ p_scope[:state]=SecureRandom.uuid
235
+ p_scope[:client_id]=@params[:client_id]
236
+ @token_auth_api.params[:auth]={:type=>:none}
237
+ end
228
238
 
229
239
  # non standard, only for global ids
230
240
  payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
@@ -233,8 +243,8 @@ module Aspera
233
243
 
234
244
  Log.log.debug("private=[#{rsa_private}]")
235
245
 
236
- Log.log.debug("JWT assertion=[#{payload}]")
237
- assertion = JWT.encode(payload, rsa_private, 'RS256')
246
+ Log.log.debug("JWT payload=[#{payload}]")
247
+ assertion = JWT.encode(payload, rsa_private, 'RS256',@params[:jwt_headers]||{})
238
248
 
239
249
  Log.log.debug("assertion=[#{assertion}]")
240
250
 
data/lib/aspera/rest.rb CHANGED
@@ -120,16 +120,6 @@ module Aspera
120
120
  # default is no auth
121
121
  @params[:auth]||={:type=>:none}
122
122
  @params[:not_auth_codes]||=['401']
123
- # translate old auth parameters, remove prefix, place in auth (TODO: delete this)
124
- # [:auth,:basic,:oauth].each do |p_sym|
125
- # p_str=p_sym.to_s+'_'
126
- # @params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
127
- # name=k_sym.to_s[p_str.length..-1]
128
- # name='grant' if k_sym.eql?(:oauth_type)
129
- # @params[:auth][name.to_sym]=@params[k_sym]
130
- # @params.delete(k_sym)
131
- # end
132
- # end
133
123
  @oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
134
124
  Log.dump('REST params(2)',@params)
135
125
  end
@@ -156,6 +146,7 @@ module Aspera
156
146
  Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
157
147
  call_data[:headers]||={}
158
148
  call_data[:headers]['User-Agent'] ||= @@user_agent
149
+ # defaults from @params are overriden by call dataz
159
150
  call_data=@params.deep_merge(call_data)
160
151
  case call_data[:auth][:type]
161
152
  when :none
@@ -278,7 +269,7 @@ module Aspera
278
269
  if e.response.is_a?(Net::HTTPRedirection)
279
270
  if tries_remain_redirect > 0
280
271
  tries_remain_redirect-=1
281
- Log.log.error("URL is moved, check your config: #{e.response['location']}")
272
+ Log.log.info("URL is moved: #{e.response['location']}")
282
273
  raise e
283
274
  # TODO: rebuild request with new location
284
275
  #retry
@@ -0,0 +1,20 @@
1
+ module Aspera
2
+ # Manage secrets in CLI using secure way (encryption, wallet, etc...)
3
+ class Secrets
4
+ attr_accessor :default_secret,:all_secrets
5
+ def initialize()
6
+ @default_secret=nil
7
+ @all_secrets={}
8
+ end
9
+
10
+ def get_secret(id=nil,mandatory=true)
11
+ secret=@default_secret || @all_secrets[id]
12
+ raise "please provide secret for #{id}" if secret.nil? and mandatory
13
+ return secret
14
+ end
15
+
16
+ def get_secrets
17
+ return @all_secrets
18
+ end
19
+ end
20
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: aspera-cli
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.1.0
4
+ version: 4.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Laurent Martin
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-06-23 00:00:00.000000000 Z
11
+ date: 2021-08-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: xml-simple
@@ -293,6 +293,7 @@ files:
293
293
  - lib/aspera/rest_call_error.rb
294
294
  - lib/aspera/rest_error_analyzer.rb
295
295
  - lib/aspera/rest_errors_aspera.rb
296
+ - lib/aspera/secrets.rb
296
297
  - lib/aspera/ssh.rb
297
298
  - lib/aspera/sync.rb
298
299
  - lib/aspera/temp_file_manager.rb