aspera-cli 4.8.0 → 4.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/README.md +445 -160
  4. data/docs/test_env.conf +1 -5
  5. data/examples/dascli +1 -4
  6. data/examples/{transfer.rb → node.rb} +17 -46
  7. data/examples/server.rb +93 -0
  8. data/lib/aspera/aoc.rb +4 -2
  9. data/lib/aspera/ats_api.rb +3 -1
  10. data/lib/aspera/cli/extended_value.rb +1 -0
  11. data/lib/aspera/cli/formater.rb +3 -1
  12. data/lib/aspera/cli/info.rb +1 -1
  13. data/lib/aspera/cli/main.rb +14 -11
  14. data/lib/aspera/cli/manager.rb +4 -4
  15. data/lib/aspera/cli/plugin.rb +50 -9
  16. data/lib/aspera/cli/plugins/aoc.rb +88 -52
  17. data/lib/aspera/cli/plugins/config.rb +5 -0
  18. data/lib/aspera/cli/plugins/faspex.rb +5 -4
  19. data/lib/aspera/cli/plugins/node.rb +3 -2
  20. data/lib/aspera/cli/plugins/server.rb +7 -108
  21. data/lib/aspera/cli/plugins/shares.rb +21 -1
  22. data/lib/aspera/cli/transfer_agent.rb +21 -14
  23. data/lib/aspera/cli/version.rb +1 -1
  24. data/lib/aspera/environment.rb +15 -2
  25. data/lib/aspera/fasp/agent_base.rb +9 -7
  26. data/lib/aspera/fasp/installation.rb +15 -19
  27. data/lib/aspera/fasp/parameters.rb +38 -30
  28. data/lib/aspera/fasp/parameters.yaml +69 -17
  29. data/lib/aspera/hash_ext.rb +14 -2
  30. data/lib/aspera/id_generator.rb +12 -10
  31. data/lib/aspera/keychain/encrypted_hash.rb +3 -3
  32. data/lib/aspera/log.rb +1 -1
  33. data/lib/aspera/nagios.rb +26 -19
  34. data/lib/aspera/oauth.rb +4 -4
  35. data/lib/aspera/open_application.rb +21 -19
  36. data/lib/aspera/persistency_folder.rb +3 -0
  37. data/lib/aspera/preview/image_error.png +0 -0
  38. data/lib/aspera/preview/video_error.png +0 -0
  39. data/lib/aspera/proxy_auto_config.rb +6 -4
  40. data/lib/aspera/rest_error_analyzer.rb +15 -13
  41. data/lib/aspera/rest_errors_aspera.rb +42 -40
  42. data/lib/aspera/secret_hider.rb +11 -5
  43. data/lib/aspera/ssh.rb +1 -0
  44. data/lib/aspera/uri_reader.rb +15 -13
  45. data.tar.gz.sig +0 -0
  46. metadata +4 -3
  47. metadata.gz.sig +0 -0
data/lib/aspera/oauth.rb CHANGED
@@ -56,7 +56,7 @@ module Aspera
56
56
  Log.log.debug('Not using persistency') # (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)
57
57
  # create NULL persistency class
58
58
  @persist = Class.new do
59
- def get(_x);nil;end;def delete(_x);nil;end;def put(_x,_y);nil;end;def garbage_collect(_x,_y);nil;end
59
+ def get(_x);nil;end;def delete(_x);nil;end;def put(_x,_y);nil;end;def garbage_collect(_x,_y);nil;end # rubocop:disable Layout/EmptyLineBetweenDefs
60
60
  end.new
61
61
  end
62
62
  return @persist
@@ -121,9 +121,9 @@ module Aspera
121
121
 
122
122
  # Authentication using Web browser
123
123
  register_token_creator :web,lambda { |oauth|
124
- verification_code = SecureRandom.uuid # used to check later
124
+ random_state = SecureRandom.uuid # used to check later
125
125
  login_page_url = Rest.build_uri("#{oauth.api[:base_url]}/#{oauth.sparams[:path_authorize]}",
126
- oauth.optional_scope_client_id.merge(response_type: 'code', redirect_uri: oauth.sparams[:redirect_uri], state: verification_code))
126
+ oauth.optional_scope_client_id.merge(response_type: 'code', redirect_uri: oauth.sparams[:redirect_uri], state: random_state))
127
127
  # here, we need a human to authorize on a web page
128
128
  Log.log.info("login_page_url=#{login_page_url}".bg_red.gray)
129
129
  # start a web server to receive request code
@@ -132,7 +132,7 @@ module Aspera
132
132
  OpenApplication.instance.uri(login_page_url)
133
133
  # wait for code in request
134
134
  received_params = webserver.received_request
135
- raise 'state does not match' unless verification_code.eql?(received_params['state'])
135
+ raise 'wrong received state' unless random_state.eql?(received_params['state'])
136
136
  # exchange code for token
137
137
  return oauth.create_token(oauth.optional_scope_client_id(add_secret: true).merge(
138
138
  grant_type: 'authorization_code',
@@ -11,27 +11,29 @@ module Aspera
11
11
  # if method is "graphical", then the URL will be opened with the default browser.
12
12
  class OpenApplication
13
13
  include Singleton
14
- # User Interfaces
15
- def self.user_interfaces; %i[text graphical]; end
14
+ class << self
15
+ # User Interfaces
16
+ def user_interfaces; %i[text graphical]; end
16
17
 
17
- def self.default_gui_mode
18
- return :graphical if [Aspera::Environment::OS_WINDOWS,Aspera::Environment::OS_X].include?(Aspera::Environment.os)
19
- # unix family
20
- return :graphical if ENV.has_key?('DISPLAY') && !ENV['DISPLAY'].empty?
21
- return :text
22
- end
18
+ def default_gui_mode
19
+ return :graphical if [Aspera::Environment::OS_WINDOWS,Aspera::Environment::OS_X].include?(Aspera::Environment.os)
20
+ # unix family
21
+ return :graphical if ENV.has_key?('DISPLAY') && !ENV['DISPLAY'].empty?
22
+ return :text
23
+ end
23
24
 
24
- # command must be non blocking
25
- def self.uri_graphical(uri)
26
- case Aspera::Environment.os
27
- when Aspera::Environment::OS_X
28
- return system('open',uri.to_s)
29
- when Aspera::Environment::OS_WINDOWS
30
- return system('start explorer "' + uri.to_s + '"')
31
- when Aspera::Environment::OS_LINUX
32
- return system("xdg-open '#{uri}'")
33
- else
34
- raise "no graphical open method for #{Aspera::Environment.os}"
25
+ # command must be non blocking
26
+ def uri_graphical(uri)
27
+ case Aspera::Environment.os
28
+ when Aspera::Environment::OS_X
29
+ return system('open',uri.to_s)
30
+ when Aspera::Environment::OS_WINDOWS
31
+ return system('start explorer "' + uri.to_s + '"')
32
+ when Aspera::Environment::OS_LINUX
33
+ return system("xdg-open '#{uri}'")
34
+ else
35
+ raise "no graphical open method for #{Aspera::Environment.os}"
36
+ end
35
37
  end
36
38
  end
37
39
 
@@ -36,7 +36,9 @@ module Aspera
36
36
  raise 'value: only String supported' unless value.is_a?(String)
37
37
  persist_filepath = id_to_filepath(object_id)
38
38
  Log.log.debug("persistency saving: #{persist_filepath}")
39
+ File.delete(persist_filepath) if File.exist?(persist_filepath)
39
40
  File.write(persist_filepath,value)
41
+ File.chmod(0400,persist_filepath)
40
42
  @cache[object_id] = value
41
43
  end
42
44
 
@@ -66,6 +68,7 @@ module Aspera
66
68
  def id_to_filepath(object_id)
67
69
  raise 'object_id: only String supported' unless object_id.is_a?(String)
68
70
  FileUtils.mkdir_p(@folder)
71
+ File.chmod(0700,@folder)
69
72
  return File.join(@folder,"#{object_id}#{FILE_SUFFIX}")
70
73
  #.gsub(/[^a-z]+/,FILE_FIELD_SEPARATOR)
71
74
  end
Binary file
Binary file
@@ -7,10 +7,12 @@ module URI
7
7
  class Generic
8
8
  # save original method that finds proxy in URI::Generic, it uses env var http_proxy
9
9
  alias_method :find_proxy_orig, :find_proxy
10
- def self.register_proxy_finder
11
- raise 'mandatory block missing' unless Kernel.block_given?
12
- # overload the method in URI : call user's provided block and fallback to original method
13
- define_method(:find_proxy) {|envars=ENV| yield(to_s) || find_proxy_orig(envars)}
10
+ class << self
11
+ def register_proxy_finder
12
+ raise 'mandatory block missing' unless Kernel.block_given?
13
+ # overload the method in URI : call user's provided block and fallback to original method
14
+ define_method(:find_proxy) {|envars=ENV| yield(to_s) || find_proxy_orig(envars)}
15
+ end
14
16
  end
15
17
  end
16
18
  end
@@ -80,19 +80,21 @@ module Aspera
80
80
  end
81
81
  end # add_simple_handler
82
82
 
83
- # used by handler to add an error description to list of errors
84
- # for logging and tracing : collect error descriptions (create file to activate)
85
- # @param call_context a Hash containing the result call_context, provided to handler
86
- # @param type a string describing type of exception, for logging purpose
87
- # @param msg one error message to add to list
88
- def self.add_error(call_context,type,msg)
89
- call_context[:messages].push(msg)
90
- logfile = instance.log_file
91
- # log error for further analysis (file must exist to activate)
92
- return if logfile.nil? || !File.exist?(logfile)
93
- File.open(logfile,'a+') do |f|
94
- f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
95
- "#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
83
+ class << self
84
+ # used by handler to add an error description to list of errors
85
+ # for logging and tracing : collect error descriptions (create file to activate)
86
+ # @param call_context a Hash containing the result call_context, provided to handler
87
+ # @param type a string describing type of exception, for logging purpose
88
+ # @param msg one error message to add to list
89
+ def add_error(call_context,type,msg)
90
+ call_context[:messages].push(msg)
91
+ logfile = instance.log_file
92
+ # log error for further analysis (file must exist to activate)
93
+ return if logfile.nil? || !File.exist?(logfile)
94
+ File.open(logfile,'a+') do |f|
95
+ f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n"\
96
+ "#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
97
+ end
96
98
  end
97
99
  end
98
100
  end
@@ -6,55 +6,57 @@ require 'aspera/log'
6
6
  module Aspera
7
7
  # REST error handlers for various Aspera REST APIs
8
8
  class RestErrorsAspera
9
- # handlers should probably be defined by plugins for modularity
10
- def self.register_handlers
11
- Log.log.debug('registering Aspera REST error handlers')
12
- # Faspex 4: both user_message and internal_message, and code 200
13
- # example: missing meta data on package creation
14
- RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
15
- RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description','error','description')
16
- RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message','error','internal_message')
17
- # AoC Automation
18
- RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
19
- RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
20
- RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
21
- RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
22
- if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
23
- call_context[:data]['errors'].each do |k,v|
24
- RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
9
+ class << self
10
+ # handlers should probably be defined by plugins for modularity
11
+ def register_handlers
12
+ Log.log.debug('registering Aspera REST error handlers')
13
+ # Faspex 4: both user_message and internal_message, and code 200
14
+ # example: missing meta data on package creation
15
+ RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
16
+ RestErrorAnalyzer.instance.add_simple_handler('Type 2: error:description','error','description')
17
+ RestErrorAnalyzer.instance.add_simple_handler('Type 3: error:internal_message','error','internal_message')
18
+ # AoC Automation
19
+ RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
20
+ RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
21
+ RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
22
+ RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
23
+ if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
24
+ call_context[:data]['errors'].each do |k,v|
25
+ RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
26
+ end
25
27
  end
26
28
  end
27
- end
28
- # call to upload_setup and download_setup of node api
29
- RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
30
- if call_context[:data].is_a?(Hash)
31
- d_t_s = call_context[:data]['transfer_specs']
32
- if d_t_s.is_a?(Array)
33
- d_t_s.each do |res|
34
- #r_err=res['transfer_spec']['error']
35
- r_err = res['error']
36
- if r_err.is_a?(Hash)
37
- RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
29
+ # call to upload_setup and download_setup of node api
30
+ RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
31
+ if call_context[:data].is_a?(Hash)
32
+ d_t_s = call_context[:data]['transfer_specs']
33
+ if d_t_s.is_a?(Array)
34
+ d_t_s.each do |res|
35
+ #r_err=res['transfer_spec']['error']
36
+ r_err = res['error']
37
+ if r_err.is_a?(Hash)
38
+ RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
39
+ end
38
40
  end
39
41
  end
40
42
  end
41
43
  end
42
- end
43
- RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
44
- RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
45
- RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
46
- if call_context[:data].is_a?(Hash)
47
- d_t_s = call_context[:data]['errors']
48
- if d_t_s.is_a?(Array)
49
- d_t_s.each do |res|
50
- r_err = res['message']
51
- if r_err.is_a?(String)
52
- RestErrorAnalyzer.add_error(call_context,type,r_err)
44
+ RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
45
+ RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
46
+ RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
47
+ if call_context[:data].is_a?(Hash)
48
+ d_t_s = call_context[:data]['errors']
49
+ if d_t_s.is_a?(Array)
50
+ d_t_s.each do |res|
51
+ r_err = res['message']
52
+ if r_err.is_a?(String)
53
+ RestErrorAnalyzer.add_error(call_context,type,r_err)
54
+ end
53
55
  end
54
56
  end
55
57
  end
56
58
  end
57
- end
58
- end # registerErrorTypes
59
+ end # register_handlers
60
+ end
59
61
  end
60
62
  end
@@ -8,11 +8,17 @@ module Aspera
8
8
  # display string for hidden secrets
9
9
  HIDDEN_PASSWORD = '🔑'
10
10
  # keys in hash that contain secrets
11
- SECRET_KEYWORDS = %w[password secret private_key passphrase].freeze
11
+ ASCP_SECRETS=%w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS].freeze
12
+ KEY_SECRETS =%w[password secret private_key passphrase].freeze
13
+ ALL_SECRETS =[ASCP_SECRETS,KEY_SECRETS].flatten.freeze
12
14
  # regex that define namec captures :begin and :end
13
15
  REGEX_LOG_REPLACES=[
14
- # replace values in logs with rendered JSON
15
- /(?<begin>["':][^"]*(#{SECRET_KEYWORDS.join('|')})[^"]*["']?[=>: ]+")[^"]+(?<end>")/,
16
+ # CLI manager get/set options
17
+ /(?<begin>[sg]et (#{KEY_SECRETS.join('|')})=).*(?<end>)/,
18
+ # env var ascp exec
19
+ /(?<begin> (#{ASCP_SECRETS.join('|')})=)[^ ]*(?<end> )/,
20
+ # rendered JSON
21
+ /(?<begin>["':][^"]*(#{ALL_SECRETS.join('|')})[^"]*["']?[=>: ]+")[^"]+(?<end>")/,
16
22
  # option "secret"
17
23
  /(?<begin>"[^"]*(secret)[^"]*"=>{)[^}]+(?<end>})/,
18
24
  # option "secrets"
@@ -20,7 +26,7 @@ module Aspera
20
26
  # private key values
21
27
  /(?<begin>--+BEGIN .+ KEY--+)[[:ascii:]]+?(?<end>--+?END .+ KEY--+)/
22
28
  ].freeze
23
- private_constant :HIDDEN_PASSWORD,:SECRET_KEYWORDS
29
+ private_constant :HIDDEN_PASSWORD,:ASCP_SECRETS,:KEY_SECRETS,:ALL_SECRETS,:REGEX_LOG_REPLACES
24
30
  @log_secrets = false
25
31
  class << self
26
32
  attr_accessor :log_secrets
@@ -40,7 +46,7 @@ module Aspera
40
46
  def secret?(keyword,value)
41
47
  keyword=keyword.to_s if keyword.is_a?(Symbol)
42
48
  # only Strings can be secrets, not booleans, or hash, arrays
43
- keyword.is_a?(String) && SECRET_KEYWORDS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
49
+ keyword.is_a?(String) && ALL_SECRETS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
44
50
  end
45
51
 
46
52
  def deep_remove_secret(obj,is_name_value: false)
data/lib/aspera/ssh.rb CHANGED
@@ -17,6 +17,7 @@ module Aspera
17
17
  # see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
18
18
  def initialize(host,username,ssh_options)
19
19
  Log.log.debug("ssh:#{username}@#{host}")
20
+ Log.log.debug("ssh_options:#{ssh_options}")
20
21
  @host = host
21
22
  @username = username
22
23
  @ssh_options = ssh_options
@@ -5,19 +5,21 @@ require 'aspera/rest'
5
5
 
6
6
  module Aspera
7
7
  module UriReader
8
- # read some content from some URI, support file: , http: and https: schemes
9
- def self.read(uri_to_read)
10
- proxy_uri = URI.parse(uri_to_read)
11
- case proxy_uri.scheme
12
- when 'http','https'
13
- return Rest.new(base_url: uri_to_read,redirect_max: 5).call(operation: 'GET', subpath: '', headers: {'Accept' => 'text/plain'})[:data]
14
- when 'file',NilClass
15
- local_file_path = proxy_uri.path
16
- raise 'URL shall have a path, check syntax' if local_file_path.nil?
17
- local_file_path = File.expand_path(local_file_path.gsub(/^\//,'')) if /^\/(~|.|..)\//.match?(local_file_path)
18
- return File.read(local_file_path)
19
- else
20
- raise "unknown scheme: [#{proxy_uri.scheme}] for [#{uri_to_read}]"
8
+ class << self
9
+ # read some content from some URI, support file: , http: and https: schemes
10
+ def read(uri_to_read)
11
+ proxy_uri = URI.parse(uri_to_read)
12
+ case proxy_uri.scheme
13
+ when 'http','https'
14
+ return Rest.new(base_url: uri_to_read,redirect_max: 5).call(operation: 'GET', subpath: '', headers: {'Accept' => 'text/plain'})[:data]
15
+ when 'file',NilClass
16
+ local_file_path = proxy_uri.path
17
+ raise 'URL shall have a path, check syntax' if local_file_path.nil?
18
+ local_file_path = File.expand_path(local_file_path.gsub(/^\//,'')) if /^\/(~|.|..)\//.match?(local_file_path)
19
+ return File.read(local_file_path)
20
+ else
21
+ raise "unknown scheme: [#{proxy_uri.scheme}] for [#{uri_to_read}]"
22
+ end
21
23
  end
22
24
  end
23
25
  end
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: aspera-cli
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.8.0
4
+ version: 4.9.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Laurent Martin
@@ -35,7 +35,7 @@ cert_chain:
35
35
  ZjkOWbUc1aLIsfaQFHWyNfisY9X2RgkFHjX0p5493wnoA7aWh52MUhc145npFh8z
36
36
  v4P9xwkT02Shkert4B4iwNvVjoAUGk+J4090svZCroAyXBjon5LV7MJ4fyw=
37
37
  -----END CERTIFICATE-----
38
- date: 2022-06-16 00:00:00.000000000 Z
38
+ date: 2022-09-14 00:00:00.000000000 Z
39
39
  dependencies:
40
40
  - !ruby/object:Gem::Dependency
41
41
  name: execjs
@@ -395,8 +395,9 @@ files:
395
395
  - examples/aoc.rb
396
396
  - examples/dascli
397
397
  - examples/faspex4.rb
398
+ - examples/node.rb
398
399
  - examples/proxy.pac
399
- - examples/transfer.rb
400
+ - examples/server.rb
400
401
  - lib/aspera/aoc.rb
401
402
  - lib/aspera/ascmd.rb
402
403
  - lib/aspera/ats_api.rb
metadata.gz.sig CHANGED
Binary file