aspera-cli 4.23.0 → 4.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +37 -1
  4. data/CONTRIBUTING.md +86 -29
  5. data/README.md +2109 -1300
  6. data/bin/ascli +2 -1
  7. data/bin/asession +4 -4
  8. data/lib/aspera/agent/base.rb +4 -0
  9. data/lib/aspera/agent/connect.rb +20 -18
  10. data/lib/aspera/agent/desktop.rb +14 -11
  11. data/lib/aspera/agent/direct.rb +39 -31
  12. data/lib/aspera/agent/httpgw.rb +2 -2
  13. data/lib/aspera/agent/node.rb +9 -11
  14. data/lib/aspera/agent/transferd.rb +18 -11
  15. data/lib/aspera/api/aoc.rb +44 -31
  16. data/lib/aspera/api/cos_node.rb +7 -5
  17. data/lib/aspera/api/httpgw.rb +15 -18
  18. data/lib/aspera/api/node.rb +104 -22
  19. data/lib/aspera/ascmd.rb +22 -16
  20. data/lib/aspera/ascp/installation.rb +37 -40
  21. data/lib/aspera/ascp/management.rb +5 -4
  22. data/lib/aspera/assert.rb +54 -23
  23. data/lib/aspera/cli/basic_auth_plugin.rb +8 -7
  24. data/lib/aspera/cli/error.rb +1 -1
  25. data/lib/aspera/cli/extended_value.rb +28 -29
  26. data/lib/aspera/cli/formatter.rb +191 -168
  27. data/lib/aspera/cli/hints.rb +29 -3
  28. data/lib/aspera/cli/main.rb +138 -107
  29. data/lib/aspera/cli/manager.rb +50 -30
  30. data/lib/aspera/cli/plugin.rb +148 -77
  31. data/lib/aspera/cli/plugin_factory.rb +2 -2
  32. data/lib/aspera/cli/plugins/aoc.rb +189 -70
  33. data/lib/aspera/cli/plugins/ats.rb +15 -13
  34. data/lib/aspera/cli/plugins/config.rb +100 -214
  35. data/lib/aspera/cli/plugins/console.rb +49 -18
  36. data/lib/aspera/cli/plugins/cos.rb +4 -4
  37. data/lib/aspera/cli/plugins/faspex.rb +45 -51
  38. data/lib/aspera/cli/plugins/faspex5.rb +164 -165
  39. data/lib/aspera/cli/plugins/faspio.rb +6 -5
  40. data/lib/aspera/cli/plugins/httpgw.rb +2 -2
  41. data/lib/aspera/cli/plugins/node.rb +144 -162
  42. data/lib/aspera/cli/plugins/orchestrator.rb +10 -14
  43. data/lib/aspera/cli/plugins/preview.rb +26 -29
  44. data/lib/aspera/cli/plugins/server.rb +28 -28
  45. data/lib/aspera/cli/plugins/shares.rb +40 -28
  46. data/lib/aspera/cli/sync_actions.rb +101 -80
  47. data/lib/aspera/cli/transfer_agent.rb +51 -50
  48. data/lib/aspera/cli/transfer_progress.rb +29 -20
  49. data/lib/aspera/cli/version.rb +1 -1
  50. data/lib/aspera/cli/wizard.rb +157 -0
  51. data/lib/aspera/colors.rb +13 -8
  52. data/lib/aspera/command_line_builder.rb +28 -22
  53. data/lib/aspera/command_line_converter.rb +31 -0
  54. data/lib/aspera/environment.rb +145 -101
  55. data/lib/aspera/faspex_gw.rb +1 -1
  56. data/lib/aspera/faspex_postproc.rb +3 -2
  57. data/lib/aspera/hash_ext.rb +1 -1
  58. data/lib/aspera/id_generator.rb +10 -10
  59. data/lib/aspera/keychain/base.rb +18 -0
  60. data/lib/aspera/keychain/encrypted_hash.rb +6 -12
  61. data/lib/aspera/keychain/factory.rb +9 -3
  62. data/lib/aspera/keychain/hashicorp_vault.rb +9 -6
  63. data/lib/aspera/keychain/macos_security.rb +13 -13
  64. data/lib/aspera/log.rb +91 -19
  65. data/lib/aspera/nagios.rb +5 -6
  66. data/lib/aspera/node_simulator.rb +12 -7
  67. data/lib/aspera/oauth/base.rb +5 -3
  68. data/lib/aspera/oauth/factory.rb +24 -18
  69. data/lib/aspera/oauth/jwt.rb +13 -1
  70. data/lib/aspera/oauth/url_json.rb +3 -3
  71. data/lib/aspera/oauth/web.rb +5 -3
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/file_types.rb +4 -3
  74. data/lib/aspera/preview/generator.rb +25 -12
  75. data/lib/aspera/preview/terminal.rb +10 -7
  76. data/lib/aspera/preview/utils.rb +11 -9
  77. data/lib/aspera/products/connect.rb +1 -1
  78. data/lib/aspera/products/desktop.rb +1 -1
  79. data/lib/aspera/products/other.rb +2 -2
  80. data/lib/aspera/products/transferd.rb +8 -6
  81. data/lib/aspera/proxy_auto_config.rb +1 -1
  82. data/lib/aspera/rest.rb +29 -22
  83. data/lib/aspera/rest_call_error.rb +1 -1
  84. data/lib/aspera/resumer.rb +1 -1
  85. data/lib/aspera/secret_hider.rb +46 -40
  86. data/lib/aspera/ssh.rb +13 -3
  87. data/lib/aspera/sync/args.schema.yaml +102 -0
  88. data/lib/aspera/sync/conf.schema.yaml +701 -0
  89. data/lib/aspera/sync/database.rb +83 -0
  90. data/lib/aspera/sync/operations.rb +296 -0
  91. data/lib/aspera/temp_file_manager.rb +3 -2
  92. data/lib/aspera/transfer/error.rb +1 -1
  93. data/lib/aspera/transfer/error_info.rb +1 -2
  94. data/lib/aspera/transfer/faux_file.rb +11 -10
  95. data/lib/aspera/transfer/parameters.rb +6 -5
  96. data/lib/aspera/transfer/spec.rb +15 -1
  97. data/lib/aspera/transfer/spec.schema.yaml +316 -293
  98. data/lib/aspera/transfer/spec_doc.rb +34 -16
  99. data/lib/aspera/transfer/uri.rb +5 -5
  100. data/lib/aspera/uri_reader.rb +14 -10
  101. data/lib/aspera/web_auth.rb +2 -2
  102. data/lib/aspera/web_server_simple.rb +2 -2
  103. data.tar.gz.sig +0 -0
  104. metadata +15 -13
  105. metadata.gz.sig +0 -0
  106. data/lib/aspera/transfer/async_conf.schema.yaml +0 -716
  107. data/lib/aspera/transfer/convert.rb +0 -29
  108. data/lib/aspera/transfer/sync.rb +0 -232
  109. data/lib/aspera/transfer/sync_instance.schema.yaml +0 -20
  110. data/lib/aspera/transfer/sync_session.schema.yaml +0 -86
@@ -4,20 +4,21 @@ require 'aspera/hash_ext'
4
4
  require 'aspera/environment'
5
5
  require 'aspera/log'
6
6
  require 'aspera/assert'
7
+ require 'aspera/keychain/base'
7
8
  require 'symmetric_encryption/core'
8
9
  require 'yaml'
9
10
 
10
11
  module Aspera
11
12
  module Keychain
12
13
  # Manage secrets in a simple Hash
13
- class EncryptedHash
14
+ class EncryptedHash < Base
14
15
  LEGACY_CIPHER_NAME = 'aes-256-cbc'
15
16
  DEFAULT_CIPHER_NAME = 'aes-256-cbc'
16
17
  FILE_TYPE = 'encrypted_hash_vault'
17
- CONTENT_KEYS = %i[label username password url description].freeze
18
18
  FILE_KEYS = %w[version type cipher data].sort.freeze
19
- private_constant :LEGACY_CIPHER_NAME, :DEFAULT_CIPHER_NAME, :FILE_TYPE, :CONTENT_KEYS, :FILE_KEYS
19
+ private_constant :LEGACY_CIPHER_NAME, :DEFAULT_CIPHER_NAME, :FILE_TYPE, :FILE_KEYS
20
20
  def initialize(file:, password:)
21
+ super()
21
22
  Aspera.assert_type(file, String){'path to vault file'}
22
23
  @path = file
23
24
  @all_secrets = {}
@@ -38,9 +39,7 @@ module Aspera
38
39
  end
39
40
  # setting password also creates the cipher
40
41
  @cipher = cipher(password)
41
- if !vault_encrypted_data.nil?
42
- @all_secrets = YAML.load_stream(@cipher.decrypt(vault_encrypted_data)).first
43
- end
42
+ @all_secrets = YAML.load_stream(@cipher.decrypt(vault_encrypted_data)).first if !vault_encrypted_data.nil?
44
43
  end
45
44
 
46
45
  def info
@@ -63,12 +62,7 @@ module Aspera
63
62
  # set a secret
64
63
  # @param options [Hash] with keys :label, :username, :password, :url, :description
65
64
  def set(options)
66
- Aspera.assert_type(options, Hash){'options'}
67
- unsupported = options.keys - CONTENT_KEYS
68
- Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
69
- options.each_pair do |k, v|
70
- Aspera.assert_type(v, String){k.to_s}
71
- end
65
+ validate_set(options)
72
66
  label = options.delete(:label)
73
67
  raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
74
68
  @all_secrets[label] = options.symbolize_keys
@@ -6,6 +6,11 @@ module Aspera
6
6
  class Factory
7
7
  LIST = %i[file system vault].freeze
8
8
  class << self
9
+ # Create a vault instance
10
+ # @param info [Hash] vault options
11
+ # @param name [String] name of the vault
12
+ # @param folder [String] folder to store the vault (if needed)
13
+ # @param password [String] password to open the vault
9
14
  def create(info, name, folder, password)
10
15
  Aspera.assert_type(info, Hash)
11
16
  Aspera.assert(info.values.all?(String)){'vault info shall have only string values'}
@@ -14,7 +19,7 @@ module Aspera
14
19
  Aspera.assert_values(vault_type, LIST.map(&:to_s)){'vault.type'}
15
20
  case vault_type
16
21
  when 'file'
17
- info[:file] ||= 'vault.bin'
22
+ info[:file] = name || 'vault.bin'
18
23
  info[:file] = File.join(folder, info[:file]) unless File.absolute_path?(info[:file])
19
24
  Aspera.assert(!password.nil?){'please provide password'}
20
25
  info[:password] = password
@@ -22,15 +27,16 @@ module Aspera
22
27
  require 'aspera/keychain/encrypted_hash'
23
28
  @vault = Keychain::EncryptedHash.new(**info)
24
29
  when 'system'
25
- case Environment.os
30
+ case Environment.instance.os
26
31
  when Environment::OS_MACOS
27
32
  info[:name] ||= name
28
33
  @vault = Keychain::MacosSystem.new(**info)
29
34
  else
30
- raise 'not implemented for this OS'
35
+ raise Error, 'not implemented for this OS'
31
36
  end
32
37
  when 'vault'
33
38
  require 'aspera/keychain/hashicorp_vault'
39
+ info[:token] ||= password
34
40
  @vault = Keychain::HashicorpVault.new(**info)
35
41
  else Aspera.error_unexpected_value(vault_type)
36
42
  end
@@ -3,17 +3,19 @@
3
3
  require 'aspera/environment'
4
4
  require 'aspera/log'
5
5
  require 'aspera/assert'
6
+ require 'aspera/keychain/base'
6
7
  require 'vault'
7
8
 
8
9
  module Aspera
9
10
  module Keychain
10
11
  # Manage secrets in a Hashicorp Vault
11
- class HashicorpVault
12
- SECRET_PATH = 'secret/data/'
12
+ class HashicorpVault < Base
13
+ STORE_PATH = 'secret/data/'
13
14
 
14
- private_constant :SECRET_PATH
15
+ private_constant :STORE_PATH
15
16
 
16
17
  def initialize(url:, token:)
18
+ super()
17
19
  Vault.configure do |config|
18
20
  config.address = url
19
21
  config.token = token
@@ -28,7 +30,7 @@ module Aspera
28
30
  end
29
31
 
30
32
  def list
31
- metadata_path = SECRET_PATH.sub('/data/', '/metadata/')
33
+ metadata_path = STORE_PATH.sub('/data/', '/metadata/')
32
34
  return Vault.logical.list(metadata_path).filter_map do |label|
33
35
  get(label: label).merge(label: label)
34
36
  end
@@ -37,6 +39,7 @@ module Aspera
37
39
  # Set a secret
38
40
  # @param options [Hash] with keys :label, :username, :password, :url, :description
39
41
  def set(options)
42
+ validate_set(options)
40
43
  label = options.fetch(:label)
41
44
  data = {
42
45
  username: options[:username],
@@ -51,7 +54,7 @@ module Aspera
51
54
  secret = Vault.logical.read(path(label))
52
55
  if secret.nil?
53
56
  raise "Secret '#{label}' not found" if exception
54
- return nil
57
+ return
55
58
  end
56
59
  return secret.data[:data]
57
60
  end
@@ -64,7 +67,7 @@ module Aspera
64
67
  private
65
68
 
66
69
  def path(label)
67
- "#{SECRET_PATH}#{label}"
70
+ "#{STORE_PATH}#{label}"
68
71
  end
69
72
  end
70
73
  end
@@ -5,6 +5,7 @@ require 'aspera/cli/info'
5
5
  require 'aspera/log'
6
6
  require 'aspera/assert'
7
7
  require 'aspera/environment'
8
+ require 'aspera/keychain/base'
8
9
 
9
10
  # enhance the gem to support other key chains
10
11
  module Aspera
@@ -37,13 +38,13 @@ module Aspera
37
38
  getpass: :g
38
39
  }.freeze
39
40
  class << self
40
- def execute(command, options=nil, supported=nil, last_opt=nil)
41
+ def execute(command, options = nil, supported = nil, last_opt = nil)
41
42
  url = options&.delete(:url)
42
43
  if !url.nil?
43
44
  uri = URI.parse(url)
44
45
  Aspera.assert(uri.scheme.eql?('https')){'only https'}
45
46
  options[:protocol] = 'htps' # cspell: disable-line
46
- raise 'host required in URL' if uri.host.nil?
47
+ raise Error, 'host required in URL' if uri.host.nil?
47
48
  options[:server] = uri.host
48
49
  options[:path] = uri.path unless ['', '/'].include?(uri.path)
49
50
  options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
@@ -71,8 +72,8 @@ module Aspera
71
72
  key_chains(execute('login-keychain')).first
72
73
  end
73
74
 
74
- def list(options={})
75
- Aspera.assert_values(options[:domain], DOMAINS, exception_class: ArgumentError){'domain'} unless options[:domain].nil?
75
+ def list(options = {})
76
+ Aspera.assert_values(options[:domain], DOMAINS, type: ArgumentError){'domain'} unless options[:domain].nil?
76
77
  key_chains(execute('list-keychains', options, LIST_OPTIONS))
77
78
  end
78
79
 
@@ -122,9 +123,9 @@ module Aspera
122
123
  end
123
124
  end
124
125
 
125
- class MacosSystem
126
- OPTIONS = %i[label username password url description].freeze
126
+ class MacosSystem < Base
127
127
  def initialize(name: nil)
128
+ super()
128
129
  @keychain_name = name.nil? ? 'default keychain' : name
129
130
  @keychain = name.nil? ? MacosSecurity::Keychain.default : MacosSecurity::Keychain.by_name(name)
130
131
  raise "no such keychain #{name}" if @keychain.nil?
@@ -138,16 +139,15 @@ module Aspera
138
139
 
139
140
  def list
140
141
  # the only way to list is `dump-keychain` which triggers security alert
141
- raise 'list not implemented, use macos keychain app'
142
+ raise Error, 'list not implemented, use macos keychain app'
142
143
  end
143
144
 
144
145
  def set(options)
145
- Aspera.assert_type(options, Hash){'options'}
146
- unsupported = options.keys - OPTIONS
147
- Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}, use #{OPTIONS.join(', ')}"}
146
+ validate_set(options)
148
147
  @keychain.password(
149
148
  :add, :generic, service: options[:label],
150
- account: options[:username] || 'none', password: options[:password], comment: options[:description])
149
+ account: options[:username] || 'none', password: options[:password], comment: options[:description]
150
+ )
151
151
  end
152
152
 
153
153
  def get(options)
@@ -155,7 +155,7 @@ module Aspera
155
155
  unsupported = options.keys - %i[label]
156
156
  Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
157
157
  info = @keychain.password(:find, :generic, label: options[:label])
158
- raise 'not found' if info.nil?
158
+ raise Error, 'not found' if info.nil?
159
159
  result = options.clone
160
160
  result[:secret] = info['password']
161
161
  result[:description] = info['icmt'] # cspell: disable-line
@@ -166,7 +166,7 @@ module Aspera
166
166
  Aspera.assert_type(options, Hash){'options'}
167
167
  unsupported = options.keys - %i[label]
168
168
  Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
169
- raise 'delete not implemented, use macos keychain app'
169
+ raise Error, 'delete not implemented, use macos keychain app'
170
170
  end
171
171
  end
172
172
  end
data/lib/aspera/log.rb CHANGED
@@ -17,12 +17,17 @@ $VERBOSE = nil
17
17
  class Logger
18
18
  # Two additionnal trace levels
19
19
  TRACE_MAX = 2
20
+
20
21
  # Add custom level to logger severity, below debug level
21
22
  module Severity
22
23
  1.upto(TRACE_MAX).each{ |level| const_set(:"TRACE#{level}", - level)}
23
24
  end
24
- # Quick access to label
25
+
26
+ # Hash : key: log level int, value: uppercase log level label
25
27
  SEVERITY_LABEL = Severity.constants.each_with_object({}){ |name, hash| hash[Severity.const_get(name)] = name}
28
+
29
+ # @param severity [Integer] Log severity as int
30
+ # @return [String] Log severity upper case label
26
31
  def format_severity(severity)
27
32
  SEVERITY_LABEL[severity] || 'ANY'
28
33
  end
@@ -51,28 +56,45 @@ module Aspera
51
56
 
52
57
  # Where logs are sent to
53
58
  LOG_TYPES = %i[stderr stdout syslog].freeze
54
- @@format = :json # rubocop:disable Style/ClassVars
59
+
60
+ # Levels are :trace2,:trace1,:debug,:info,:warn,:error,fatal,:unknown
61
+ LEVELS = Logger::Severity.constants.sort{ |a, b| Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{ |c| c.downcase.to_sym}.freeze
62
+
55
63
  # Class methods
56
64
  class << self
57
- # levels are :debug,:info,:warn,:error,fatal,:unknown
58
- def levels; Logger::Severity.constants.sort{ |a, b| Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{ |c| c.downcase.to_sym}; end
65
+ # Applies the provided list of string decoration (colors) to the value.
66
+ # @param value [String] Value to enhance.
67
+ # @param colors [Array(Symbol)] List of decorations
68
+ def apply_colors(value, colors)
69
+ colors.inject(value){ |s, c| s.send(c)}
70
+ end
59
71
 
60
- # get the logger object of singleton
72
+ # Get the logger object of singleton
61
73
  def log; instance.logger; end
62
74
 
63
- # dump object suitable for Log.log.debug
64
- # @param name string or symbol
65
- # @param format either pp or json format
66
- def dump(name, object)
67
- result =
68
- case @@format
75
+ # Dump object (`Hash`) to log using specified level
76
+ #
77
+ # @param name [String, Symbol] Name of object dumped
78
+ # @param object [Hash, nil] Data to dump
79
+ # @param level [Symbol] Debug level
80
+ # @param block [Proc, nil] Give computed object
81
+ def dump(name, object = nil, level: :debug)
82
+ return unless instance.logger.send(:"#{level}?")
83
+ object = yield if block_given?
84
+ instance.logger.send(level, obj_dump(name, object))
85
+ end
86
+
87
+ # @return [String] Dump of object
88
+ def obj_dump(name, object)
89
+ dump_text =
90
+ case instance.dump_format
69
91
  when :json
70
92
  JSON.pretty_generate(object) rescue PP.pp(object, +'')
71
93
  when :ruby
72
94
  PP.pp(object, +'')
73
- else error_unexpected_value(@@format){'dump format'}
95
+ else error_unexpected_value(instance.dump_format){'dump format'}
74
96
  end
75
- "#{name.to_s.green} (#{@@format})=\n#{result}"
97
+ "#{name.to_s.green} (#{instance.dump_format})=\n#{dump_text}"
76
98
  end
77
99
 
78
100
  # Capture the output of $stderr and log it at debug level
@@ -87,14 +109,36 @@ module Aspera
87
109
  end
88
110
 
89
111
  attr_reader :logger_type, :logger
90
- attr_writer :program_name
112
+ attr_accessor :dump_format
113
+
114
+ def program_name=(value)
115
+ @program_name = value
116
+ self.logger_type = @logger_type
117
+ end
91
118
 
92
119
  # Set log level of underlying logger given symbol level
120
+ # @param new_level [Symbol] One of LEVELS
93
121
  def level=(new_level)
94
122
  @logger.level = Logger::Severity.const_get(new_level.to_sym.upcase)
95
123
  end
96
124
 
125
+ def formatter=(formatter)
126
+ if formatter.is_a?(String)
127
+ raise Error, "Unknown formatter #{formatter}, use one of: #{FORMATTERS.keys.join(', ')}" unless FORMATTERS.key?(formatter.to_sym)
128
+ formatter = FORMATTERS[formatter.to_sym]
129
+ elsif !formatter.respond_to?(:call) && !formatter.is_a?(Logger::Formatter)
130
+ raise Error, 'Formatter must be a String, a Logger::Formatter or a Proc'
131
+ end
132
+ # Update formatter with password hiding
133
+ @logger.formatter = SecretHider.instance.log_formatter(formatter)
134
+ end
135
+
136
+ def formatter
137
+ @logger.formatter
138
+ end
139
+
97
140
  # Get symbol of debug level of underlying logger
141
+ # @return [Symbol] One of LEVELS
98
142
  def level
99
143
  Logger::Severity.constants.each do |name|
100
144
  return name.downcase.to_sym if @logger.level.eql?(Logger::Severity.const_get(name))
@@ -109,9 +153,9 @@ module Aspera
109
153
  current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
110
154
  case new_log_type
111
155
  when :stderr
112
- @logger = Logger.new($stderr)
156
+ @logger = Logger.new($stderr, progname: @program_name, formatter: DEFAULT_FORMATTER)
113
157
  when :stdout
114
- @logger = Logger.new($stdout)
158
+ @logger = Logger.new($stdout, progname: @program_name, formatter: DEFAULT_FORMATTER)
115
159
  when :syslog
116
160
  require 'syslog/logger'
117
161
  # the syslog class automatically creates methods from the severity names
@@ -122,13 +166,14 @@ module Aspera
122
166
  Logger::Severity.constants.each do |severity|
123
167
  Syslog::Logger.make_methods(severity.downcase)
124
168
  end
169
+ # Use `local2` facility, like other Aspera components
125
170
  @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
126
171
  else error_unexpected_value(new_log_type){"log type (#{LOG_TYPES.join(', ')})"}
127
172
  end
128
173
  @logger.level = current_severity_integer
129
174
  @logger_type = new_log_type
130
- # Update formatter with password hiding
131
- @logger.formatter = SecretHider.log_formatter(@logger.formatter)
175
+ # add secret hider to default logger
176
+ self.formatter = @logger.formatter
132
177
  end
133
178
 
134
179
  private
@@ -136,8 +181,35 @@ module Aspera
136
181
  def initialize
137
182
  @logger = nil
138
183
  @program_name = 'aspera'
184
+ @dump_format = :json
185
+ @logger_type = :stderr
139
186
  # This sets @logger and @logger_type (self needed to call method instead of local var)
140
- self.logger_type = :stderr
187
+ self.logger_type = @logger_type
141
188
  end
189
+
190
+ # Define decoration of levels
191
+ LVL_DECO = {
192
+ TRACE2: %i{dim},
193
+ TRACE1: %i{blue},
194
+ DEBUG: %i{cyan},
195
+ INFO: %i{green},
196
+ WARN: %i{bg_brown black},
197
+ ERROR: %i{bg_red blink},
198
+ FATAL: %i{magenta},
199
+ UNKNOWN: %i{blink}
200
+ }.freeze
201
+
202
+ # Short levels with color
203
+ LVL_COLOR = LVL_DECO.map{ |k, v| [k, apply_colors("#{k[..2]}#{k[-1]}", v)]}.to_h.freeze
204
+
205
+ DEFAULT_FORMATTER = ->(s, _d, _p, m){"#{LVL_COLOR[s]} #{m}\n"}
206
+
207
+ # pre-defined formatters
208
+ FORMATTERS = {
209
+ standard: Logger::Formatter.new,
210
+ default: DEFAULT_FORMATTER
211
+ }.freeze
212
+
213
+ private_constant :LVL_DECO, :LVL_COLOR, :DEFAULT_FORMATTER, :FORMATTERS
142
214
  end
143
215
  end
data/lib/aspera/nagios.rb CHANGED
@@ -36,9 +36,8 @@ module Aspera
36
36
  # build message: if multiple components: concatenate
37
37
  # message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
38
38
  message = data
39
- .map{ |i| i['component']}
40
- .uniq
41
- .map{ |comp| comp + ':' + data.select{ |d| d['component'].eql?(comp)}.map{ |d| d['message']}.join(',')}
39
+ .group_by{ |d| d['component']}
40
+ .map{ |comp, items| "#{comp}:#{items.map{ |d| d['message']}.join(',')}"}
42
41
  .join(', ')
43
42
  .tr("\n", ' ')
44
43
  status = data.first['status'].upcase
@@ -58,8 +57,8 @@ module Aspera
58
57
  # compare remote time with local time
59
58
  def check_time_offset(remote_date, component)
60
59
  # check date if specified : 2015-10-13T07:32:01Z
61
- remote_time = DateTime.strptime(remote_date)
62
- diff_time = (remote_time - DateTime.now).abs
60
+ remote_time = Time.strptime(remote_date)
61
+ diff_time = (remote_time - Time.now).abs
63
62
  diff_rounded = diff_time.round(-2)
64
63
  Log.log.debug{"DATE: #{remote_date} #{remote_time} diff=#{diff_rounded}"}
65
64
  msg = "offset #{diff_rounded} sec"
@@ -79,7 +78,7 @@ module Aspera
79
78
 
80
79
  # translate for display
81
80
  def result
82
- raise 'missing result' if @data.empty?
81
+ Aspera.assert(!@data.empty?){'missing result'}
83
82
  {type: :object_list, data: @data.map{ |i| {'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
84
83
  end
85
84
  end
@@ -119,9 +119,10 @@ module Aspera
119
119
  checksum: nil,
120
120
  start_byte: 0,
121
121
  bytes_written: 26,
122
- session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'}]
122
+ session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'
123
+ }]
123
124
  }
124
- Log.log.trace2{Log.dump(:job, result)}
125
+ Log.dump(:job, result, level: :trace2)
125
126
  return result
126
127
  end
127
128
 
@@ -276,7 +277,8 @@ module Aspera
276
277
  token_encryption_key
277
278
  byok_enabled
278
279
  bandwidth_flow_network_rc_module
279
- file_checksum_type],
280
+ file_checksum_type
281
+ ],
280
282
  server: %w[
281
283
  activity_event_logging
282
284
  activity_file_event_logging
@@ -289,7 +291,8 @@ module Aspera
289
291
  discovery
290
292
  auto_delete
291
293
  allow
292
- deny]
294
+ deny
295
+ ]
293
296
  },
294
297
  capabilities: [
295
298
  {name: 'sync', value: true},
@@ -302,7 +305,8 @@ module Aspera
302
305
  {name: 'aej_version', value: '1.0'},
303
306
  {name: 'page', value: true},
304
307
  {name: 'file_id_version', value: '2.0'},
305
- {name: 'auto_delete', value: false}],
308
+ {name: 'auto_delete', value: false}
309
+ ],
306
310
  settings: [
307
311
  {name: 'content_protection_required', value: false},
308
312
  {name: 'content_protection_strong_pass_required', value: false},
@@ -310,7 +314,8 @@ module Aspera
310
314
  {name: 'ssh_fingerprint', value: nil},
311
315
  {name: 'wss_enabled', value: false},
312
316
  {name: 'wss_port', value: 443}
313
- ]})
317
+ ]
318
+ })
314
319
  when PATH_TRANSFERS
315
320
  set_json_response(request, response, @simulator.all_sessions)
316
321
  when PATH_ONE_TRANSFER
@@ -325,7 +330,7 @@ module Aspera
325
330
  response.status = code
326
331
  response['Content-Type'] = Rest::MIME_JSON
327
332
  response.body = json.to_json
328
- Log.log.trace1{Log.dump("response for #{request.request_method} #{request.path}", json)}
333
+ Log.log.trace1{Log.obj_dump("response for #{request.request_method} #{request.path}", json)}
329
334
  end
330
335
  end
331
336
  end
@@ -31,9 +31,11 @@ module Aspera
31
31
  cache_ids: nil,
32
32
  **rest_params
33
33
  )
34
- Aspera.assert(respond_to?(:create_token), 'create_token method must be defined', exception_class: InternalError)
34
+ Aspera.assert(respond_to?(:create_token), 'create_token method must be defined', type: InternalError)
35
35
  # this is the OAuth API
36
36
  @api = Rest.new(**rest_params)
37
+ @scope = nil
38
+ @token_cache_id = nil
37
39
  @path_token = path_token
38
40
  @token_field = token_field
39
41
  @client_id = client_id
@@ -54,7 +56,7 @@ module Aspera
54
56
  @token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
55
57
  end
56
58
 
57
- attr_reader :scope
59
+ attr_reader :scope, :api, :path_token
58
60
 
59
61
  # helper method to create token as per RFC
60
62
  def create_token_call(creation_params)
@@ -84,7 +86,7 @@ module Aspera
84
86
 
85
87
  # @return value suitable for Authorization header
86
88
  def authorization(**kwargs)
87
- return OAuth::Factory.bearer_build(token(**kwargs))
89
+ return OAuth::Factory.bearer_authorization(token(**kwargs))
88
90
  end
89
91
 
90
92
  # get an OAuth v2 token (generated, cached, refreshed)
@@ -12,32 +12,37 @@ module Aspera
12
12
 
13
13
  # a prefix for persistency of tokens (simplify garbage collect)
14
14
  PERSIST_CATEGORY_TOKEN = 'token'
15
- # prefix for bearer token when in header
16
- BEARER_PREFIX = 'Bearer '
15
+ # prefix for bearer authorization when in header
16
+ SPACE_BEARER_AUTH_SCHEME = 'Bearer '
17
17
  TOKEN_FIELD = 'access_token'
18
18
 
19
- private_constant :PERSIST_CATEGORY_TOKEN, :BEARER_PREFIX
19
+ private_constant :PERSIST_CATEGORY_TOKEN, :SPACE_BEARER_AUTH_SCHEME
20
20
 
21
21
  class << self
22
- def bearer_build(token)
23
- return "#{BEARER_PREFIX}#{token}"
22
+ # @param token [String] The token alone
23
+ # @return [String] Value suitable for Authorization header
24
+ def bearer_authorization(token)
25
+ return "#{SPACE_BEARER_AUTH_SCHEME}#{token}"
24
26
  end
25
27
 
26
- def bearer?(token)
27
- return token.start_with?(BEARER_PREFIX)
28
+ # @return true if the authorization contains a bearer token , i.e. auth scheme is bearer
29
+ def bearer_auth?(authorization)
30
+ return authorization.start_with?(SPACE_BEARER_AUTH_SCHEME)
28
31
  end
29
32
 
30
- def bearer_extract(token)
31
- Aspera.assert(bearer?(token)){'not a bearer token, wrong prefix'}
32
- return token[BEARER_PREFIX.length..-1]
33
+ # Extract only token from Authorization (remove scheme)
34
+ def bearer_token(authorization)
35
+ Aspera.assert(bearer_auth?(authorization)){'not a bearer token, wrong prefix scheme'}
36
+ return authorization[SPACE_BEARER_AUTH_SCHEME.length..-1]
33
37
  end
34
38
 
35
- # @return a cache identifier
39
+ # @return a unique cache identifier
36
40
  def cache_id(url, creator_class, *params)
37
41
  return IdGenerator.from_list([
38
42
  PERSIST_CATEGORY_TOKEN,
39
43
  url,
40
- Factory.class_to_id(creator_class)] +
44
+ Factory.class_to_id(creator_class)
45
+ ] +
41
46
  params)
42
47
  end
43
48
 
@@ -54,6 +59,7 @@ module Aspera
54
59
  @persist = nil
55
60
  # token creation methods
56
61
  @token_type_classes = {}
62
+ # list of lambda
57
63
  @decoders = []
58
64
  # default parameters, others can be added by handlers
59
65
  @parameters = {
@@ -80,7 +86,7 @@ module Aspera
80
86
  Log.log.debug('Not using persistency')
81
87
  # create NULL persistency class
82
88
  @persist = Class.new do
83
- def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Layout/EmptyLineBetweenDefs, Style/Semicolon, Layout/LineLength
89
+ def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Style/Semicolon
84
90
  end.new
85
91
  end
86
92
  return @persist
@@ -107,17 +113,17 @@ module Aspera
107
113
  # @return [Hash] token internal information , including Date object for `expiration_date`
108
114
  def get_token_info(id)
109
115
  token_raw_string = persist_mgr.get(id)
110
- return nil if token_raw_string.nil?
116
+ return if token_raw_string.nil?
111
117
  token_data = JSON.parse(token_raw_string)
112
118
  Aspera.assert_type(token_data, Hash)
113
119
  decoded_token = decode_token(token_data[TOKEN_FIELD])
114
120
  info = {data: token_data}
115
- Log.log.debug{Log.dump('decoded_token', decoded_token)}
121
+ Log.dump(:decoded_token, decoded_token)
116
122
  if decoded_token.is_a?(Hash)
117
123
  info[:decoded] = decoded_token
118
124
  # TODO: move date decoding to token decoder ?
119
125
  expiration_date =
120
- if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
126
+ if decoded_token['expires_at'].is_a?(String) then Time.parse(decoded_token['expires_at']).to_time
121
127
  elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
122
128
  end
123
129
  unless expiration_date.nil?
@@ -140,7 +146,7 @@ module Aspera
140
146
  result = decoder.call(token) rescue nil
141
147
  return result unless result.nil?
142
148
  end
143
- return nil
149
+ return
144
150
  end
145
151
 
146
152
  # register a token creation method
@@ -164,6 +170,6 @@ module Aspera
164
170
  end
165
171
  end
166
172
  # JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
167
- Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
173
+ Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon
168
174
  end
169
175
  end
@@ -3,6 +3,7 @@
3
3
  require 'aspera/oauth/base'
4
4
  require 'aspera/assert'
5
5
  require 'securerandom'
6
+ require 'openssl'
6
7
  module Aspera
7
8
  module OAuth
8
9
  # remove 5 minutes to account for time offset between client and server (TODO: configurable?)
@@ -13,6 +14,17 @@ module Aspera
13
14
  # https://tools.ietf.org/html/rfc7523
14
15
  # https://tools.ietf.org/html/rfc7519
15
16
  class Jwt < Base
17
+ class << self
18
+ def generate_rsa_private_key(path:, length: DEFAULT_PRIV_KEY_LENGTH)
19
+ priv_key = OpenSSL::PKey::RSA.new(length)
20
+ File.write(path, priv_key.to_s)
21
+ File.write("#{path}.pub", priv_key.public_key.to_s)
22
+ Environment.restrict_file_access(path)
23
+ Environment.restrict_file_access("#{path}.pub")
24
+ nil
25
+ end
26
+ end
27
+ DEFAULT_PRIV_KEY_LENGTH = 4096
16
28
  GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
17
29
  # @param private_key_obj private key object
18
30
  # @param payload payload to be included in the JWT
@@ -45,7 +57,7 @@ module Aspera
45
57
  iat: seconds_since_epoch - OAuth::Factory.instance.parameters[:jwt_accepted_offset_sec] + 1, # issued at
46
58
  jti: SecureRandom.uuid # JWT id
47
59
  }.merge(@additional_payload)
48
- Log.log.debug{Log.dump(:jwt_payload, jwt_payload)}
60
+ Log.dump(:jwt_payload, jwt_payload)
49
61
  Log.log.debug{"private=[#{@private_key_obj}]"}
50
62
  assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
51
63
  Log.log.debug{"assertion=[#{assertion}]"}