aspera-cli 4.23.0 → 4.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +32 -1
- data/CONTRIBUTING.md +86 -29
- data/README.md +1651 -856
- data/bin/ascli +2 -1
- data/bin/asession +4 -4
- data/lib/aspera/agent/base.rb +4 -0
- data/lib/aspera/agent/connect.rb +20 -18
- data/lib/aspera/agent/desktop.rb +14 -11
- data/lib/aspera/agent/direct.rb +39 -31
- data/lib/aspera/agent/httpgw.rb +2 -2
- data/lib/aspera/agent/node.rb +9 -11
- data/lib/aspera/agent/transferd.rb +18 -11
- data/lib/aspera/api/aoc.rb +44 -31
- data/lib/aspera/api/cos_node.rb +7 -5
- data/lib/aspera/api/httpgw.rb +15 -18
- data/lib/aspera/api/node.rb +104 -22
- data/lib/aspera/ascmd.rb +22 -16
- data/lib/aspera/ascp/installation.rb +37 -40
- data/lib/aspera/ascp/management.rb +5 -4
- data/lib/aspera/assert.rb +54 -23
- data/lib/aspera/cli/basic_auth_plugin.rb +8 -7
- data/lib/aspera/cli/error.rb +1 -1
- data/lib/aspera/cli/extended_value.rb +28 -29
- data/lib/aspera/cli/formatter.rb +191 -168
- data/lib/aspera/cli/hints.rb +29 -3
- data/lib/aspera/cli/main.rb +138 -107
- data/lib/aspera/cli/manager.rb +50 -30
- data/lib/aspera/cli/plugin.rb +148 -77
- data/lib/aspera/cli/plugin_factory.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +189 -70
- data/lib/aspera/cli/plugins/ats.rb +15 -13
- data/lib/aspera/cli/plugins/config.rb +86 -213
- data/lib/aspera/cli/plugins/console.rb +49 -18
- data/lib/aspera/cli/plugins/cos.rb +4 -4
- data/lib/aspera/cli/plugins/faspex.rb +45 -51
- data/lib/aspera/cli/plugins/faspex5.rb +162 -163
- data/lib/aspera/cli/plugins/faspio.rb +6 -5
- data/lib/aspera/cli/plugins/httpgw.rb +2 -2
- data/lib/aspera/cli/plugins/node.rb +144 -162
- data/lib/aspera/cli/plugins/orchestrator.rb +10 -14
- data/lib/aspera/cli/plugins/preview.rb +26 -29
- data/lib/aspera/cli/plugins/server.rb +28 -28
- data/lib/aspera/cli/plugins/shares.rb +40 -28
- data/lib/aspera/cli/sync_actions.rb +101 -80
- data/lib/aspera/cli/transfer_agent.rb +51 -50
- data/lib/aspera/cli/transfer_progress.rb +29 -20
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +160 -0
- data/lib/aspera/colors.rb +13 -8
- data/lib/aspera/command_line_builder.rb +28 -22
- data/lib/aspera/command_line_converter.rb +31 -0
- data/lib/aspera/environment.rb +144 -101
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +3 -2
- data/lib/aspera/hash_ext.rb +1 -1
- data/lib/aspera/id_generator.rb +10 -10
- data/lib/aspera/keychain/base.rb +18 -0
- data/lib/aspera/keychain/encrypted_hash.rb +6 -12
- data/lib/aspera/keychain/factory.rb +9 -3
- data/lib/aspera/keychain/hashicorp_vault.rb +9 -6
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/log.rb +69 -20
- data/lib/aspera/nagios.rb +5 -6
- data/lib/aspera/node_simulator.rb +12 -7
- data/lib/aspera/oauth/base.rb +5 -3
- data/lib/aspera/oauth/factory.rb +24 -18
- data/lib/aspera/oauth/jwt.rb +13 -1
- data/lib/aspera/oauth/url_json.rb +3 -3
- data/lib/aspera/oauth/web.rb +5 -3
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +4 -3
- data/lib/aspera/preview/generator.rb +25 -12
- data/lib/aspera/preview/terminal.rb +10 -7
- data/lib/aspera/preview/utils.rb +11 -9
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/other.rb +2 -2
- data/lib/aspera/products/transferd.rb +8 -6
- data/lib/aspera/proxy_auto_config.rb +1 -1
- data/lib/aspera/rest.rb +29 -22
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/resumer.rb +1 -1
- data/lib/aspera/secret_hider.rb +46 -40
- data/lib/aspera/ssh.rb +13 -3
- data/lib/aspera/sync/args.schema.yaml +102 -0
- data/lib/aspera/sync/conf.schema.yaml +701 -0
- data/lib/aspera/sync/database.rb +83 -0
- data/lib/aspera/{transfer/sync.rb → sync/operations.rb} +132 -65
- data/lib/aspera/temp_file_manager.rb +3 -2
- data/lib/aspera/transfer/error.rb +1 -1
- data/lib/aspera/transfer/error_info.rb +1 -2
- data/lib/aspera/transfer/faux_file.rb +11 -10
- data/lib/aspera/transfer/parameters.rb +6 -5
- data/lib/aspera/transfer/spec.rb +15 -1
- data/lib/aspera/transfer/spec.schema.yaml +316 -293
- data/lib/aspera/transfer/spec_doc.rb +34 -16
- data/lib/aspera/transfer/uri.rb +5 -5
- data/lib/aspera/uri_reader.rb +14 -10
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +15 -13
- metadata.gz.sig +2 -2
- data/lib/aspera/transfer/async_conf.schema.yaml +0 -716
- data/lib/aspera/transfer/convert.rb +0 -29
- data/lib/aspera/transfer/sync_instance.schema.yaml +0 -20
- data/lib/aspera/transfer/sync_session.schema.yaml +0 -86
@@ -4,20 +4,21 @@ require 'aspera/hash_ext'
|
|
4
4
|
require 'aspera/environment'
|
5
5
|
require 'aspera/log'
|
6
6
|
require 'aspera/assert'
|
7
|
+
require 'aspera/keychain/base'
|
7
8
|
require 'symmetric_encryption/core'
|
8
9
|
require 'yaml'
|
9
10
|
|
10
11
|
module Aspera
|
11
12
|
module Keychain
|
12
13
|
# Manage secrets in a simple Hash
|
13
|
-
class EncryptedHash
|
14
|
+
class EncryptedHash < Base
|
14
15
|
LEGACY_CIPHER_NAME = 'aes-256-cbc'
|
15
16
|
DEFAULT_CIPHER_NAME = 'aes-256-cbc'
|
16
17
|
FILE_TYPE = 'encrypted_hash_vault'
|
17
|
-
CONTENT_KEYS = %i[label username password url description].freeze
|
18
18
|
FILE_KEYS = %w[version type cipher data].sort.freeze
|
19
|
-
private_constant :LEGACY_CIPHER_NAME, :DEFAULT_CIPHER_NAME, :FILE_TYPE, :
|
19
|
+
private_constant :LEGACY_CIPHER_NAME, :DEFAULT_CIPHER_NAME, :FILE_TYPE, :FILE_KEYS
|
20
20
|
def initialize(file:, password:)
|
21
|
+
super()
|
21
22
|
Aspera.assert_type(file, String){'path to vault file'}
|
22
23
|
@path = file
|
23
24
|
@all_secrets = {}
|
@@ -38,9 +39,7 @@ module Aspera
|
|
38
39
|
end
|
39
40
|
# setting password also creates the cipher
|
40
41
|
@cipher = cipher(password)
|
41
|
-
if !vault_encrypted_data.nil?
|
42
|
-
@all_secrets = YAML.load_stream(@cipher.decrypt(vault_encrypted_data)).first
|
43
|
-
end
|
42
|
+
@all_secrets = YAML.load_stream(@cipher.decrypt(vault_encrypted_data)).first if !vault_encrypted_data.nil?
|
44
43
|
end
|
45
44
|
|
46
45
|
def info
|
@@ -63,12 +62,7 @@ module Aspera
|
|
63
62
|
# set a secret
|
64
63
|
# @param options [Hash] with keys :label, :username, :password, :url, :description
|
65
64
|
def set(options)
|
66
|
-
|
67
|
-
unsupported = options.keys - CONTENT_KEYS
|
68
|
-
Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
69
|
-
options.each_pair do |k, v|
|
70
|
-
Aspera.assert_type(v, String){k.to_s}
|
71
|
-
end
|
65
|
+
validate_set(options)
|
72
66
|
label = options.delete(:label)
|
73
67
|
raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
|
74
68
|
@all_secrets[label] = options.symbolize_keys
|
@@ -6,6 +6,11 @@ module Aspera
|
|
6
6
|
class Factory
|
7
7
|
LIST = %i[file system vault].freeze
|
8
8
|
class << self
|
9
|
+
# Create a vault instance
|
10
|
+
# @param info [Hash] vault options
|
11
|
+
# @param name [String] name of the vault
|
12
|
+
# @param folder [String] folder to store the vault (if needed)
|
13
|
+
# @param password [String] password to open the vault
|
9
14
|
def create(info, name, folder, password)
|
10
15
|
Aspera.assert_type(info, Hash)
|
11
16
|
Aspera.assert(info.values.all?(String)){'vault info shall have only string values'}
|
@@ -14,7 +19,7 @@ module Aspera
|
|
14
19
|
Aspera.assert_values(vault_type, LIST.map(&:to_s)){'vault.type'}
|
15
20
|
case vault_type
|
16
21
|
when 'file'
|
17
|
-
info[:file]
|
22
|
+
info[:file] = name || 'vault.bin'
|
18
23
|
info[:file] = File.join(folder, info[:file]) unless File.absolute_path?(info[:file])
|
19
24
|
Aspera.assert(!password.nil?){'please provide password'}
|
20
25
|
info[:password] = password
|
@@ -22,15 +27,16 @@ module Aspera
|
|
22
27
|
require 'aspera/keychain/encrypted_hash'
|
23
28
|
@vault = Keychain::EncryptedHash.new(**info)
|
24
29
|
when 'system'
|
25
|
-
case Environment.os
|
30
|
+
case Environment.instance.os
|
26
31
|
when Environment::OS_MACOS
|
27
32
|
info[:name] ||= name
|
28
33
|
@vault = Keychain::MacosSystem.new(**info)
|
29
34
|
else
|
30
|
-
raise 'not implemented for this OS'
|
35
|
+
raise Error, 'not implemented for this OS'
|
31
36
|
end
|
32
37
|
when 'vault'
|
33
38
|
require 'aspera/keychain/hashicorp_vault'
|
39
|
+
info[:token] ||= password
|
34
40
|
@vault = Keychain::HashicorpVault.new(**info)
|
35
41
|
else Aspera.error_unexpected_value(vault_type)
|
36
42
|
end
|
@@ -3,17 +3,19 @@
|
|
3
3
|
require 'aspera/environment'
|
4
4
|
require 'aspera/log'
|
5
5
|
require 'aspera/assert'
|
6
|
+
require 'aspera/keychain/base'
|
6
7
|
require 'vault'
|
7
8
|
|
8
9
|
module Aspera
|
9
10
|
module Keychain
|
10
11
|
# Manage secrets in a Hashicorp Vault
|
11
|
-
class HashicorpVault
|
12
|
-
|
12
|
+
class HashicorpVault < Base
|
13
|
+
STORE_PATH = 'secret/data/'
|
13
14
|
|
14
|
-
private_constant :
|
15
|
+
private_constant :STORE_PATH
|
15
16
|
|
16
17
|
def initialize(url:, token:)
|
18
|
+
super()
|
17
19
|
Vault.configure do |config|
|
18
20
|
config.address = url
|
19
21
|
config.token = token
|
@@ -28,7 +30,7 @@ module Aspera
|
|
28
30
|
end
|
29
31
|
|
30
32
|
def list
|
31
|
-
metadata_path =
|
33
|
+
metadata_path = STORE_PATH.sub('/data/', '/metadata/')
|
32
34
|
return Vault.logical.list(metadata_path).filter_map do |label|
|
33
35
|
get(label: label).merge(label: label)
|
34
36
|
end
|
@@ -37,6 +39,7 @@ module Aspera
|
|
37
39
|
# Set a secret
|
38
40
|
# @param options [Hash] with keys :label, :username, :password, :url, :description
|
39
41
|
def set(options)
|
42
|
+
validate_set(options)
|
40
43
|
label = options.fetch(:label)
|
41
44
|
data = {
|
42
45
|
username: options[:username],
|
@@ -51,7 +54,7 @@ module Aspera
|
|
51
54
|
secret = Vault.logical.read(path(label))
|
52
55
|
if secret.nil?
|
53
56
|
raise "Secret '#{label}' not found" if exception
|
54
|
-
return
|
57
|
+
return
|
55
58
|
end
|
56
59
|
return secret.data[:data]
|
57
60
|
end
|
@@ -64,7 +67,7 @@ module Aspera
|
|
64
67
|
private
|
65
68
|
|
66
69
|
def path(label)
|
67
|
-
"#{
|
70
|
+
"#{STORE_PATH}#{label}"
|
68
71
|
end
|
69
72
|
end
|
70
73
|
end
|
@@ -5,6 +5,7 @@ require 'aspera/cli/info'
|
|
5
5
|
require 'aspera/log'
|
6
6
|
require 'aspera/assert'
|
7
7
|
require 'aspera/environment'
|
8
|
+
require 'aspera/keychain/base'
|
8
9
|
|
9
10
|
# enhance the gem to support other key chains
|
10
11
|
module Aspera
|
@@ -37,13 +38,13 @@ module Aspera
|
|
37
38
|
getpass: :g
|
38
39
|
}.freeze
|
39
40
|
class << self
|
40
|
-
def execute(command, options=nil, supported=nil, last_opt=nil)
|
41
|
+
def execute(command, options = nil, supported = nil, last_opt = nil)
|
41
42
|
url = options&.delete(:url)
|
42
43
|
if !url.nil?
|
43
44
|
uri = URI.parse(url)
|
44
45
|
Aspera.assert(uri.scheme.eql?('https')){'only https'}
|
45
46
|
options[:protocol] = 'htps' # cspell: disable-line
|
46
|
-
raise 'host required in URL' if uri.host.nil?
|
47
|
+
raise Error, 'host required in URL' if uri.host.nil?
|
47
48
|
options[:server] = uri.host
|
48
49
|
options[:path] = uri.path unless ['', '/'].include?(uri.path)
|
49
50
|
options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
|
@@ -71,8 +72,8 @@ module Aspera
|
|
71
72
|
key_chains(execute('login-keychain')).first
|
72
73
|
end
|
73
74
|
|
74
|
-
def list(options={})
|
75
|
-
Aspera.assert_values(options[:domain], DOMAINS,
|
75
|
+
def list(options = {})
|
76
|
+
Aspera.assert_values(options[:domain], DOMAINS, type: ArgumentError){'domain'} unless options[:domain].nil?
|
76
77
|
key_chains(execute('list-keychains', options, LIST_OPTIONS))
|
77
78
|
end
|
78
79
|
|
@@ -122,9 +123,9 @@ module Aspera
|
|
122
123
|
end
|
123
124
|
end
|
124
125
|
|
125
|
-
class MacosSystem
|
126
|
-
OPTIONS = %i[label username password url description].freeze
|
126
|
+
class MacosSystem < Base
|
127
127
|
def initialize(name: nil)
|
128
|
+
super()
|
128
129
|
@keychain_name = name.nil? ? 'default keychain' : name
|
129
130
|
@keychain = name.nil? ? MacosSecurity::Keychain.default : MacosSecurity::Keychain.by_name(name)
|
130
131
|
raise "no such keychain #{name}" if @keychain.nil?
|
@@ -138,16 +139,15 @@ module Aspera
|
|
138
139
|
|
139
140
|
def list
|
140
141
|
# the only way to list is `dump-keychain` which triggers security alert
|
141
|
-
raise 'list not implemented, use macos keychain app'
|
142
|
+
raise Error, 'list not implemented, use macos keychain app'
|
142
143
|
end
|
143
144
|
|
144
145
|
def set(options)
|
145
|
-
|
146
|
-
unsupported = options.keys - OPTIONS
|
147
|
-
Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}, use #{OPTIONS.join(', ')}"}
|
146
|
+
validate_set(options)
|
148
147
|
@keychain.password(
|
149
148
|
:add, :generic, service: options[:label],
|
150
|
-
account: options[:username] || 'none', password: options[:password], comment: options[:description]
|
149
|
+
account: options[:username] || 'none', password: options[:password], comment: options[:description]
|
150
|
+
)
|
151
151
|
end
|
152
152
|
|
153
153
|
def get(options)
|
@@ -155,7 +155,7 @@ module Aspera
|
|
155
155
|
unsupported = options.keys - %i[label]
|
156
156
|
Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
157
157
|
info = @keychain.password(:find, :generic, label: options[:label])
|
158
|
-
raise 'not found' if info.nil?
|
158
|
+
raise Error, 'not found' if info.nil?
|
159
159
|
result = options.clone
|
160
160
|
result[:secret] = info['password']
|
161
161
|
result[:description] = info['icmt'] # cspell: disable-line
|
@@ -166,7 +166,7 @@ module Aspera
|
|
166
166
|
Aspera.assert_type(options, Hash){'options'}
|
167
167
|
unsupported = options.keys - %i[label]
|
168
168
|
Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
169
|
-
raise 'delete not implemented, use macos keychain app'
|
169
|
+
raise Error, 'delete not implemented, use macos keychain app'
|
170
170
|
end
|
171
171
|
end
|
172
172
|
end
|
data/lib/aspera/log.rb
CHANGED
@@ -51,28 +51,54 @@ module Aspera
|
|
51
51
|
|
52
52
|
# Where logs are sent to
|
53
53
|
LOG_TYPES = %i[stderr stdout syslog].freeze
|
54
|
-
|
54
|
+
DEFAULT_FORMATTER = ->(s, _d, _p, m){"#{Log.color_level(s)} #{m}\n"}
|
55
|
+
FORMATTERS = {
|
56
|
+
standard: Logger::Formatter.new,
|
57
|
+
default: DEFAULT_FORMATTER
|
58
|
+
}.freeze
|
55
59
|
# Class methods
|
56
60
|
class << self
|
61
|
+
def color_level(level)
|
62
|
+
case level
|
63
|
+
when :TRACE2 then 'TR2'.dim
|
64
|
+
when :TRACE1 then 'TR1'.blue
|
65
|
+
when :DEBUG then 'DBG'.cyan
|
66
|
+
when :INFO then 'INF'.green
|
67
|
+
when :WARN then 'WRN'.bg_brown.black
|
68
|
+
when :ERROR then 'ERR'.bg_red.blink
|
69
|
+
when :FATAL then 'FTL'.magenta
|
70
|
+
when :UNKNOWN then 'UKN'.blink
|
71
|
+
else Aspera.error_unexpected_value(level){'log level'}
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
57
75
|
# levels are :debug,:info,:warn,:error,fatal,:unknown
|
58
76
|
def levels; Logger::Severity.constants.sort{ |a, b| Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{ |c| c.downcase.to_sym}; end
|
59
77
|
|
60
78
|
# get the logger object of singleton
|
61
79
|
def log; instance.logger; end
|
62
80
|
|
63
|
-
#
|
64
|
-
#
|
65
|
-
# @param
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
81
|
+
# Dump object (`Hash`) using specified level
|
82
|
+
#
|
83
|
+
# @param name [String, Symbol] Name of object dumped
|
84
|
+
# @param object [Hash, nil] Data to dump
|
85
|
+
# @param level [Symbol] Debug level
|
86
|
+
# @param block [Proc, nil] Give computed object
|
87
|
+
def dump(name, object = nil, level: :debug)
|
88
|
+
return unless instance.logger.send(:"#{level}?")
|
89
|
+
object = yield if block_given?
|
90
|
+
instance.logger.send(level, obj_dump(name, object))
|
91
|
+
end
|
92
|
+
|
93
|
+
def obj_dump(name, object)
|
94
|
+
dump_text = case instance.dump_format
|
95
|
+
when :json
|
96
|
+
JSON.pretty_generate(object) rescue PP.pp(object, +'')
|
97
|
+
when :ruby
|
98
|
+
PP.pp(object, +'')
|
99
|
+
else error_unexpected_value(instance.dump_format){'dump format'}
|
100
|
+
end
|
101
|
+
"#{name.to_s.green} (#{instance.dump_format})=\n#{dump_text}"
|
76
102
|
end
|
77
103
|
|
78
104
|
# Capture the output of $stderr and log it at debug level
|
@@ -87,13 +113,33 @@ module Aspera
|
|
87
113
|
end
|
88
114
|
|
89
115
|
attr_reader :logger_type, :logger
|
90
|
-
|
116
|
+
attr_accessor :dump_format
|
117
|
+
|
118
|
+
def program_name=(value)
|
119
|
+
@program_name = value
|
120
|
+
self.logger_type = @logger_type
|
121
|
+
end
|
91
122
|
|
92
123
|
# Set log level of underlying logger given symbol level
|
93
124
|
def level=(new_level)
|
94
125
|
@logger.level = Logger::Severity.const_get(new_level.to_sym.upcase)
|
95
126
|
end
|
96
127
|
|
128
|
+
def formatter=(formatter)
|
129
|
+
if formatter.is_a?(String)
|
130
|
+
raise Error, "Unknown formatter #{formatter}, use one of: #{FORMATTERS.keys.join(', ')}" unless FORMATTERS.key?(formatter.to_sym)
|
131
|
+
formatter = FORMATTERS[formatter.to_sym]
|
132
|
+
elsif !formatter.respond_to?(:call) && !formatter.is_a?(Logger::Formatter)
|
133
|
+
raise Error, 'Formatter must be a String, a Logger::Formatter or a Proc'
|
134
|
+
end
|
135
|
+
# Update formatter with password hiding
|
136
|
+
@logger.formatter = SecretHider.instance.log_formatter(formatter)
|
137
|
+
end
|
138
|
+
|
139
|
+
def formatter
|
140
|
+
@logger.formatter
|
141
|
+
end
|
142
|
+
|
97
143
|
# Get symbol of debug level of underlying logger
|
98
144
|
def level
|
99
145
|
Logger::Severity.constants.each do |name|
|
@@ -109,9 +155,9 @@ module Aspera
|
|
109
155
|
current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
|
110
156
|
case new_log_type
|
111
157
|
when :stderr
|
112
|
-
@logger = Logger.new($stderr)
|
158
|
+
@logger = Logger.new($stderr, progname: @program_name, formatter: DEFAULT_FORMATTER)
|
113
159
|
when :stdout
|
114
|
-
@logger = Logger.new($stdout)
|
160
|
+
@logger = Logger.new($stdout, progname: @program_name, formatter: DEFAULT_FORMATTER)
|
115
161
|
when :syslog
|
116
162
|
require 'syslog/logger'
|
117
163
|
# the syslog class automatically creates methods from the severity names
|
@@ -122,13 +168,14 @@ module Aspera
|
|
122
168
|
Logger::Severity.constants.each do |severity|
|
123
169
|
Syslog::Logger.make_methods(severity.downcase)
|
124
170
|
end
|
171
|
+
# Use `local2` facility, like other Aspera components
|
125
172
|
@logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
|
126
173
|
else error_unexpected_value(new_log_type){"log type (#{LOG_TYPES.join(', ')})"}
|
127
174
|
end
|
128
175
|
@logger.level = current_severity_integer
|
129
176
|
@logger_type = new_log_type
|
130
|
-
#
|
131
|
-
|
177
|
+
# add secret hider to default logger
|
178
|
+
self.formatter = @logger.formatter
|
132
179
|
end
|
133
180
|
|
134
181
|
private
|
@@ -136,8 +183,10 @@ module Aspera
|
|
136
183
|
def initialize
|
137
184
|
@logger = nil
|
138
185
|
@program_name = 'aspera'
|
186
|
+
@dump_format = :json
|
187
|
+
@logger_type = :stderr
|
139
188
|
# This sets @logger and @logger_type (self needed to call method instead of local var)
|
140
|
-
self.logger_type =
|
189
|
+
self.logger_type = @logger_type
|
141
190
|
end
|
142
191
|
end
|
143
192
|
end
|
data/lib/aspera/nagios.rb
CHANGED
@@ -36,9 +36,8 @@ module Aspera
|
|
36
36
|
# build message: if multiple components: concatenate
|
37
37
|
# message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
38
38
|
message = data
|
39
|
-
.
|
40
|
-
.
|
41
|
-
.map{ |comp| comp + ':' + data.select{ |d| d['component'].eql?(comp)}.map{ |d| d['message']}.join(',')}
|
39
|
+
.group_by{ |d| d['component']}
|
40
|
+
.map{ |comp, items| "#{comp}:#{items.map{ |d| d['message']}.join(',')}"}
|
42
41
|
.join(', ')
|
43
42
|
.tr("\n", ' ')
|
44
43
|
status = data.first['status'].upcase
|
@@ -58,8 +57,8 @@ module Aspera
|
|
58
57
|
# compare remote time with local time
|
59
58
|
def check_time_offset(remote_date, component)
|
60
59
|
# check date if specified : 2015-10-13T07:32:01Z
|
61
|
-
remote_time =
|
62
|
-
diff_time = (remote_time -
|
60
|
+
remote_time = Time.strptime(remote_date)
|
61
|
+
diff_time = (remote_time - Time.now).abs
|
63
62
|
diff_rounded = diff_time.round(-2)
|
64
63
|
Log.log.debug{"DATE: #{remote_date} #{remote_time} diff=#{diff_rounded}"}
|
65
64
|
msg = "offset #{diff_rounded} sec"
|
@@ -79,7 +78,7 @@ module Aspera
|
|
79
78
|
|
80
79
|
# translate for display
|
81
80
|
def result
|
82
|
-
|
81
|
+
Aspera.assert(!@data.empty?){'missing result'}
|
83
82
|
{type: :object_list, data: @data.map{ |i| {'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
|
84
83
|
end
|
85
84
|
end
|
@@ -119,9 +119,10 @@ module Aspera
|
|
119
119
|
checksum: nil,
|
120
120
|
start_byte: 0,
|
121
121
|
bytes_written: 26,
|
122
|
-
session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'
|
122
|
+
session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'
|
123
|
+
}]
|
123
124
|
}
|
124
|
-
Log.
|
125
|
+
Log.dump(:job, result, level: :trace2)
|
125
126
|
return result
|
126
127
|
end
|
127
128
|
|
@@ -276,7 +277,8 @@ module Aspera
|
|
276
277
|
token_encryption_key
|
277
278
|
byok_enabled
|
278
279
|
bandwidth_flow_network_rc_module
|
279
|
-
file_checksum_type
|
280
|
+
file_checksum_type
|
281
|
+
],
|
280
282
|
server: %w[
|
281
283
|
activity_event_logging
|
282
284
|
activity_file_event_logging
|
@@ -289,7 +291,8 @@ module Aspera
|
|
289
291
|
discovery
|
290
292
|
auto_delete
|
291
293
|
allow
|
292
|
-
deny
|
294
|
+
deny
|
295
|
+
]
|
293
296
|
},
|
294
297
|
capabilities: [
|
295
298
|
{name: 'sync', value: true},
|
@@ -302,7 +305,8 @@ module Aspera
|
|
302
305
|
{name: 'aej_version', value: '1.0'},
|
303
306
|
{name: 'page', value: true},
|
304
307
|
{name: 'file_id_version', value: '2.0'},
|
305
|
-
{name: 'auto_delete', value: false}
|
308
|
+
{name: 'auto_delete', value: false}
|
309
|
+
],
|
306
310
|
settings: [
|
307
311
|
{name: 'content_protection_required', value: false},
|
308
312
|
{name: 'content_protection_strong_pass_required', value: false},
|
@@ -310,7 +314,8 @@ module Aspera
|
|
310
314
|
{name: 'ssh_fingerprint', value: nil},
|
311
315
|
{name: 'wss_enabled', value: false},
|
312
316
|
{name: 'wss_port', value: 443}
|
313
|
-
]
|
317
|
+
]
|
318
|
+
})
|
314
319
|
when PATH_TRANSFERS
|
315
320
|
set_json_response(request, response, @simulator.all_sessions)
|
316
321
|
when PATH_ONE_TRANSFER
|
@@ -325,7 +330,7 @@ module Aspera
|
|
325
330
|
response.status = code
|
326
331
|
response['Content-Type'] = Rest::MIME_JSON
|
327
332
|
response.body = json.to_json
|
328
|
-
Log.log.trace1{Log.
|
333
|
+
Log.log.trace1{Log.obj_dump("response for #{request.request_method} #{request.path}", json)}
|
329
334
|
end
|
330
335
|
end
|
331
336
|
end
|
data/lib/aspera/oauth/base.rb
CHANGED
@@ -31,9 +31,11 @@ module Aspera
|
|
31
31
|
cache_ids: nil,
|
32
32
|
**rest_params
|
33
33
|
)
|
34
|
-
Aspera.assert(respond_to?(:create_token), 'create_token method must be defined',
|
34
|
+
Aspera.assert(respond_to?(:create_token), 'create_token method must be defined', type: InternalError)
|
35
35
|
# this is the OAuth API
|
36
36
|
@api = Rest.new(**rest_params)
|
37
|
+
@scope = nil
|
38
|
+
@token_cache_id = nil
|
37
39
|
@path_token = path_token
|
38
40
|
@token_field = token_field
|
39
41
|
@client_id = client_id
|
@@ -54,7 +56,7 @@ module Aspera
|
|
54
56
|
@token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
|
55
57
|
end
|
56
58
|
|
57
|
-
attr_reader :scope
|
59
|
+
attr_reader :scope, :api, :path_token
|
58
60
|
|
59
61
|
# helper method to create token as per RFC
|
60
62
|
def create_token_call(creation_params)
|
@@ -84,7 +86,7 @@ module Aspera
|
|
84
86
|
|
85
87
|
# @return value suitable for Authorization header
|
86
88
|
def authorization(**kwargs)
|
87
|
-
return OAuth::Factory.
|
89
|
+
return OAuth::Factory.bearer_authorization(token(**kwargs))
|
88
90
|
end
|
89
91
|
|
90
92
|
# get an OAuth v2 token (generated, cached, refreshed)
|
data/lib/aspera/oauth/factory.rb
CHANGED
@@ -12,32 +12,37 @@ module Aspera
|
|
12
12
|
|
13
13
|
# a prefix for persistency of tokens (simplify garbage collect)
|
14
14
|
PERSIST_CATEGORY_TOKEN = 'token'
|
15
|
-
# prefix for bearer
|
16
|
-
|
15
|
+
# prefix for bearer authorization when in header
|
16
|
+
SPACE_BEARER_AUTH_SCHEME = 'Bearer '
|
17
17
|
TOKEN_FIELD = 'access_token'
|
18
18
|
|
19
|
-
private_constant :PERSIST_CATEGORY_TOKEN, :
|
19
|
+
private_constant :PERSIST_CATEGORY_TOKEN, :SPACE_BEARER_AUTH_SCHEME
|
20
20
|
|
21
21
|
class << self
|
22
|
-
|
23
|
-
|
22
|
+
# @param token [String] The token alone
|
23
|
+
# @return [String] Value suitable for Authorization header
|
24
|
+
def bearer_authorization(token)
|
25
|
+
return "#{SPACE_BEARER_AUTH_SCHEME}#{token}"
|
24
26
|
end
|
25
27
|
|
26
|
-
|
27
|
-
|
28
|
+
# @return true if the authorization contains a bearer token , i.e. auth scheme is bearer
|
29
|
+
def bearer_auth?(authorization)
|
30
|
+
return authorization.start_with?(SPACE_BEARER_AUTH_SCHEME)
|
28
31
|
end
|
29
32
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
+
# Extract only token from Authorization (remove scheme)
|
34
|
+
def bearer_token(authorization)
|
35
|
+
Aspera.assert(bearer_auth?(authorization)){'not a bearer token, wrong prefix scheme'}
|
36
|
+
return authorization[SPACE_BEARER_AUTH_SCHEME.length..-1]
|
33
37
|
end
|
34
38
|
|
35
|
-
# @return a cache identifier
|
39
|
+
# @return a unique cache identifier
|
36
40
|
def cache_id(url, creator_class, *params)
|
37
41
|
return IdGenerator.from_list([
|
38
42
|
PERSIST_CATEGORY_TOKEN,
|
39
43
|
url,
|
40
|
-
Factory.class_to_id(creator_class)
|
44
|
+
Factory.class_to_id(creator_class)
|
45
|
+
] +
|
41
46
|
params)
|
42
47
|
end
|
43
48
|
|
@@ -54,6 +59,7 @@ module Aspera
|
|
54
59
|
@persist = nil
|
55
60
|
# token creation methods
|
56
61
|
@token_type_classes = {}
|
62
|
+
# list of lambda
|
57
63
|
@decoders = []
|
58
64
|
# default parameters, others can be added by handlers
|
59
65
|
@parameters = {
|
@@ -80,7 +86,7 @@ module Aspera
|
|
80
86
|
Log.log.debug('Not using persistency')
|
81
87
|
# create NULL persistency class
|
82
88
|
@persist = Class.new do
|
83
|
-
def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable
|
89
|
+
def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Style/Semicolon
|
84
90
|
end.new
|
85
91
|
end
|
86
92
|
return @persist
|
@@ -107,17 +113,17 @@ module Aspera
|
|
107
113
|
# @return [Hash] token internal information , including Date object for `expiration_date`
|
108
114
|
def get_token_info(id)
|
109
115
|
token_raw_string = persist_mgr.get(id)
|
110
|
-
return
|
116
|
+
return if token_raw_string.nil?
|
111
117
|
token_data = JSON.parse(token_raw_string)
|
112
118
|
Aspera.assert_type(token_data, Hash)
|
113
119
|
decoded_token = decode_token(token_data[TOKEN_FIELD])
|
114
120
|
info = {data: token_data}
|
115
|
-
Log.
|
121
|
+
Log.dump(:decoded_token, decoded_token)
|
116
122
|
if decoded_token.is_a?(Hash)
|
117
123
|
info[:decoded] = decoded_token
|
118
124
|
# TODO: move date decoding to token decoder ?
|
119
125
|
expiration_date =
|
120
|
-
if decoded_token['expires_at'].is_a?(String) then
|
126
|
+
if decoded_token['expires_at'].is_a?(String) then Time.parse(decoded_token['expires_at']).to_time
|
121
127
|
elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
|
122
128
|
end
|
123
129
|
unless expiration_date.nil?
|
@@ -140,7 +146,7 @@ module Aspera
|
|
140
146
|
result = decoder.call(token) rescue nil
|
141
147
|
return result unless result.nil?
|
142
148
|
end
|
143
|
-
return
|
149
|
+
return
|
144
150
|
end
|
145
151
|
|
146
152
|
# register a token creation method
|
@@ -164,6 +170,6 @@ module Aspera
|
|
164
170
|
end
|
165
171
|
end
|
166
172
|
# JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
|
167
|
-
Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon
|
173
|
+
Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon
|
168
174
|
end
|
169
175
|
end
|
data/lib/aspera/oauth/jwt.rb
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
require 'aspera/oauth/base'
|
4
4
|
require 'aspera/assert'
|
5
5
|
require 'securerandom'
|
6
|
+
require 'openssl'
|
6
7
|
module Aspera
|
7
8
|
module OAuth
|
8
9
|
# remove 5 minutes to account for time offset between client and server (TODO: configurable?)
|
@@ -13,6 +14,17 @@ module Aspera
|
|
13
14
|
# https://tools.ietf.org/html/rfc7523
|
14
15
|
# https://tools.ietf.org/html/rfc7519
|
15
16
|
class Jwt < Base
|
17
|
+
class << self
|
18
|
+
def generate_rsa_private_key(path:, length: DEFAULT_PRIV_KEY_LENGTH)
|
19
|
+
priv_key = OpenSSL::PKey::RSA.new(length)
|
20
|
+
File.write(path, priv_key.to_s)
|
21
|
+
File.write("#{path}.pub", priv_key.public_key.to_s)
|
22
|
+
Environment.restrict_file_access(path)
|
23
|
+
Environment.restrict_file_access("#{path}.pub")
|
24
|
+
nil
|
25
|
+
end
|
26
|
+
end
|
27
|
+
DEFAULT_PRIV_KEY_LENGTH = 4096
|
16
28
|
GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
|
17
29
|
# @param private_key_obj private key object
|
18
30
|
# @param payload payload to be included in the JWT
|
@@ -45,7 +57,7 @@ module Aspera
|
|
45
57
|
iat: seconds_since_epoch - OAuth::Factory.instance.parameters[:jwt_accepted_offset_sec] + 1, # issued at
|
46
58
|
jti: SecureRandom.uuid # JWT id
|
47
59
|
}.merge(@additional_payload)
|
48
|
-
Log.
|
60
|
+
Log.dump(:jwt_payload, jwt_payload)
|
49
61
|
Log.log.debug{"private=[#{@private_key_obj}]"}
|
50
62
|
assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
|
51
63
|
Log.log.debug{"assertion=[#{assertion}]"}
|
@@ -19,10 +19,10 @@ module Aspera
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def create_token
|
22
|
-
|
22
|
+
api.call(
|
23
23
|
operation: 'POST',
|
24
|
-
subpath:
|
25
|
-
query: @query.merge(scope:
|
24
|
+
subpath: path_token,
|
25
|
+
query: @query.merge(scope: scope), # scope is here because it may change over time (node)
|
26
26
|
content_type: Rest::MIME_JSON,
|
27
27
|
body: @body,
|
28
28
|
headers: {'Accept' => Rest::MIME_JSON}
|