aspera-cli 4.21.2 → 4.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +1 -1
- data/CHANGELOG.md +34 -16
- data/CONTRIBUTING.md +6 -10
- data/README.md +805 -574
- data/examples/get_proto_file.rb +1 -1
- data/lib/aspera/agent/base.rb +9 -5
- data/lib/aspera/agent/connect.rb +30 -28
- data/lib/aspera/agent/desktop.rb +29 -25
- data/lib/aspera/agent/direct.rb +137 -125
- data/lib/aspera/agent/httpgw.rb +22 -26
- data/lib/aspera/agent/node.rb +14 -11
- data/lib/aspera/agent/transferd.rb +6 -2
- data/lib/aspera/api/aoc.rb +6 -6
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/httpgw.rb +7 -3
- data/lib/aspera/api/node.rb +6 -4
- data/lib/aspera/ascmd.rb +3 -3
- data/lib/aspera/ascp/installation.rb +15 -16
- data/lib/aspera/ascp/management.rb +1 -1
- data/lib/aspera/assert.rb +11 -2
- data/lib/aspera/cli/error.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +38 -19
- data/lib/aspera/cli/formatter.rb +48 -48
- data/lib/aspera/cli/hints.rb +1 -1
- data/lib/aspera/cli/main.rb +190 -168
- data/lib/aspera/cli/manager.rb +15 -15
- data/lib/aspera/cli/plugin.rb +23 -20
- data/lib/aspera/cli/plugin_factory.rb +1 -1
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +144 -107
- data/lib/aspera/cli/plugins/ats.rb +19 -17
- data/lib/aspera/cli/plugins/config.rb +67 -83
- data/lib/aspera/cli/plugins/console.rb +5 -3
- data/lib/aspera/cli/plugins/faspex.rb +39 -35
- data/lib/aspera/cli/plugins/faspex5.rb +104 -80
- data/lib/aspera/cli/plugins/faspio.rb +13 -1
- data/lib/aspera/cli/plugins/httpgw.rb +13 -1
- data/lib/aspera/cli/plugins/node.rb +306 -179
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +6 -6
- data/lib/aspera/cli/plugins/shares.rb +5 -5
- data/lib/aspera/cli/sync_actions.rb +19 -18
- data/lib/aspera/cli/transfer_agent.rb +5 -5
- data/lib/aspera/cli/transfer_progress.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +116 -95
- data/lib/aspera/coverage.rb +4 -3
- data/lib/aspera/environment.rb +6 -6
- data/lib/aspera/faspex_gw.rb +14 -14
- data/lib/aspera/faspex_postproc.rb +7 -6
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +47 -34
- data/lib/aspera/keychain/factory.rb +41 -0
- data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
- data/lib/aspera/keychain/macos_security.rb +19 -11
- data/lib/aspera/log.rb +28 -34
- data/lib/aspera/nagios.rb +6 -6
- data/lib/aspera/node_simulator.rb +8 -8
- data/lib/aspera/oauth/base.rb +8 -6
- data/lib/aspera/oauth/factory.rb +5 -6
- data/lib/aspera/oauth/url_json.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +6 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +1 -1
- data/lib/aspera/preview/options.rb +16 -16
- data/lib/aspera/preview/terminal.rb +3 -3
- data/lib/aspera/preview/utils.rb +11 -13
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +2 -2
- data/lib/aspera/rest.rb +52 -43
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/secret_hider.rb +5 -5
- data/lib/aspera/ssh.rb +4 -4
- data/lib/aspera/transfer/convert.rb +29 -0
- data/lib/aspera/transfer/error_info.rb +66 -66
- data/lib/aspera/transfer/parameters.rb +13 -68
- data/lib/aspera/transfer/spec.rb +5 -6
- data/lib/aspera/transfer/spec.schema.yaml +753 -0
- data/lib/aspera/transfer/spec_doc.rb +62 -0
- data/lib/aspera/transfer/sync.rb +23 -72
- data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
- data/lib/aspera/transfer/uri.rb +6 -6
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +53 -44
- data.tar.gz.sig +1 -2
- metadata +37 -4
- metadata.gz.sig +0 -0
- data/examples/build_package.sh +0 -28
- data/lib/aspera/transfer/spec.yaml +0 -718
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Aspera
|
4
|
+
module Keychain
|
5
|
+
# Manage secrets in a Hashicorp Vault
|
6
|
+
class Factory
|
7
|
+
LIST = %i[file system vault].freeze
|
8
|
+
class << self
|
9
|
+
def create(info, name, folder, password)
|
10
|
+
Aspera.assert_type(info, Hash)
|
11
|
+
Aspera.assert(info.values.all?(String)){'vault info shall have only string values'}
|
12
|
+
info = info.symbolize_keys
|
13
|
+
vault_type = info.delete(:type)
|
14
|
+
Aspera.assert_values(vault_type, LIST.map(&:to_s)){'vault.type'}
|
15
|
+
case vault_type
|
16
|
+
when 'file'
|
17
|
+
info[:file] ||= 'vault.bin'
|
18
|
+
info[:file] = File.join(folder, info[:file]) unless File.absolute_path?(info[:file])
|
19
|
+
Aspera.assert(!password.nil?){'please provide password'}
|
20
|
+
info[:password] = password
|
21
|
+
# this module requires compilation, so it is optional
|
22
|
+
require 'aspera/keychain/encrypted_hash'
|
23
|
+
@vault = Keychain::EncryptedHash.new(**info)
|
24
|
+
when 'system'
|
25
|
+
case Environment.os
|
26
|
+
when Environment::OS_MACOS
|
27
|
+
info[:name] ||= name
|
28
|
+
@vault = Keychain::MacosSystem.new(**info)
|
29
|
+
else
|
30
|
+
raise 'not implemented for this OS'
|
31
|
+
end
|
32
|
+
when 'vault'
|
33
|
+
require 'aspera/keychain/hashicorp_vault'
|
34
|
+
@vault = Keychain::HashicorpVault.new(**info)
|
35
|
+
else Aspera.error_unexpected_value(vault_type)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/environment'
|
4
|
+
require 'aspera/log'
|
5
|
+
require 'aspera/assert'
|
6
|
+
require 'vault'
|
7
|
+
|
8
|
+
module Aspera
|
9
|
+
module Keychain
|
10
|
+
# Manage secrets in a Hashicorp Vault
|
11
|
+
class HashicorpVault
|
12
|
+
SECRET_PATH = 'secret/data/'
|
13
|
+
|
14
|
+
private_constant :SECRET_PATH
|
15
|
+
|
16
|
+
def initialize(url:, token:)
|
17
|
+
Vault.configure do |config|
|
18
|
+
config.address = url
|
19
|
+
config.token = token
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def info
|
24
|
+
{
|
25
|
+
url: Vault.address,
|
26
|
+
password: Vault.auth_token
|
27
|
+
}
|
28
|
+
end
|
29
|
+
|
30
|
+
def list
|
31
|
+
metadata_path = SECRET_PATH.sub('/data/', '/metadata/')
|
32
|
+
return Vault.logical.list(metadata_path).filter_map do |label|
|
33
|
+
get(label: label).merge(label: label)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# Set a secret
|
38
|
+
# @param options [Hash] with keys :label, :username, :password, :url, :description
|
39
|
+
def set(options)
|
40
|
+
label = options.fetch(:label)
|
41
|
+
data = {
|
42
|
+
username: options[:username],
|
43
|
+
password: options[:password],
|
44
|
+
url: options[:url],
|
45
|
+
description: options[:description]
|
46
|
+
}.compact
|
47
|
+
Vault.logical.write(path(label), data: data)
|
48
|
+
end
|
49
|
+
|
50
|
+
def get(label:, exception: true)
|
51
|
+
secret = Vault.logical.read(path(label))
|
52
|
+
if secret.nil?
|
53
|
+
raise "Secret '#{label}' not found" if exception
|
54
|
+
return nil
|
55
|
+
end
|
56
|
+
return secret.data[:data]
|
57
|
+
end
|
58
|
+
|
59
|
+
def delete(label:)
|
60
|
+
path = path(label)
|
61
|
+
Vault.logical.delete(path)
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
def path(label)
|
67
|
+
"#{SECRET_PATH}#{label}"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -60,7 +60,7 @@ module Aspera
|
|
60
60
|
end
|
61
61
|
|
62
62
|
def key_chains(output)
|
63
|
-
output.split("\n").collect
|
63
|
+
output.split("\n").collect{ |line| new(line.strip.gsub(/^"|"$/, ''))}
|
64
64
|
end
|
65
65
|
|
66
66
|
def default
|
@@ -77,7 +77,7 @@ module Aspera
|
|
77
77
|
end
|
78
78
|
|
79
79
|
def by_name(name)
|
80
|
-
list.find{|kc|kc.path.end_with?("/#{name}.keychain-db")}
|
80
|
+
list.find{ |kc| kc.path.end_with?("/#{name}.keychain-db")}
|
81
81
|
end
|
82
82
|
end
|
83
83
|
attr_reader :path
|
@@ -123,15 +123,28 @@ module Aspera
|
|
123
123
|
end
|
124
124
|
|
125
125
|
class MacosSystem
|
126
|
-
|
127
|
-
|
126
|
+
OPTIONS = %i[label username password url description].freeze
|
127
|
+
def initialize(name: nil)
|
128
|
+
@keychain_name = name.nil? ? 'default keychain' : name
|
129
|
+
@keychain = name.nil? ? MacosSecurity::Keychain.default : MacosSecurity::Keychain.by_name(name)
|
128
130
|
raise "no such keychain #{name}" if @keychain.nil?
|
129
131
|
end
|
130
132
|
|
133
|
+
def info
|
134
|
+
return {
|
135
|
+
keychain: @keychain_name
|
136
|
+
}
|
137
|
+
end
|
138
|
+
|
139
|
+
def list
|
140
|
+
# the only way to list is `dump-keychain` which triggers security alert
|
141
|
+
raise 'list not implemented, use macos keychain app'
|
142
|
+
end
|
143
|
+
|
131
144
|
def set(options)
|
132
145
|
Aspera.assert_type(options, Hash){'options'}
|
133
|
-
unsupported = options.keys -
|
134
|
-
Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
146
|
+
unsupported = options.keys - OPTIONS
|
147
|
+
Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}, use #{OPTIONS.join(', ')}"}
|
135
148
|
@keychain.password(
|
136
149
|
:add, :generic, service: options[:label],
|
137
150
|
account: options[:username] || 'none', password: options[:password], comment: options[:description])
|
@@ -149,11 +162,6 @@ module Aspera
|
|
149
162
|
return result
|
150
163
|
end
|
151
164
|
|
152
|
-
def list
|
153
|
-
# the only way to list is `dump-keychain` which triggers security alert
|
154
|
-
raise 'list not implemented, use macos keychain app'
|
155
|
-
end
|
156
|
-
|
157
165
|
def delete(options)
|
158
166
|
Aspera.assert_type(options, Hash){'options'}
|
159
167
|
unsupported = options.keys - %i[label]
|
data/lib/aspera/log.rb
CHANGED
@@ -9,57 +9,52 @@ require 'json'
|
|
9
9
|
require 'singleton'
|
10
10
|
require 'stringio'
|
11
11
|
|
12
|
+
# Ignore warnings
|
12
13
|
old_verbose = $VERBOSE
|
13
14
|
$VERBOSE = nil
|
14
15
|
|
15
|
-
#
|
16
|
+
# Extend Ruby logger with trace levels
|
16
17
|
class Logger
|
18
|
+
# Two additionnal trace levels
|
17
19
|
TRACE_MAX = 2
|
18
|
-
#
|
20
|
+
# Add custom level to logger severity, below debug level
|
19
21
|
module Severity
|
20
|
-
1.upto(TRACE_MAX).each
|
22
|
+
1.upto(TRACE_MAX).each{ |level| const_set(:"TRACE#{level}", - level)}
|
21
23
|
end
|
22
|
-
#
|
23
|
-
SEVERITY_LABEL = Severity.constants.each_with_object({})
|
24
|
+
# Quick access to label
|
25
|
+
SEVERITY_LABEL = Severity.constants.each_with_object({}){ |name, hash| hash[Severity.const_get(name)] = name}
|
24
26
|
def format_severity(severity)
|
25
27
|
SEVERITY_LABEL[severity] || 'ANY'
|
26
28
|
end
|
27
29
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
def #{str_level}?
|
38
|
-
level <= #{int_level}
|
39
|
-
end
|
40
|
-
|
41
|
-
def #{str_level}!
|
42
|
-
self.level = #{int_level}
|
43
|
-
end
|
44
|
-
EOM
|
30
|
+
class << self
|
31
|
+
# Define methods for a given log level
|
32
|
+
def make_methods(str_level)
|
33
|
+
int_level = ::Logger.const_get(str_level.upcase)
|
34
|
+
method_base = str_level.downcase
|
35
|
+
define_method(method_base, ->(message = nil, &block){add(int_level, message, &block)})
|
36
|
+
define_method("#{method_base}?", ->{level <= int_level})
|
37
|
+
define_method("#{method_base}!", ->{self.level = int_level})
|
38
|
+
end
|
45
39
|
end
|
46
|
-
#
|
47
|
-
Logger::Severity.constants.each
|
40
|
+
# Declare methods for all levels
|
41
|
+
Logger::Severity.constants.each{ |severity| make_methods(severity)}
|
48
42
|
end
|
49
43
|
|
44
|
+
# Restore warnings
|
50
45
|
$VERBOSE = old_verbose
|
51
46
|
|
52
47
|
module Aspera
|
53
48
|
# Singleton object for logging
|
54
49
|
class Log
|
55
50
|
include Singleton
|
56
|
-
#
|
51
|
+
# Where logs are sent to
|
57
52
|
LOG_TYPES = %i[stderr stdout syslog].freeze
|
58
53
|
@@format = :json # rubocop:disable Style/ClassVars
|
59
|
-
#
|
54
|
+
# Class methods
|
60
55
|
class << self
|
61
56
|
# levels are :debug,:info,:warn,:error,fatal,:unknown
|
62
|
-
def levels; Logger::Severity.constants.sort{|a, b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym}; end
|
57
|
+
def levels; Logger::Severity.constants.sort{ |a, b| Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{ |c| c.downcase.to_sym}; end
|
63
58
|
|
64
59
|
# get the logger object of singleton
|
65
60
|
def log; instance.logger; end
|
@@ -83,7 +78,7 @@ module Aspera
|
|
83
78
|
def capture_stderr
|
84
79
|
real_stderr = $stderr
|
85
80
|
$stderr = StringIO.new
|
86
|
-
yield
|
81
|
+
yield if block_given?
|
87
82
|
log.debug($stderr.string)
|
88
83
|
ensure
|
89
84
|
$stderr = real_stderr
|
@@ -93,12 +88,12 @@ module Aspera
|
|
93
88
|
attr_reader :logger_type, :logger
|
94
89
|
attr_writer :program_name
|
95
90
|
|
96
|
-
#
|
91
|
+
# Set log level of underlying logger given symbol level
|
97
92
|
def level=(new_level)
|
98
93
|
@logger.level = Logger::Severity.const_get(new_level.to_sym.upcase)
|
99
94
|
end
|
100
95
|
|
101
|
-
#
|
96
|
+
# Get symbol of debug level of underlying logger
|
102
97
|
def level
|
103
98
|
Logger::Severity.constants.each do |name|
|
104
99
|
return name.downcase.to_sym if @logger.level.eql?(Logger::Severity.const_get(name))
|
@@ -106,7 +101,7 @@ module Aspera
|
|
106
101
|
Aspera.error_unexpected_value(@logger.level){'log level'}
|
107
102
|
end
|
108
103
|
|
109
|
-
#
|
104
|
+
# Change underlying logger, but keep log level
|
110
105
|
def logger_type=(new_log_type)
|
111
106
|
current_severity_integer = @logger.level unless @logger.nil?
|
112
107
|
current_severity_integer = ENV.fetch('AS_LOG_LEVEL', nil) if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
|
@@ -131,7 +126,7 @@ module Aspera
|
|
131
126
|
end
|
132
127
|
@logger.level = current_severity_integer
|
133
128
|
@logger_type = new_log_type
|
134
|
-
#
|
129
|
+
# Update formatter with password hiding
|
135
130
|
@logger.formatter = SecretHider.log_formatter(@logger.formatter)
|
136
131
|
end
|
137
132
|
|
@@ -140,9 +135,8 @@ module Aspera
|
|
140
135
|
def initialize
|
141
136
|
@logger = nil
|
142
137
|
@program_name = 'aspera'
|
143
|
-
#
|
138
|
+
# This sets @logger and @logger_type (self needed to call method instead of local var)
|
144
139
|
self.logger_type = :stderr
|
145
|
-
raise 'error logger shall be defined' if @logger.nil?
|
146
140
|
end
|
147
141
|
end
|
148
142
|
end
|
data/lib/aspera/nagios.rb
CHANGED
@@ -17,7 +17,7 @@ module Aspera
|
|
17
17
|
# add methods to add nagios error levels, each take component name and message
|
18
18
|
LEVELS.each_index do |code|
|
19
19
|
name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
|
20
|
-
define_method(name){|comp, msg
|
20
|
+
define_method(name){ |comp, msg| @data.push({code: code, comp: comp, msg: msg})}
|
21
21
|
end
|
22
22
|
|
23
23
|
class << self
|
@@ -28,17 +28,17 @@ module Aspera
|
|
28
28
|
%w[status component message].each do |c|
|
29
29
|
Aspera.assert(data.first.key?(c)){"result must have #{c}"}
|
30
30
|
end
|
31
|
-
res_errors = data.reject{|s|s['status'].eql?('ok')}
|
31
|
+
res_errors = data.reject{ |s| s['status'].eql?('ok')}
|
32
32
|
# keep only errors in case of problem, other ok are assumed so
|
33
33
|
data = res_errors unless res_errors.empty?
|
34
34
|
# first is most critical
|
35
|
-
data.sort!{|a, b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
35
|
+
data.sort!{ |a, b| LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
36
36
|
# build message: if multiple components: concatenate
|
37
37
|
# message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
38
38
|
message = data
|
39
|
-
.map{|i|i['component']}
|
39
|
+
.map{ |i| i['component']}
|
40
40
|
.uniq
|
41
|
-
.map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
|
41
|
+
.map{ |comp| comp + ':' + data.select{ |d| d['component'].eql?(comp)}.map{ |d| d['message']}.join(',')}
|
42
42
|
.join(', ')
|
43
43
|
.tr("\n", ' ')
|
44
44
|
status = data.first['status'].upcase
|
@@ -80,7 +80,7 @@ module Aspera
|
|
80
80
|
# translate for display
|
81
81
|
def result
|
82
82
|
raise 'missing result' if @data.empty?
|
83
|
-
{type: :object_list, data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
|
83
|
+
{type: :object_list, data: @data.map{ |i| {'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
|
84
84
|
end
|
85
85
|
end
|
86
86
|
end
|
@@ -18,7 +18,7 @@ module Aspera
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def all_sessions
|
21
|
-
@agent.sessions.map
|
21
|
+
@agent.sessions.map{ |session| session[:job_id]}.uniq.each.map{ |job_id| job_to_transfer(job_id)}
|
22
22
|
end
|
23
23
|
|
24
24
|
# status: ('waiting', 'partially_completed', 'unknown', 'waiting(read error)',] 'running', 'completed', 'failed'
|
@@ -185,9 +185,9 @@ module Aspera
|
|
185
185
|
'size' => folder_stat.size,
|
186
186
|
'mtime' => folder_stat.mtime.utc.iso8601,
|
187
187
|
'permissions' => [
|
188
|
-
{
|
189
|
-
{
|
190
|
-
{
|
188
|
+
{'name' => 'view'},
|
189
|
+
{'name' => 'edit'},
|
190
|
+
{'name' => 'delete'}
|
191
191
|
]
|
192
192
|
},
|
193
193
|
'items' => []
|
@@ -208,9 +208,9 @@ module Aspera
|
|
208
208
|
'size' => item_stat.size,
|
209
209
|
'mtime' => item_stat.mtime.utc.iso8601,
|
210
210
|
'permissions' => [
|
211
|
-
{
|
212
|
-
{
|
213
|
-
{
|
211
|
+
{'name' => 'view'},
|
212
|
+
{'name' => 'edit'},
|
213
|
+
{'name' => 'delete'}
|
214
214
|
]
|
215
215
|
}
|
216
216
|
|
@@ -323,7 +323,7 @@ module Aspera
|
|
323
323
|
|
324
324
|
def set_json_response(request, response, json, code: 200)
|
325
325
|
response.status = code
|
326
|
-
response['Content-Type'] =
|
326
|
+
response['Content-Type'] = Rest::MIME_JSON
|
327
327
|
response.body = json.to_json
|
328
328
|
Log.log.trace1{Log.dump("response for #{request.request_method} #{request.path}", json)}
|
329
329
|
end
|
data/lib/aspera/oauth/base.rb
CHANGED
@@ -10,14 +10,15 @@ module Aspera
|
|
10
10
|
# OAuth 2 client for the REST client
|
11
11
|
# Generate bearer token
|
12
12
|
# Bearer tokens are cached in memory and in a file cache for later re-use
|
13
|
-
# https://tools.ietf.org/html/rfc6749
|
13
|
+
# OAuth 2.0 Authorization Framework: https://tools.ietf.org/html/rfc6749
|
14
|
+
# Bearer Token Usage: https://tools.ietf.org/html/rfc6750
|
14
15
|
class Base
|
15
16
|
# @param ** Parameters for REST
|
16
17
|
# @param client_id [String, nil]
|
17
18
|
# @param client_secret [String, nil]
|
18
19
|
# @param scope [String, nil]
|
19
20
|
# @param use_query [bool] Provide parameters in query instead of body
|
20
|
-
# @param path_token [String] API end point to create a token
|
21
|
+
# @param path_token [String] API end point to create a token from base URL
|
21
22
|
# @param token_field [String] Field in result that contains the token
|
22
23
|
# @param cache_ids [Array, nil] List of unique identifiers for cache id generation
|
23
24
|
def initialize(
|
@@ -56,7 +57,7 @@ module Aspera
|
|
56
57
|
# helper method to create token as per RFC
|
57
58
|
def create_token_call(creation_params)
|
58
59
|
Log.log.debug{'Generating a new token'.bg_green}
|
59
|
-
payload = {
|
60
|
+
payload = {content_type: Rest::MIME_WWW}
|
60
61
|
if @use_query
|
61
62
|
payload[:query] = creation_params
|
62
63
|
else
|
@@ -65,7 +66,7 @@ module Aspera
|
|
65
66
|
return @api.call(
|
66
67
|
operation: 'POST',
|
67
68
|
subpath: @path_token,
|
68
|
-
headers: {'Accept' =>
|
69
|
+
headers: {'Accept' => Rest::MIME_JSON},
|
69
70
|
**payload
|
70
71
|
)
|
71
72
|
end
|
@@ -96,12 +97,13 @@ module Aspera
|
|
96
97
|
unless token_info.nil?
|
97
98
|
token_data = token_info[:data]
|
98
99
|
# Optional optimization:
|
99
|
-
#
|
100
|
+
# Check if token is expired based on decoded content then force refresh if close enough
|
100
101
|
# might help in case the transfer agent cannot refresh himself
|
101
102
|
# `direct` agent is equipped with refresh code
|
102
103
|
# an API was already called, but failed, we need to regenerate or refresh
|
103
104
|
if refresh || token_info[:expired]
|
104
|
-
|
105
|
+
refresh_token = nil
|
106
|
+
if token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
|
105
107
|
# save possible refresh token, before deleting the cache
|
106
108
|
refresh_token = token_data['refresh_token']
|
107
109
|
end
|
data/lib/aspera/oauth/factory.rb
CHANGED
@@ -36,9 +36,8 @@ module Aspera
|
|
36
36
|
return IdGenerator.from_list([
|
37
37
|
PERSIST_CATEGORY_TOKEN,
|
38
38
|
url,
|
39
|
-
Factory.class_to_id(creator_class)
|
40
|
-
|
41
|
-
].flatten)
|
39
|
+
Factory.class_to_id(creator_class)] +
|
40
|
+
params)
|
42
41
|
end
|
43
42
|
|
44
43
|
# @return snake version of class name
|
@@ -111,7 +110,7 @@ module Aspera
|
|
111
110
|
token_data = JSON.parse(token_raw_string)
|
112
111
|
Aspera.assert_type(token_data, Hash)
|
113
112
|
decoded_token = decode_token(token_data[TOKEN_FIELD])
|
114
|
-
info = {
|
113
|
+
info = {data: token_data}
|
115
114
|
Log.log.debug{Log.dump('decoded_token', decoded_token)}
|
116
115
|
if decoded_token.is_a?(Hash)
|
117
116
|
info[:decoded] = decoded_token
|
@@ -159,11 +158,11 @@ module Aspera
|
|
159
158
|
Aspera.assert_type(parameters, Hash)
|
160
159
|
id = parameters[:grant_method]
|
161
160
|
Aspera.assert(@token_type_classes.key?(id)){"token grant method unknown: '#{id}'"}
|
162
|
-
create_parameters = parameters.reject
|
161
|
+
create_parameters = parameters.reject{ |k, _v| k.eql?(:grant_method)}
|
163
162
|
@token_type_classes[id].new(**create_parameters)
|
164
163
|
end
|
165
164
|
end
|
166
165
|
# JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
|
167
|
-
Factory.instance.register_decoder(lambda
|
166
|
+
Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
|
168
167
|
end
|
169
168
|
end
|
@@ -20,12 +20,12 @@ module Aspera
|
|
20
20
|
|
21
21
|
def create_token
|
22
22
|
@api.call(
|
23
|
-
operation:
|
24
|
-
subpath:
|
25
|
-
|
26
|
-
|
27
|
-
body:
|
28
|
-
|
23
|
+
operation: 'POST',
|
24
|
+
subpath: @path_token,
|
25
|
+
query: @query.merge(scope: @scope), # scope is here because it may change over time (node)
|
26
|
+
content_type: Rest::MIME_JSON,
|
27
|
+
body: @body,
|
28
|
+
headers: {'Accept' => Rest::MIME_JSON}
|
29
29
|
)
|
30
30
|
end
|
31
31
|
end
|
@@ -8,9 +8,9 @@ module Aspera
|
|
8
8
|
# Persist data on file system
|
9
9
|
class PersistencyActionOnce
|
10
10
|
DELETE_DEFAULT = lambda(&:empty?)
|
11
|
-
PARSE_DEFAULT = lambda
|
12
|
-
FORMAT_DEFAULT = lambda
|
13
|
-
MERGE_DEFAULT = lambda
|
11
|
+
PARSE_DEFAULT = lambda{ |t| JSON.parse(t)}
|
12
|
+
FORMAT_DEFAULT = lambda{ |h| JSON.generate(h)}
|
13
|
+
MERGE_DEFAULT = lambda{ |current, file| current.concat(file).uniq rescue current}
|
14
14
|
MANAGER_METHODS = %i[get put delete]
|
15
15
|
private_constant :DELETE_DEFAULT, :PARSE_DEFAULT, :FORMAT_DEFAULT, :MERGE_DEFAULT, :MANAGER_METHODS
|
16
16
|
|
@@ -22,7 +22,7 @@ module Aspera
|
|
22
22
|
# @param :format Optional dump method (default to JSON)
|
23
23
|
# @param :merge Optional merge data from file to current data
|
24
24
|
def initialize(manager:, data:, id:, delete: DELETE_DEFAULT, parse: PARSE_DEFAULT, format: FORMAT_DEFAULT, merge: MERGE_DEFAULT)
|
25
|
-
Aspera.assert(MANAGER_METHODS.all?{|i|manager.respond_to?(i)}){"Manager must answer to #{MANAGER_METHODS}"}
|
25
|
+
Aspera.assert(MANAGER_METHODS.all?{ |i| manager.respond_to?(i)}){"Manager must answer to #{MANAGER_METHODS}"}
|
26
26
|
Aspera.assert(!data.nil?)
|
27
27
|
Aspera.assert_type(id, String)
|
28
28
|
Aspera.assert(!id.empty?)
|
@@ -39,6 +39,7 @@ module Aspera
|
|
39
39
|
merge.call(@persisted_object, parse.call(value)) unless value.nil?
|
40
40
|
end
|
41
41
|
|
42
|
+
# Save persisted object on storage
|
42
43
|
def save
|
43
44
|
if @delete_condition.call(@persisted_object)
|
44
45
|
@manager.delete(@object_id)
|
@@ -47,6 +48,7 @@ module Aspera
|
|
47
48
|
end
|
48
49
|
end
|
49
50
|
|
51
|
+
# @return internal persisted object, in order to modify its content
|
50
52
|
def data
|
51
53
|
return @persisted_object
|
52
54
|
end
|
@@ -64,7 +64,7 @@ module Aspera
|
|
64
64
|
garbage_files = current_files(persist_category)
|
65
65
|
if !max_age_seconds.nil?
|
66
66
|
current_time = Time.now
|
67
|
-
garbage_files.select!
|
67
|
+
garbage_files.select!{ |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
|
68
68
|
end
|
69
69
|
garbage_files.each do |filepath|
|
70
70
|
File.delete(filepath)
|
@@ -79,7 +79,7 @@ module Aspera
|
|
79
79
|
end
|
80
80
|
|
81
81
|
def current_items(persist_category)
|
82
|
-
current_files(persist_category).each_with_object({})
|
82
|
+
current_files(persist_category).each_with_object({}){ |i, h| h[File.basename(i, FILE_SUFFIX)] = File.read(i)}
|
83
83
|
end
|
84
84
|
|
85
85
|
private
|
@@ -240,7 +240,7 @@ module Aspera
|
|
240
240
|
# text to png
|
241
241
|
def convert_plaintext_to_png
|
242
242
|
# get 100 first lines of text file
|
243
|
-
first_lines = File.open(@source_file_path){|f|Array.new(100){f.readline rescue ''}.join}
|
243
|
+
first_lines = File.open(@source_file_path){ |f| Array.new(100){f.readline rescue ''}.join}
|
244
244
|
Utils.external_command(:magick, [
|
245
245
|
'convert',
|
246
246
|
'-size', "#{@options.thumb_img_size}x#{@options.thumb_img_size}",
|
@@ -15,22 +15,22 @@ module Aspera
|
|
15
15
|
# iw/ih : input width or height
|
16
16
|
# -x : keep aspect ratio, having value a multiple of x
|
17
17
|
DESCRIPTIONS = [
|
18
|
-
{
|
19
|
-
{
|
20
|
-
{
|
21
|
-
{
|
22
|
-
{
|
23
|
-
{
|
24
|
-
{
|
25
|
-
{
|
26
|
-
{
|
27
|
-
{
|
28
|
-
{
|
29
|
-
{
|
30
|
-
{
|
31
|
-
{
|
32
|
-
{
|
33
|
-
{
|
18
|
+
{name: :max_size, default: 1 << 24, description: 'maximum size (in bytes) of preview file'},
|
19
|
+
{name: :thumb_vid_scale, default: "-1:'min(ih,100)'", description: 'png: video: size (ffmpeg scale argument)'},
|
20
|
+
{name: :thumb_vid_fraction, default: 0.1, description: 'png: video: time percent position of snapshot'},
|
21
|
+
{name: :thumb_img_size, default: 800, description: 'png: non-video: height (and width)'},
|
22
|
+
{name: :thumb_text_font, default: 'Courier', description: 'png: plaintext: font for text rendering: `magick identify -list font`'},
|
23
|
+
{name: :video_conversion, default: :reencode, description: 'mp4: method for preview generation', values: VIDEO_CONVERSION_METHODS},
|
24
|
+
{name: :video_png_conv, default: :fixed, description: 'mp4: method for thumbnail generation', values: VIDEO_THUMBNAIL_METHODS},
|
25
|
+
{name: :video_scale, default: "'min(iw,360)':-2", description: 'mp4: all: video scale (ffmpeg scale argument)'},
|
26
|
+
{name: :video_start_sec, default: 10, description: 'mp4: all: start offset (seconds) of video preview'},
|
27
|
+
{name: :reencode_ffmpeg, default: {}, description: 'mp4: reencode: options to ffmpeg'},
|
28
|
+
{name: :blend_keyframes, default: 30, description: 'mp4: blend: # key frames'},
|
29
|
+
{name: :blend_pauseframes, default: 3, description: 'mp4: blend: # pause frames'},
|
30
|
+
{name: :blend_transframes, default: 5, description: 'mp4: blend: # transition blend frames'},
|
31
|
+
{name: :blend_fps, default: 15, description: 'mp4: blend: frame per second'},
|
32
|
+
{name: :clips_count, default: 5, description: 'mp4: clips: number of clips'},
|
33
|
+
{name: :clips_length, default: 5, description: 'mp4: clips: length in seconds of each clips'}
|
34
34
|
].freeze
|
35
35
|
# add accessors
|
36
36
|
DESCRIPTIONS.each do |opt|
|
@@ -50,7 +50,7 @@ module Aspera
|
|
50
50
|
pixel_colors = []
|
51
51
|
image.each_pixel do |pixel, col, row|
|
52
52
|
pixel_rgb = [pixel.red, pixel.green, pixel.blue]
|
53
|
-
pixel_rgb = pixel_rgb.map
|
53
|
+
pixel_rgb = pixel_rgb.map{ |color| color >> shift_for_8_bit} unless shift_for_8_bit.eql?(0)
|
54
54
|
# init 2-dim array
|
55
55
|
pixel_colors[row] ||= []
|
56
56
|
pixel_colors[row][col] = pixel_rgb
|
@@ -82,7 +82,7 @@ module Aspera
|
|
82
82
|
size: blob.length
|
83
83
|
# width: image.columns,
|
84
84
|
# height: image.rows
|
85
|
-
}.map
|
85
|
+
}.map{ |k, v| "#{k}=#{v}"}.join(';')
|
86
86
|
# \a is BEL, \e is ESC : https://github.com/ruby/ruby/blob/master/doc/syntax/literals.rdoc#label-Strings
|
87
87
|
# escape sequence for iTerm2 image display
|
88
88
|
return "\e]1337;File=#{arguments}:#{Base64.encode64(blob)}\a"
|
@@ -91,7 +91,7 @@ module Aspera
|
|
91
91
|
# @return [Boolean] true if the terminal supports iTerm2 image display
|
92
92
|
def iterm_supported?
|
93
93
|
TERM_ENV_VARS.each do |env_var|
|
94
|
-
return true if ITERM_NAMES.any?
|
94
|
+
return true if ITERM_NAMES.any?{ |term| ENV[env_var]&.include?(term)}
|
95
95
|
end
|
96
96
|
false
|
97
97
|
end
|