aspera-cli 4.14.0 → 4.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +54 -3
- data/CONTRIBUTING.md +7 -7
- data/README.md +1457 -880
- data/bin/ascli +18 -9
- data/bin/asession +12 -14
- data/examples/proxy.pac +1 -1
- data/lib/aspera/aoc.rb +198 -127
- data/lib/aspera/ascmd.rb +24 -14
- data/lib/aspera/cli/basic_auth_plugin.rb +9 -6
- data/lib/aspera/cli/error.rb +17 -0
- data/lib/aspera/cli/extended_value.rb +47 -12
- data/lib/aspera/cli/formatter.rb +260 -171
- data/lib/aspera/cli/hints.rb +80 -0
- data/lib/aspera/cli/main.rb +101 -147
- data/lib/aspera/cli/manager.rb +160 -124
- data/lib/aspera/cli/plugin.rb +70 -59
- data/lib/aspera/cli/plugins/alee.rb +0 -1
- data/lib/aspera/cli/plugins/aoc.rb +239 -273
- data/lib/aspera/cli/plugins/ats.rb +8 -5
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +516 -375
- data/lib/aspera/cli/plugins/console.rb +40 -0
- data/lib/aspera/cli/plugins/cos.rb +4 -5
- data/lib/aspera/cli/plugins/faspex.rb +99 -84
- data/lib/aspera/cli/plugins/faspex5.rb +179 -148
- data/lib/aspera/cli/plugins/node.rb +219 -153
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -17
- data/lib/aspera/cli/plugins/preview.rb +46 -32
- data/lib/aspera/cli/plugins/server.rb +57 -17
- data/lib/aspera/cli/plugins/shares.rb +34 -12
- data/lib/aspera/cli/sync_actions.rb +68 -0
- data/lib/aspera/cli/transfer_agent.rb +45 -55
- data/lib/aspera/cli/transfer_progress.rb +74 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -1
- data/lib/aspera/command_line_builder.rb +14 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/environment.rb +17 -6
- data/lib/aspera/fasp/agent_aspera.rb +126 -0
- data/lib/aspera/fasp/agent_base.rb +31 -77
- data/lib/aspera/fasp/agent_connect.rb +21 -22
- data/lib/aspera/fasp/agent_direct.rb +88 -102
- data/lib/aspera/fasp/agent_httpgw.rb +196 -192
- data/lib/aspera/fasp/agent_node.rb +41 -34
- data/lib/aspera/fasp/agent_trsdk.rb +75 -34
- data/lib/aspera/fasp/error_info.rb +2 -2
- data/lib/aspera/fasp/faux_file.rb +52 -0
- data/lib/aspera/fasp/installation.rb +43 -184
- data/lib/aspera/fasp/management.rb +244 -0
- data/lib/aspera/fasp/parameters.rb +59 -26
- data/lib/aspera/fasp/parameters.yaml +75 -8
- data/lib/aspera/fasp/products.rb +162 -0
- data/lib/aspera/fasp/transfer_spec.rb +1 -1
- data/lib/aspera/fasp/uri.rb +4 -4
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +49 -0
- data/lib/aspera/line_logger.rb +23 -0
- data/lib/aspera/log.rb +57 -16
- data/lib/aspera/node.rb +97 -14
- data/lib/aspera/oauth.rb +36 -18
- data/lib/aspera/open_application.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +4 -2
- data/lib/aspera/preview/generator.rb +22 -35
- data/lib/aspera/preview/options.rb +2 -0
- data/lib/aspera/preview/terminal.rb +24 -13
- data/lib/aspera/preview/utils.rb +19 -26
- data/lib/aspera/rest.rb +103 -72
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +15 -14
- data/lib/aspera/rest_errors_aspera.rb +37 -34
- data/lib/aspera/secret_hider.rb +14 -16
- data/lib/aspera/ssh.rb +4 -1
- data/lib/aspera/sync.rb +128 -122
- data/lib/aspera/temp_file_manager.rb +10 -3
- data/lib/aspera/web_auth.rb +10 -7
- data/lib/aspera/web_server_simple.rb +9 -4
- data.tar.gz.sig +0 -0
- metadata +33 -15
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/listener/line_dump.rb +0 -19
- data/lib/aspera/cli/listener/logger.rb +0 -22
- data/lib/aspera/cli/listener/progress.rb +0 -50
- data/lib/aspera/cli/listener/progress_multi.rb +0 -84
- data/lib/aspera/cli/plugins/sync.rb +0 -44
- data/lib/aspera/fasp/listener.rb +0 -13
@@ -0,0 +1,162 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# cspell:ignore LOCALAPPDATA
|
4
|
+
require 'aspera/environment'
|
5
|
+
|
6
|
+
module Aspera
|
7
|
+
module Fasp
|
8
|
+
# find Aspera standard products installation in standard paths
|
9
|
+
class Products
|
10
|
+
# known product names
|
11
|
+
CONNECT = 'IBM Aspera Connect'
|
12
|
+
ASPERA = 'IBM Aspera (Client)'
|
13
|
+
CLI_V1 = 'Aspera CLI (deprecated)'
|
14
|
+
DRIVE = 'Aspera Drive (deprecated)'
|
15
|
+
HSTS = 'IBM Aspera High-Speed Transfer Server'
|
16
|
+
# product information manifest: XML (part of aspera product)
|
17
|
+
INFO_META_FILE = 'product-info.mf'
|
18
|
+
BIN_SUBFOLDER = 'bin'
|
19
|
+
ETC_SUBFOLDER = 'etc'
|
20
|
+
VAR_RUN_SUBFOLDER = File.join('var', 'run')
|
21
|
+
|
22
|
+
@@found_products = nil # rubocop:disable Style/ClassVars
|
23
|
+
class << self
|
24
|
+
# @return product folders depending on OS fields
|
25
|
+
# :expected M app name is taken from the manifest if present, else defaults to this value
|
26
|
+
# :app_root M main folder for the application
|
27
|
+
# :log_root O location of log files (Linux uses syslog)
|
28
|
+
# :run_root O only for Connect Client, location of http port file
|
29
|
+
# :sub_bin O subfolder with executables, default : bin
|
30
|
+
def product_locations_on_current_os
|
31
|
+
result =
|
32
|
+
case Aspera::Environment.os
|
33
|
+
when Aspera::Environment::OS_WINDOWS then [{
|
34
|
+
expected: CONNECT,
|
35
|
+
app_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Programs', 'Aspera', 'Aspera Connect'),
|
36
|
+
log_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Aspera', 'Aspera Connect', 'var', 'log'),
|
37
|
+
run_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Aspera', 'Aspera Connect')
|
38
|
+
}, {
|
39
|
+
expected: CLI_V1,
|
40
|
+
app_root: File.join('C:', 'Program Files', 'Aspera', 'cli'),
|
41
|
+
log_root: File.join('C:', 'Program Files', 'Aspera', 'cli', 'var', 'log')
|
42
|
+
}, {
|
43
|
+
expected: HSTS,
|
44
|
+
app_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server'),
|
45
|
+
log_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server', 'var', 'log')
|
46
|
+
}]
|
47
|
+
when Aspera::Environment::OS_X then [{
|
48
|
+
expected: CONNECT,
|
49
|
+
app_root: File.join(Dir.home, 'Applications', 'Aspera Connect.app'),
|
50
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
|
51
|
+
run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
|
52
|
+
sub_bin: File.join('Contents', 'Resources')
|
53
|
+
}, {
|
54
|
+
expected: CONNECT,
|
55
|
+
app_root: File.join('', 'Applications', 'Aspera Connect.app'),
|
56
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
|
57
|
+
run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
|
58
|
+
sub_bin: File.join('Contents', 'Resources')
|
59
|
+
}, {
|
60
|
+
expected: CLI_V1,
|
61
|
+
app_root: File.join(Dir.home, 'Applications', 'Aspera CLI'),
|
62
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera')
|
63
|
+
}, {
|
64
|
+
expected: HSTS,
|
65
|
+
app_root: File.join('', 'Library', 'Aspera'),
|
66
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera')
|
67
|
+
}, {
|
68
|
+
expected: DRIVE,
|
69
|
+
app_root: File.join('', 'Applications', 'Aspera Drive.app'),
|
70
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Drive'),
|
71
|
+
sub_bin: File.join('Contents', 'Resources')
|
72
|
+
}, {
|
73
|
+
expected: ASPERA,
|
74
|
+
app_root: File.join('', 'Applications', 'IBM Aspera.app'),
|
75
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'IBM Aspera'),
|
76
|
+
sub_bin: File.join('Contents', 'Resources', 'sdk', 'aspera', 'bin')
|
77
|
+
}]
|
78
|
+
else [{ # other: Linux and Unix family
|
79
|
+
expected: CONNECT,
|
80
|
+
app_root: File.join(Dir.home, '.aspera', 'connect'),
|
81
|
+
run_root: File.join(Dir.home, '.aspera', 'connect')
|
82
|
+
}, {
|
83
|
+
expected: CLI_V1,
|
84
|
+
app_root: File.join(Dir.home, '.aspera', 'cli')
|
85
|
+
}, {
|
86
|
+
expected: HSTS,
|
87
|
+
app_root: File.join('', 'opt', 'aspera')
|
88
|
+
}]
|
89
|
+
end
|
90
|
+
result # .each {|item| item.deep_do {|h, _k, _v, _m|h.freeze}}.freeze
|
91
|
+
end
|
92
|
+
|
93
|
+
# @return the list of installed products in format of product_locations_on_current_os
|
94
|
+
def installed_products
|
95
|
+
if @@found_products.nil?
|
96
|
+
scan_locations = product_locations_on_current_os.clone
|
97
|
+
# add SDK as first search path
|
98
|
+
scan_locations.unshift({
|
99
|
+
expected: 'SDK',
|
100
|
+
app_root: Installation.instance.sdk_folder,
|
101
|
+
sub_bin: ''
|
102
|
+
})
|
103
|
+
# search installed products: with ascp
|
104
|
+
@@found_products = scan_locations.select! do |item| # rubocop:disable Style/ClassVars
|
105
|
+
# skip if not main folder
|
106
|
+
next false unless Dir.exist?(item[:app_root])
|
107
|
+
Log.log.debug{"Found #{item[:app_root]}"}
|
108
|
+
sub_bin = item[:sub_bin] || BIN_SUBFOLDER
|
109
|
+
item[:ascp_path] = File.join(item[:app_root], sub_bin, ascp_filename)
|
110
|
+
# skip if no ascp
|
111
|
+
next false unless File.exist?(item[:ascp_path])
|
112
|
+
# read info from product info file if present
|
113
|
+
product_info_file = "#{item[:app_root]}/#{INFO_META_FILE}"
|
114
|
+
if File.exist?(product_info_file)
|
115
|
+
res_s = XmlSimple.xml_in(File.read(product_info_file), {'ForceArray' => false})
|
116
|
+
item[:name] = res_s['name']
|
117
|
+
item[:version] = res_s['version']
|
118
|
+
else
|
119
|
+
item[:name] = item[:expected]
|
120
|
+
end
|
121
|
+
true # select this version
|
122
|
+
end
|
123
|
+
end
|
124
|
+
return @@found_products
|
125
|
+
end
|
126
|
+
|
127
|
+
# filename for ascp with optional extension (Windows)
|
128
|
+
def ascp_filename
|
129
|
+
return 'ascp' + Environment.exe_extension
|
130
|
+
end
|
131
|
+
|
132
|
+
# @return folder paths for specified applications
|
133
|
+
# @param name Connect or CLI
|
134
|
+
def folders(name)
|
135
|
+
found = Products.installed_products.select{|i|i[:expected].eql?(name) || i[:name].eql?(name)}
|
136
|
+
raise "Product: #{name} not found, please install." if found.empty?
|
137
|
+
return found.first
|
138
|
+
end
|
139
|
+
|
140
|
+
# @return the file path of local connect where API's URI can be read
|
141
|
+
def connect_uri
|
142
|
+
connect = folders(CONNECT)
|
143
|
+
folder = File.join(connect[:run_root], VAR_RUN_SUBFOLDER)
|
144
|
+
['', 's'].each do |ext|
|
145
|
+
uri_file = File.join(folder, "http#{ext}.uri")
|
146
|
+
Log.log.debug{"checking connect port file: #{uri_file}"}
|
147
|
+
if File.exist?(uri_file)
|
148
|
+
return File.open(uri_file, &:gets).strip
|
149
|
+
end
|
150
|
+
end
|
151
|
+
raise "no connect uri file found in #{folder}"
|
152
|
+
end
|
153
|
+
|
154
|
+
# @ return path to configuration file of aspera CLI
|
155
|
+
# def cli_conf_file
|
156
|
+
# connect = folders(PRODUCT_CLI_V1)
|
157
|
+
# return File.join(connect[:app_root], BIN_SUBFOLDER, '.aspera_cli_conf')
|
158
|
+
# end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
@@ -41,7 +41,7 @@ module Aspera
|
|
41
41
|
return case tspec['direction']
|
42
42
|
when DIRECTION_SEND then :upload
|
43
43
|
when DIRECTION_RECEIVE then :download
|
44
|
-
else raise
|
44
|
+
else raise "Error: upload or download only, not #{tspec['direction']} (#{tspec['direction'].class})"
|
45
45
|
end
|
46
46
|
end
|
47
47
|
end
|
data/lib/aspera/fasp/uri.rb
CHANGED
@@ -1,14 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
# cspell:words httpport targetrate minrate bwcap createpath lockpolicy lockminrate
|
3
|
+
# cspell:words httpport targetrate minrate bwcap createpath lockpolicy lockminrate faspe
|
4
4
|
|
5
5
|
require 'aspera/log'
|
6
|
+
require 'aspera/rest'
|
6
7
|
require 'aspera/command_line_builder'
|
7
8
|
|
8
9
|
module Aspera
|
9
10
|
module Fasp
|
10
11
|
# translates a "faspe:" URI (used in Faspex 4) into transfer spec hash
|
11
12
|
class Uri
|
13
|
+
SCHEME = 'faspe'
|
12
14
|
def initialize(fasp_link)
|
13
15
|
@fasp_uri = URI.parse(fasp_link.gsub(' ', '%20'))
|
14
16
|
# TODO: check scheme is faspe
|
@@ -23,9 +25,7 @@ module Aspera
|
|
23
25
|
# faspex does not encode trailing base64 padding, fix that to be able to decode properly
|
24
26
|
fixed_query = @fasp_uri.query.gsub(/(=+)$/){|x|'%3D' * x.length}
|
25
27
|
|
26
|
-
|
27
|
-
name = i[0]
|
28
|
-
value = i[1]
|
28
|
+
Rest.decode_query(fixed_query).each do |name, value|
|
29
29
|
case name
|
30
30
|
when 'cookie' then result_ts['cookie'] = value
|
31
31
|
when 'token' then result_ts['token'] = value
|
data/lib/aspera/faspex_gw.rb
CHANGED
@@ -68,9 +68,9 @@ module Aspera
|
|
68
68
|
faspex_pkg_parameters = JSON.parse(request.body)
|
69
69
|
Log.log.debug{"faspex pkg create parameters=#{faspex_pkg_parameters}"}
|
70
70
|
faspex_package_create_result =
|
71
|
-
if @app_api.
|
71
|
+
if @app_api.class.name.eql?('Aspera::AoC')
|
72
72
|
faspex4_send_to_aoc(faspex_pkg_parameters)
|
73
|
-
elsif @app_api.
|
73
|
+
elsif @app_api.class.name.eql?('Aspera::Rest')
|
74
74
|
faspex4_send_to_faspex5(faspex_pkg_parameters)
|
75
75
|
else
|
76
76
|
raise "No such adapter: #{@app_api.class}"
|
@@ -13,7 +13,7 @@ module Aspera
|
|
13
13
|
def initialize(server, parameters)
|
14
14
|
raise 'parameters must be Hash' unless parameters.is_a?(Hash)
|
15
15
|
@parameters = parameters.symbolize_keys
|
16
|
-
Log.dump(:post_proc_parameters, @parameters)
|
16
|
+
Log.log.debug{Log.dump(:post_proc_parameters, @parameters)}
|
17
17
|
raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
|
18
18
|
@parameters[:script_folder] ||= '.'
|
19
19
|
@parameters[:fail_on_error] ||= false
|
@@ -44,7 +44,7 @@ module Aspera
|
|
44
44
|
script_path = File.join(@parameters[:script_folder], script_file)
|
45
45
|
Log.log.debug{"script=#{script_path}"}
|
46
46
|
webhook_parameters = JSON.parse(request.body)
|
47
|
-
Log.dump(:webhook_parameters, webhook_parameters)
|
47
|
+
Log.log.debug{Log.dump(:webhook_parameters, webhook_parameters)}
|
48
48
|
# env expects only strings
|
49
49
|
environment = webhook_parameters.each_with_object({}) { |(k, v), h| h[k] = v.to_s }
|
50
50
|
post_proc_pid = Process.spawn(environment, [script_path, script_path])
|
data/lib/aspera/hash_ext.rb
CHANGED
@@ -24,8 +24,8 @@ end
|
|
24
24
|
unless Hash.method_defined?(:transform_keys)
|
25
25
|
class Hash
|
26
26
|
def transform_keys
|
27
|
-
|
28
|
-
|
27
|
+
raise 'missing block' unless block_given?
|
28
|
+
return each_with_object({}){|(k, v), memo|memo[yield(k)] = v}
|
29
29
|
end
|
30
30
|
end
|
31
31
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# cspell:ignore blankslate
|
4
|
+
|
5
|
+
require 'aspera/rest_error_analyzer'
|
6
|
+
require 'blankslate'
|
7
|
+
|
8
|
+
Aspera::RestErrorAnalyzer.instance.add_simple_handler(name: 'JSON RPC', path: %w[error message], always: true)
|
9
|
+
|
10
|
+
module Aspera
|
11
|
+
# a very simple JSON RPC client
|
12
|
+
class JsonRpcClient < BlankSlate
|
13
|
+
JSON_RPC_VERSION = '2.0'
|
14
|
+
reveal :instance_variable_get
|
15
|
+
reveal :inspect
|
16
|
+
reveal :to_s
|
17
|
+
|
18
|
+
def initialize(api, namespace = nil)
|
19
|
+
super()
|
20
|
+
@api = api
|
21
|
+
@namespace = namespace
|
22
|
+
@request_id = 0
|
23
|
+
end
|
24
|
+
|
25
|
+
def respond_to_missing?(sym, include_private = false)
|
26
|
+
true
|
27
|
+
end
|
28
|
+
|
29
|
+
def method_missing(method, *args, &block)
|
30
|
+
args = args.first if args.size == 1 && args.first.is_a?(Hash)
|
31
|
+
data = @api.create('', {
|
32
|
+
jsonrpc: JSON_RPC_VERSION,
|
33
|
+
method: "#{@namespace}#{method}",
|
34
|
+
params: args,
|
35
|
+
id: @request_id += 1
|
36
|
+
})[:data]
|
37
|
+
raise 'response shall be Hash' unless data.is_a?(Hash)
|
38
|
+
raise 'bad version in response' unless data['jsonrpc'] == JSON_RPC_VERSION
|
39
|
+
raise 'missing id in response' unless data.key?('id')
|
40
|
+
raise 'both error and response' if data.key?('error') && data.key?('result')
|
41
|
+
raise 'bad error response' unless
|
42
|
+
!data.key?('error') ||
|
43
|
+
data['error'].is_a?(Hash) &&
|
44
|
+
data['error']['code'].is_a?(Integer) &&
|
45
|
+
data['error']['message'].is_a?(String)
|
46
|
+
return data['result']
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/log'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
# used for logging http
|
7
|
+
class LineLogger
|
8
|
+
def initialize(level)
|
9
|
+
@level = level
|
10
|
+
@buffer = []
|
11
|
+
end
|
12
|
+
|
13
|
+
def <<(string)
|
14
|
+
return if string.nil? || string.empty?
|
15
|
+
if !string.end_with?("\n")
|
16
|
+
@buffer.push(string)
|
17
|
+
return
|
18
|
+
end
|
19
|
+
Log.log.send(@level, @buffer.join('') + string.chomp)
|
20
|
+
@buffer.clear
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/aspera/log.rb
CHANGED
@@ -2,17 +2,53 @@
|
|
2
2
|
|
3
3
|
require 'aspera/colors'
|
4
4
|
require 'aspera/secret_hider'
|
5
|
+
require 'aspera/environment'
|
5
6
|
require 'logger'
|
6
7
|
require 'pp'
|
7
8
|
require 'json'
|
8
9
|
require 'singleton'
|
9
10
|
|
11
|
+
# extend Ruby logger with trace levels
|
12
|
+
class Logger
|
13
|
+
TRACE_MAX = 2
|
14
|
+
# add custom level to logger severity
|
15
|
+
module Severity
|
16
|
+
1.upto(TRACE_MAX).each { |level| const_set("TRACE#{level}", - level)}
|
17
|
+
end
|
18
|
+
# quick access to label
|
19
|
+
SEVERITY_LABEL = Severity.constants.each_with_object({}) { |name, hash| hash[Severity.const_get(name)] = name}
|
20
|
+
def format_severity(severity)
|
21
|
+
SEVERITY_LABEL[severity] || 'ANY'
|
22
|
+
end
|
23
|
+
|
24
|
+
# define methods for a given level
|
25
|
+
def self.make_methods(str_level) # rubocop:disable Style/ClassMethodsDefinitions
|
26
|
+
int_level = ::Logger.const_get(str_level.upcase)
|
27
|
+
str_level = str_level.downcase
|
28
|
+
Kernel.send('lave'.reverse, <<-EOM, nil, __FILE__, __LINE__ + 1)
|
29
|
+
def #{str_level}(message = nil, &block)
|
30
|
+
add(#{int_level}, message, &block)
|
31
|
+
end
|
32
|
+
|
33
|
+
def #{str_level}?
|
34
|
+
level <= #{int_level}
|
35
|
+
end
|
36
|
+
|
37
|
+
def #{str_level}!
|
38
|
+
self.level = #{int_level}
|
39
|
+
end
|
40
|
+
EOM
|
41
|
+
end
|
42
|
+
Logger::Severity.constants.each { |severity| make_methods(severity) }
|
43
|
+
end
|
44
|
+
|
10
45
|
module Aspera
|
11
46
|
# Singleton object for logging
|
12
47
|
class Log
|
13
48
|
include Singleton
|
14
49
|
# where logs are sent to
|
15
50
|
LOG_TYPES = %i[stderr stdout syslog].freeze
|
51
|
+
@@format = :json # rubocop:disable Style/ClassVars
|
16
52
|
# class methods
|
17
53
|
class << self
|
18
54
|
# levels are :debug,:info,:warn,:error,fatal,:unknown
|
@@ -21,22 +57,20 @@ module Aspera
|
|
21
57
|
# get the logger object of singleton
|
22
58
|
def log; instance.logger; end
|
23
59
|
|
24
|
-
# dump object
|
60
|
+
# dump object suitable for Log.log.debug
|
25
61
|
# @param name string or symbol
|
26
62
|
# @param format either pp or json format
|
27
|
-
def dump(name, object
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
"#{name.to_s.green} (#{format})=\n#{result}"
|
39
|
-
end
|
63
|
+
def dump(name, object)
|
64
|
+
result =
|
65
|
+
case @@format
|
66
|
+
when :json
|
67
|
+
JSON.pretty_generate(object) rescue PP.pp(object, +'')
|
68
|
+
when :ruby
|
69
|
+
PP.pp(object, +'')
|
70
|
+
else
|
71
|
+
raise 'wrong parameter, expect ruby or json'
|
72
|
+
end
|
73
|
+
"#{name.to_s.green} (#{@@format})=\n#{result}"
|
40
74
|
end
|
41
75
|
|
42
76
|
# Capture the output of $stderr and log it at debug level
|
@@ -70,16 +104,23 @@ module Aspera
|
|
70
104
|
# change underlying logger, but keep log level
|
71
105
|
def logger_type=(new_log_type)
|
72
106
|
current_severity_integer = @logger.level unless @logger.nil?
|
73
|
-
current_severity_integer = ENV
|
107
|
+
current_severity_integer = ENV.fetch('AS_LOG_LEVEL', nil) if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
|
74
108
|
current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
|
75
109
|
case new_log_type
|
76
110
|
when :stderr
|
77
|
-
# typed: Logger
|
78
111
|
@logger = Logger.new($stderr)
|
79
112
|
when :stdout
|
80
113
|
@logger = Logger.new($stdout)
|
81
114
|
when :syslog
|
82
115
|
require 'syslog/logger'
|
116
|
+
# the syslog class automatically creates methods from the severity names
|
117
|
+
# we just need to add the mapping (but syslog lowest is DEBUG)
|
118
|
+
1.upto(Logger::TRACE_MAX).each do |level|
|
119
|
+
Syslog::Logger.const_get(:LEVEL_MAP)[Logger.const_get("TRACE#{level}")] = Syslog::LOG_DEBUG
|
120
|
+
end
|
121
|
+
Logger::Severity.constants.each do |severity|
|
122
|
+
Syslog::Logger.make_methods(severity.downcase)
|
123
|
+
end
|
83
124
|
@logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
|
84
125
|
else
|
85
126
|
raise "unknown log type: #{new_log_type.class} #{new_log_type}"
|
data/lib/aspera/node.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/cli/error'
|
3
4
|
require 'aspera/fasp/transfer_spec'
|
4
5
|
require 'aspera/rest'
|
5
6
|
require 'aspera/oauth'
|
@@ -13,14 +14,22 @@ module Aspera
|
|
13
14
|
class Node < Aspera::Rest
|
14
15
|
# permissions
|
15
16
|
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
16
|
-
# prefix for ruby code for filter
|
17
|
+
# prefix for ruby code for filter (deprecated)
|
17
18
|
MATCH_EXEC_PREFIX = 'exec:'
|
19
|
+
MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
|
18
20
|
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
19
21
|
PATH_SEPARATOR = '/'
|
20
22
|
TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
|
23
|
+
SCOPE_USER = 'user:all'
|
24
|
+
SCOPE_ADMIN = 'admin:all'
|
25
|
+
SCOPE_PREFIX = 'node.'
|
26
|
+
SCOPE_SEPARATOR = ':'
|
27
|
+
SIGNATURE_DELIMITER = '==SIGNATURE=='
|
28
|
+
BEARER_TOKEN_VALIDITY_DEFAULT = 86400
|
29
|
+
BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
|
21
30
|
|
22
31
|
# register node special token decoder
|
23
|
-
Oauth.register_decoder(lambda{|token|
|
32
|
+
Oauth.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
|
24
33
|
|
25
34
|
# class instance variable, access with accessors on class
|
26
35
|
@use_standard_ports = true
|
@@ -28,16 +37,79 @@ module Aspera
|
|
28
37
|
class << self
|
29
38
|
attr_accessor :use_standard_ports
|
30
39
|
|
31
|
-
#
|
32
|
-
# if no prefix: regex
|
33
|
-
# if prefix: ruby code
|
34
|
-
# if expression is nil, then always match
|
40
|
+
# For access keys: provide expression to match entry in folder
|
35
41
|
def file_matcher(match_expression)
|
36
|
-
match_expression
|
37
|
-
|
38
|
-
|
42
|
+
case match_expression
|
43
|
+
when Proc then return match_expression
|
44
|
+
when Regexp then return ->(f){f['name'].match?(match_expression)}
|
45
|
+
when String
|
46
|
+
if match_expression.start_with?(MATCH_EXEC_PREFIX)
|
47
|
+
code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
|
48
|
+
Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
|
49
|
+
return Environment.secure_eval(code, __FILE__, __LINE__)
|
50
|
+
end
|
51
|
+
return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
|
52
|
+
when NilClass then return ->(_){true}
|
53
|
+
else raise Cli::BadArgument, "Invalid match expression type: #{match_expression.class}"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def file_matcher_from_argument(options)
|
58
|
+
return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
|
59
|
+
end
|
60
|
+
|
61
|
+
# node API scopes
|
62
|
+
def token_scope(access_key, scope)
|
63
|
+
return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
|
64
|
+
end
|
65
|
+
|
66
|
+
def decode_scope(scope)
|
67
|
+
items = scope.split(SCOPE_SEPARATOR, 2)
|
68
|
+
raise "invalid scope: #{scope}" unless items.length.eql?(2)
|
69
|
+
raise "invalid scope: #{scope}" unless items[0].start_with?(SCOPE_PREFIX)
|
70
|
+
return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
|
71
|
+
end
|
72
|
+
|
73
|
+
# Create an Aspera Node bearer token
|
74
|
+
# @param payload [String] JSON payload to be included in the token
|
75
|
+
# @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
|
76
|
+
def bearer_token(access_key:, payload:, private_key:)
|
77
|
+
raise 'payload shall be Hash' unless payload.is_a?(Hash)
|
78
|
+
raise 'missing user_id' unless payload.key?('user_id')
|
79
|
+
raise 'user_id must be a String' unless payload['user_id'].is_a?(String)
|
80
|
+
raise 'user_id must not be empty' if payload['user_id'].empty?
|
81
|
+
raise 'private_key shall be OpenSSL::PKey::RSA' unless private_key.is_a?(OpenSSL::PKey::RSA)
|
82
|
+
# manage convenience parameters
|
83
|
+
expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
|
84
|
+
payload.delete('_validity')
|
85
|
+
scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
|
86
|
+
payload.delete('_scope')
|
87
|
+
payload['scope'] ||= token_scope(access_key, scope)
|
88
|
+
payload['auth_type'] ||= 'access_key'
|
89
|
+
payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
|
90
|
+
payload_json = JSON.generate(payload)
|
91
|
+
return Base64.strict_encode64(Zlib::Deflate.deflate([
|
92
|
+
payload_json,
|
93
|
+
SIGNATURE_DELIMITER,
|
94
|
+
Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
|
95
|
+
''
|
96
|
+
].join("\n")))
|
97
|
+
end
|
98
|
+
|
99
|
+
def decode_bearer_token(token)
|
100
|
+
return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
|
101
|
+
end
|
102
|
+
|
103
|
+
def bearer_headers(bearer_auth, access_key: nil)
|
104
|
+
# if username is not provided, use the access key from the token
|
105
|
+
if access_key.nil?
|
106
|
+
access_key = Aspera::Node.decode_scope(Aspera::Node.decode_bearer_token(Oauth.bearer_extract(bearer_auth))['scope'])[:access_key]
|
107
|
+
raise "internal error #{access_key}" if access_key.nil?
|
39
108
|
end
|
40
|
-
return
|
109
|
+
return {
|
110
|
+
Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
|
111
|
+
'Authorization' => bearer_auth
|
112
|
+
}
|
41
113
|
end
|
42
114
|
end
|
43
115
|
|
@@ -52,6 +124,7 @@ module Aspera
|
|
52
124
|
# @param params [Hash] Rest parameters
|
53
125
|
# @param app_info [Hash,NilClass] special processing for AoC
|
54
126
|
def initialize(params:, app_info: nil, add_tspec: nil)
|
127
|
+
# init Rest
|
55
128
|
super(params)
|
56
129
|
@app_info = app_info
|
57
130
|
# this is added to transfer spec, for instance to add tags (COS)
|
@@ -90,13 +163,13 @@ module Aspera
|
|
90
163
|
# @param state [Object] state object sent to processing method
|
91
164
|
# @param top_file_id [String] file id to start at (default = access key root file id)
|
92
165
|
# @param top_file_path [String] path of top folder (default = /)
|
93
|
-
# @param block [Proc] processing method,
|
166
|
+
# @param block [Proc] processing method, arguments: entry, path, state
|
94
167
|
def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
|
95
168
|
raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
|
96
169
|
raise 'INTERNAL ERROR: Missing block' unless block
|
97
170
|
# start at top folder
|
98
171
|
folders_to_explore = [{id: top_file_id, path: top_file_path}]
|
99
|
-
Log.dump(:folders_to_explore, folders_to_explore)
|
172
|
+
Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
|
100
173
|
until folders_to_explore.empty?
|
101
174
|
current_item = folders_to_explore.shift
|
102
175
|
Log.log.debug{"searching #{current_item[:path]}".bg_green}
|
@@ -108,7 +181,7 @@ module Aspera
|
|
108
181
|
Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
|
109
182
|
[]
|
110
183
|
end
|
111
|
-
Log.dump(:folder_contents, folder_contents)
|
184
|
+
Log.log.debug{Log.dump(:folder_contents, folder_contents)}
|
112
185
|
folder_contents.each do |entry|
|
113
186
|
relative_path = File.join(current_item[:path], entry['name'])
|
114
187
|
Log.log.debug{"process_folder_tree checking #{relative_path}"}
|
@@ -204,7 +277,7 @@ module Aspera
|
|
204
277
|
when :basic
|
205
278
|
ak_name = params[:auth][:username]
|
206
279
|
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
207
|
-
ak_token = Rest.
|
280
|
+
ak_token = Rest.basic_token(params[:auth][:username], params[:auth][:password])
|
208
281
|
when :oauth2
|
209
282
|
ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
|
210
283
|
# TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
|
@@ -235,9 +308,19 @@ module Aspera
|
|
235
308
|
transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
|
236
309
|
# by default: same address as node API
|
237
310
|
transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
|
311
|
+
# AoC allows specification of other url
|
238
312
|
if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
|
239
313
|
transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
|
240
314
|
end
|
315
|
+
info = read('info')[:data]
|
316
|
+
# get the transfer user from info on access key
|
317
|
+
transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
|
318
|
+
# get settings from name.value array to hash key.value
|
319
|
+
settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
|
320
|
+
# check WSS ports
|
321
|
+
%w[wss_enabled wss_port].each do |i|
|
322
|
+
transfer_spec[i] = settings[i] if settings.key?(i)
|
323
|
+
end if settings.is_a?(Hash)
|
241
324
|
else
|
242
325
|
# retrieve values from API (and keep a copy/cache)
|
243
326
|
@std_t_spec_cache ||= create(
|