aspera-cli 4.9.0 → 4.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +20 -0
- data/CHANGELOG.md +509 -0
- data/CONTRIBUTING.md +118 -0
- data/README.md +1241 -916
- data/bin/ascli +4 -4
- data/bin/asession +11 -11
- data/docs/test_env.conf +32 -21
- data/examples/aoc.rb +4 -4
- data/examples/dascli +16 -9
- data/examples/faspex4.rb +8 -8
- data/examples/node.rb +12 -12
- data/examples/server.rb +10 -10
- data/lib/aspera/aoc.rb +273 -266
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formater.rb +64 -64
- data/lib/aspera/cli/info.rb +2 -2
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +14 -19
- data/lib/aspera/cli/main.rb +66 -67
- data/lib/aspera/cli/manager.rb +112 -110
- data/lib/aspera/cli/plugin.rb +57 -36
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +309 -670
- data/lib/aspera/cli/plugins/ats.rb +44 -46
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +497 -378
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +112 -114
- data/lib/aspera/cli/plugins/faspex5.rb +71 -46
- data/lib/aspera/cli/plugins/node.rb +379 -283
- data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
- data/lib/aspera/cli/plugins/preview.rb +122 -114
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +30 -29
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +60 -59
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +27 -27
- data/lib/aspera/cos_node.rb +22 -20
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +35 -15
- data/lib/aspera/fasp/agent_base.rb +15 -15
- data/lib/aspera/fasp/agent_connect.rb +23 -21
- data/lib/aspera/fasp/agent_direct.rb +66 -64
- data/lib/aspera/fasp/agent_httpgw.rb +141 -78
- data/lib/aspera/fasp/agent_node.rb +23 -21
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +79 -79
- data/lib/aspera/fasp/listener.rb +1 -1
- data/lib/aspera/fasp/parameters.rb +86 -71
- data/lib/aspera/fasp/parameters.yaml +7 -4
- data/lib/aspera/fasp/resume_policy.rb +8 -8
- data/lib/aspera/fasp/transfer_spec.rb +35 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +38 -105
- data/lib/aspera/keychain/macos_security.rb +128 -57
- data/lib/aspera/log.rb +7 -7
- data/lib/aspera/nagios.rb +19 -18
- data/lib/aspera/node.rb +209 -35
- data/lib/aspera/oauth.rb +37 -36
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +16 -15
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +41 -41
- data/lib/aspera/proxy_auto_config.rb +21 -14
- data/lib/aspera/rest.rb +72 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +18 -17
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +15 -13
- data/lib/aspera/ssh.rb +11 -10
- data/lib/aspera/sync.rb +158 -44
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +14 -13
- data.tar.gz.sig +0 -0
- metadata +11 -36
- metadata.gz.sig +0 -0
data/lib/aspera/nagios.rb
CHANGED
@@ -10,32 +10,32 @@ module Aspera
|
|
10
10
|
# date offset levels
|
11
11
|
DATE_WARN_OFFSET = 2
|
12
12
|
DATE_CRIT_OFFSET = 5
|
13
|
-
private_constant :LEVELS
|
13
|
+
private_constant :LEVELS, :ADD_PREFIX, :DATE_WARN_OFFSET, :DATE_CRIT_OFFSET
|
14
14
|
|
15
15
|
# add methods to add nagios error levels, each take component name and message
|
16
16
|
LEVELS.each_index do |code|
|
17
17
|
name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
|
18
|
-
define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
|
18
|
+
define_method(name){|comp, msg|@data.push({code: code, comp: comp, msg: msg})}
|
19
19
|
end
|
20
20
|
|
21
21
|
class << self
|
22
22
|
# process results of a analysis and display status and exit with code
|
23
23
|
def process(data)
|
24
24
|
raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
|
25
|
-
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.
|
25
|
+
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.key?(c)}
|
26
26
|
res_errors = data.reject{|s|s['status'].eql?('ok')}
|
27
27
|
# keep only errors in case of problem, other ok are assumed so
|
28
28
|
data = res_errors unless res_errors.empty?
|
29
29
|
# first is most critical
|
30
|
-
data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
30
|
+
data.sort!{|a, b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
31
31
|
# build message: if multiple components: concatenate
|
32
|
-
#message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
33
|
-
message = data
|
34
|
-
map{|i|i['component']}
|
35
|
-
uniq
|
36
|
-
map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
|
37
|
-
join(', ')
|
38
|
-
tr("\n",' ')
|
32
|
+
# message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
33
|
+
message = data
|
34
|
+
.map{|i|i['component']}
|
35
|
+
.uniq
|
36
|
+
.map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
|
37
|
+
.join(', ')
|
38
|
+
.tr("\n", ' ')
|
39
39
|
status = data.first['status'].upcase
|
40
40
|
# display status for nagios
|
41
41
|
puts("#{status} - [#{message}]\n")
|
@@ -45,6 +45,7 @@ module Aspera
|
|
45
45
|
end
|
46
46
|
|
47
47
|
attr_reader :data
|
48
|
+
|
48
49
|
def initialize
|
49
50
|
@data = []
|
50
51
|
end
|
@@ -55,26 +56,26 @@ module Aspera
|
|
55
56
|
rtime = DateTime.strptime(remote_date)
|
56
57
|
diff_time = (rtime - DateTime.now).abs
|
57
58
|
diff_disp = diff_time.round(-2)
|
58
|
-
Log.log.debug
|
59
|
+
Log.log.debug{"DATE: #{remote_date} #{rtime} diff=#{diff_disp}"}
|
59
60
|
msg = "offset #{diff_disp} sec"
|
60
61
|
if diff_time >= DATE_CRIT_OFFSET
|
61
|
-
add_critical(component,msg)
|
62
|
+
add_critical(component, msg)
|
62
63
|
elsif diff_time >= DATE_WARN_OFFSET
|
63
|
-
add_warning(component,msg)
|
64
|
+
add_warning(component, msg)
|
64
65
|
else
|
65
|
-
add_ok(component,msg)
|
66
|
+
add_ok(component, msg)
|
66
67
|
end
|
67
68
|
end
|
68
69
|
|
69
70
|
def check_product_version(component, _product, version)
|
70
|
-
add_ok(component,"version #{version}")
|
71
|
-
# TODO check on database if latest version
|
71
|
+
add_ok(component, "version #{version}")
|
72
|
+
# TODO: check on database if latest version
|
72
73
|
end
|
73
74
|
|
74
75
|
# translate for display
|
75
76
|
def result
|
76
77
|
raise 'missing result' if @data.empty?
|
77
|
-
{type: :object_list,data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s,'component' => i[:comp],'message' => i[:msg]}}}
|
78
|
+
{type: :object_list, data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
|
78
79
|
end
|
79
80
|
end
|
80
81
|
end
|
data/lib/aspera/node.rb
CHANGED
@@ -9,22 +9,23 @@ require 'zlib'
|
|
9
9
|
require 'base64'
|
10
10
|
|
11
11
|
module Aspera
|
12
|
-
# Provides additional functions using node API
|
13
|
-
class Node < Rest
|
12
|
+
# Provides additional functions using node API with gen4 extensions (access keys)
|
13
|
+
class Node < Aspera::Rest
|
14
14
|
# permissions
|
15
15
|
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
16
16
|
# prefix for ruby code for filter
|
17
17
|
MATCH_EXEC_PREFIX = 'exec:'
|
18
|
+
X_ASPERA_ACCESSKEY = 'X-Aspera-AccessKey'
|
19
|
+
PATH_SEPARATOR = '/'
|
18
20
|
|
19
21
|
# register node special token decoder
|
20
22
|
Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
|
21
23
|
|
24
|
+
# class instance variable, access with accessors on class
|
25
|
+
@use_standard_ports = true
|
26
|
+
|
22
27
|
class << self
|
23
|
-
|
24
|
-
Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, "\
|
25
|
-
"but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
26
|
-
ts['token'] = Rest.basic_creds(ak,secret)
|
27
|
-
end
|
28
|
+
attr_accessor :use_standard_ports
|
28
29
|
|
29
30
|
# for access keys: provide expression to match entry in folder
|
30
31
|
# if no prefix: regex
|
@@ -39,49 +40,222 @@ module Aspera
|
|
39
40
|
end
|
40
41
|
end
|
41
42
|
|
42
|
-
|
43
|
-
|
44
|
-
|
43
|
+
REQUIRED_APP_INFO_FIELDS = %i[node_info app api plugin].freeze
|
44
|
+
REQUIRED_APP_API_METHODS = %i[node_id_to_api add_ts_tags].freeze
|
45
|
+
private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
|
46
|
+
|
47
|
+
attr_reader :app_info
|
48
|
+
|
49
|
+
# @param params [Hash] Rest parameters
|
50
|
+
# @param app_info [Hash,NilClass] special processing for AoC
|
51
|
+
def initialize(params:, app_info: nil, add_tspec: nil)
|
52
|
+
super(params)
|
53
|
+
@app_info = app_info
|
54
|
+
# this is added to transfer spec, for instance to add tags (COS)
|
55
|
+
@add_tspec = add_tspec
|
56
|
+
if !@app_info.nil?
|
57
|
+
REQUIRED_APP_INFO_FIELDS.each do |field|
|
58
|
+
raise "INTERNAL ERROR: app_info lacks field #{field}" unless @app_info.key?(field)
|
59
|
+
end
|
60
|
+
REQUIRED_APP_API_METHODS.each do |method|
|
61
|
+
raise "INTERNAL ERROR: #{@app_info[:api].class} lacks method #{method}" unless @app_info[:api].respond_to?(method)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# update transfer spec with special additional tags
|
67
|
+
def add_tspec_info(tspec)
|
68
|
+
tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
|
69
|
+
return tspec
|
70
|
+
end
|
71
|
+
|
72
|
+
# @returns [Aspera::Node] a Node or nil
|
73
|
+
def node_id_to_node(node_id)
|
74
|
+
return self if !@app_info.nil? && @app_info[:node_info]['id'].eql?(node_id)
|
75
|
+
return @app_info[:api].node_id_to_api(node_id) unless @app_info.nil?
|
76
|
+
Log.log.warn{"cannot resolve link with node id #{node_id}"}
|
77
|
+
return nil
|
78
|
+
end
|
45
79
|
|
46
|
-
# recursively
|
80
|
+
# recursively browse in a folder (with non-recursive method)
|
47
81
|
# subfolders a processed if the processing method returns true
|
48
|
-
# @param
|
49
|
-
# @param
|
50
|
-
#
|
51
|
-
#
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
opt[:method] ||= :process_entry
|
59
|
-
raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
|
60
|
-
Log.log.debug("crawl #{opt}")
|
61
|
-
#top_info=read("files/#{opt[:top_file_id]}")[:data]
|
62
|
-
folders_to_explore = [{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
|
63
|
-
Log.dump(:folders_to_explore,folders_to_explore)
|
64
|
-
while !folders_to_explore.empty?
|
82
|
+
# @param state [Object] state object sent to processing method
|
83
|
+
# @param method [Symbol] processing method name
|
84
|
+
# @param top_file_id [String] file id to start at (default = access key root file id)
|
85
|
+
# @param top_file_path [String] path of top folder (default = /)
|
86
|
+
def process_folder_tree(state:, method:, top_file_id:, top_file_path: '/')
|
87
|
+
raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
|
88
|
+
raise "INTERNAL ERROR: Missing method #{method}" unless respond_to?(method)
|
89
|
+
folders_to_explore = [{id: top_file_id, relpath: top_file_path}]
|
90
|
+
Log.dump(:folders_to_explore, folders_to_explore)
|
91
|
+
until folders_to_explore.empty?
|
65
92
|
current_item = folders_to_explore.shift
|
66
|
-
Log.log.debug
|
93
|
+
Log.log.debug{"searching #{current_item[:relpath]}".bg_green}
|
67
94
|
# get folder content
|
68
95
|
folder_contents =
|
69
96
|
begin
|
70
97
|
read("files/#{current_item[:id]}/files")[:data]
|
71
98
|
rescue StandardError => e
|
72
|
-
Log.log.warn
|
99
|
+
Log.log.warn{"#{current_item[:relpath]}: #{e.class} #{e.message}"}
|
73
100
|
[]
|
74
101
|
end
|
75
|
-
Log.dump(:folder_contents,folder_contents)
|
102
|
+
Log.dump(:folder_contents, folder_contents)
|
76
103
|
folder_contents.each do |entry|
|
77
|
-
relative_path = File.join(current_item[:relpath],entry['name'])
|
78
|
-
Log.log.debug
|
104
|
+
relative_path = File.join(current_item[:relpath], entry['name'])
|
105
|
+
Log.log.debug{"looking #{relative_path}".bg_green}
|
106
|
+
# continue only if method returns true
|
107
|
+
next unless send(method, entry, relative_path, state)
|
79
108
|
# entry type is file, folder or link
|
80
|
-
|
81
|
-
|
109
|
+
case entry['type']
|
110
|
+
when 'folder'
|
111
|
+
folders_to_explore.push({id: entry['id'], relpath: relative_path})
|
112
|
+
when 'link'
|
113
|
+
node_id_to_node(entry['target_node_id'])&.process_folder_tree(
|
114
|
+
state: state,
|
115
|
+
method: method,
|
116
|
+
top_file_id: entry['target_id'],
|
117
|
+
top_file_path: relative_path)
|
82
118
|
end
|
83
119
|
end
|
84
120
|
end
|
121
|
+
end # process_folder_tree
|
122
|
+
|
123
|
+
# processing method to resolve a file path to id
|
124
|
+
# @returns true if processing need to continue
|
125
|
+
def process_resolve_node_path(entry, _path, state)
|
126
|
+
# stop digging here if not in right path
|
127
|
+
return false unless entry['name'].eql?(state[:path].first)
|
128
|
+
# ok it matches, so we remove the match
|
129
|
+
state[:path].shift
|
130
|
+
case entry['type']
|
131
|
+
when 'file'
|
132
|
+
# file must be terminal
|
133
|
+
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
134
|
+
# it's terminal, we found it
|
135
|
+
state[:result] = {api: self, file_id: entry['id']}
|
136
|
+
return false
|
137
|
+
when 'folder'
|
138
|
+
if state[:path].empty?
|
139
|
+
# we found it
|
140
|
+
state[:result] = {api: self, file_id: entry['id']}
|
141
|
+
return false
|
142
|
+
end
|
143
|
+
when 'link'
|
144
|
+
if state[:path].empty?
|
145
|
+
# we found it
|
146
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
147
|
+
raise 'cannot resolve link' if other_node.nil?
|
148
|
+
state[:result] = {api: other_node, file_id: entry['target_id']}
|
149
|
+
return false
|
150
|
+
end
|
151
|
+
else
|
152
|
+
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
153
|
+
end
|
154
|
+
# continue to dig folder
|
155
|
+
return true
|
156
|
+
end
|
157
|
+
|
158
|
+
# Navigate the path from given file id
|
159
|
+
# @param id initial file id
|
160
|
+
# @param path file path
|
161
|
+
# @return {.api,.file_id}
|
162
|
+
def resolve_api_fid(top_file_id, path)
|
163
|
+
raise 'file id shall be String' unless top_file_id.is_a?(String)
|
164
|
+
path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
|
165
|
+
return {api: self, file_id: top_file_id} if path_elements.empty?
|
166
|
+
resolve_state = {path: path_elements, result: nil}
|
167
|
+
process_folder_tree(state: resolve_state, method: :process_resolve_node_path, top_file_id: top_file_id)
|
168
|
+
raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
|
169
|
+
return resolve_state[:result]
|
170
|
+
end
|
171
|
+
|
172
|
+
# add entry to list if test block is success
|
173
|
+
def process_find_files(entry, path, state)
|
174
|
+
begin
|
175
|
+
# add to result if match filter
|
176
|
+
state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
|
177
|
+
# process link
|
178
|
+
if entry[:type].eql?('link')
|
179
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
180
|
+
other_node.process_folder_tree(state: state, method: process_find_files, top_file_id: entry['target_id'], top_file_path: path)
|
181
|
+
end
|
182
|
+
rescue StandardError => e
|
183
|
+
Log.log.error{"#{path}: #{e.message}"}
|
184
|
+
end
|
185
|
+
# process all folders
|
186
|
+
return true
|
187
|
+
end
|
188
|
+
|
189
|
+
def find_files(top_file_id, test_block)
|
190
|
+
Log.log.debug{"find_files: fileid=#{top_file_id}"}
|
191
|
+
find_state = {found: [], test_block: test_block}
|
192
|
+
process_folder_tree(state: find_state, method: :process_find_files, top_file_id: top_file_id)
|
193
|
+
return find_state[:found]
|
194
|
+
end
|
195
|
+
|
196
|
+
# Create transfer spec for gen4
|
197
|
+
def transfer_spec_gen4(file_id, direction, ts_merge=nil)
|
198
|
+
ak_name = nil
|
199
|
+
ak_token = nil
|
200
|
+
case params[:auth][:type]
|
201
|
+
when :basic
|
202
|
+
ak_name = params[:auth][:username]
|
203
|
+
when :oauth2
|
204
|
+
ak_name = params[:headers][X_ASPERA_ACCESSKEY]
|
205
|
+
token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
|
206
|
+
ak_token = token_generation_lambda.call(false) # first time, use cache
|
207
|
+
@app_info[:plugin].transfer.token_regenerator = token_generation_lambda unless @app_info.nil?
|
208
|
+
else raise "Unsupported auth method for node gen4: #{params[:auth][:type]}"
|
209
|
+
end
|
210
|
+
transfer_spec = {
|
211
|
+
'direction' => direction,
|
212
|
+
'token' => ak_token,
|
213
|
+
'tags' => {
|
214
|
+
'aspera' => {
|
215
|
+
'node' => {
|
216
|
+
'access_key' => ak_name,
|
217
|
+
'file_id' => file_id
|
218
|
+
} # node
|
219
|
+
} # aspera
|
220
|
+
} # tags
|
221
|
+
}
|
222
|
+
# add specials tags (cos)
|
223
|
+
add_tspec_info(transfer_spec)
|
224
|
+
transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
|
225
|
+
# add application specific tags (AoC)
|
226
|
+
the_app = app_info
|
227
|
+
the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
|
228
|
+
# add basic token
|
229
|
+
if transfer_spec['token'].nil?
|
230
|
+
ts_basic_token(transfer_spec)
|
231
|
+
end
|
232
|
+
# add remote host info
|
233
|
+
if self.class.use_standard_ports
|
234
|
+
# get default TCP/UDP ports and transfer user
|
235
|
+
transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
|
236
|
+
# by default: same address as node API
|
237
|
+
transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
|
238
|
+
if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
|
239
|
+
transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
|
240
|
+
end
|
241
|
+
else
|
242
|
+
# retrieve values from API
|
243
|
+
std_t_spec = create(
|
244
|
+
'files/download_setup',
|
245
|
+
{transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
|
246
|
+
)[:data]['transfer_specs'].first['transfer_spec']
|
247
|
+
# copy some parts
|
248
|
+
%w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].each {|i| transfer_spec[i] = std_t_spec[i] if std_t_spec.key?(i)}
|
249
|
+
end
|
250
|
+
return transfer_spec
|
251
|
+
end
|
252
|
+
|
253
|
+
# set basic token in transfer spec
|
254
|
+
def ts_basic_token(ts)
|
255
|
+
Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
|
256
|
+
unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
257
|
+
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
258
|
+
ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
|
85
259
|
end
|
86
260
|
end
|
87
261
|
end
|
data/lib/aspera/oauth.rb
CHANGED
@@ -35,7 +35,7 @@ module Aspera
|
|
35
35
|
# a prefix for persistency of tokens (simplify garbage collect)
|
36
36
|
PERSIST_CATEGORY_TOKEN = 'token'
|
37
37
|
|
38
|
-
private_constant :JWT_NOTBEFORE_OFFSET_SEC
|
38
|
+
private_constant :JWT_NOTBEFORE_OFFSET_SEC, :JWT_EXPIRY_OFFSET_SEC, :TOKEN_CACHE_EXPIRY_SEC, :PERSIST_CATEGORY_TOKEN, :TOKEN_EXPIRATION_GUARD_SEC
|
39
39
|
|
40
40
|
# persistency manager
|
41
41
|
@persist = nil
|
@@ -48,7 +48,7 @@ module Aspera
|
|
48
48
|
def persist_mgr=(manager)
|
49
49
|
@persist = manager
|
50
50
|
# cleanup expired tokens
|
51
|
-
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_CACHE_EXPIRY_SEC)
|
51
|
+
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN, TOKEN_CACHE_EXPIRY_SEC)
|
52
52
|
end
|
53
53
|
|
54
54
|
def persist_mgr
|
@@ -56,7 +56,7 @@ module Aspera
|
|
56
56
|
Log.log.debug('Not using persistency') # (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)
|
57
57
|
# create NULL persistency class
|
58
58
|
@persist = Class.new do
|
59
|
-
def get(_x);nil;end;def delete(_x);nil;end;def put(_x,_y);nil;end;def garbage_collect(_x,_y);nil;end # rubocop:disable Layout/EmptyLineBetweenDefs
|
59
|
+
def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Layout/EmptyLineBetweenDefs, Style/Semicolon, Layout/LineLength
|
60
60
|
end.new
|
61
61
|
end
|
62
62
|
return @persist
|
@@ -64,7 +64,7 @@ module Aspera
|
|
64
64
|
|
65
65
|
# delete all existing tokens
|
66
66
|
def flush_tokens
|
67
|
-
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
|
67
|
+
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN, nil)
|
68
68
|
end
|
69
69
|
|
70
70
|
# register a bearer token decoder, mainly to inspect expiry date
|
@@ -94,24 +94,24 @@ module Aspera
|
|
94
94
|
|
95
95
|
# @return one of the registered creators for the given create type
|
96
96
|
def token_creator(id)
|
97
|
-
raise "token creator type unknown: #{id}/#{id.class}" unless @create_handlers.
|
97
|
+
raise "token creator type unknown: #{id}/#{id.class}" unless @create_handlers.key?(id)
|
98
98
|
@create_handlers[id]
|
99
99
|
end
|
100
100
|
|
101
101
|
# list of identifiers foundn in creation parameters that can be used to uniquely identify the token
|
102
102
|
def id_creator(id)
|
103
|
-
raise "id creator type unknown: #{id}/#{id.class}" unless @id_handlers.
|
103
|
+
raise "id creator type unknown: #{id}/#{id.class}" unless @id_handlers.key?(id)
|
104
104
|
@id_handlers[id]
|
105
105
|
end
|
106
106
|
end # self
|
107
107
|
|
108
108
|
# JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
|
109
|
-
register_decoder lambda { |token| parts = token.split('.'); raise 'not aoc token' unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))}
|
109
|
+
register_decoder lambda { |token| parts = token.split('.'); raise 'not aoc token' unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))} # rubocop:disable Style/Semicolon, Layout/LineLength
|
110
110
|
|
111
111
|
# generic token creation, parameters are provided in :generic
|
112
|
-
register_token_creator :generic,lambda { |oauth|
|
112
|
+
register_token_creator :generic, lambda { |oauth|
|
113
113
|
return oauth.create_token(oauth.sparams)
|
114
|
-
},lambda { |oauth|
|
114
|
+
}, lambda { |oauth|
|
115
115
|
return [
|
116
116
|
oauth.sparams[:grant_type]&.split(':')&.last,
|
117
117
|
oauth.sparams[:apikey],
|
@@ -120,12 +120,13 @@ module Aspera
|
|
120
120
|
}
|
121
121
|
|
122
122
|
# Authentication using Web browser
|
123
|
-
register_token_creator :web,lambda { |oauth|
|
123
|
+
register_token_creator :web, lambda { |oauth|
|
124
124
|
random_state = SecureRandom.uuid # used to check later
|
125
|
-
login_page_url = Rest.build_uri(
|
125
|
+
login_page_url = Rest.build_uri(
|
126
|
+
"#{oauth.api.params[:base_url]}/#{oauth.sparams[:path_authorize]}",
|
126
127
|
oauth.optional_scope_client_id.merge(response_type: 'code', redirect_uri: oauth.sparams[:redirect_uri], state: random_state))
|
127
128
|
# here, we need a human to authorize on a web page
|
128
|
-
Log.log.info
|
129
|
+
Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
|
129
130
|
# start a web server to receive request code
|
130
131
|
webserver = WebAuth.new(oauth.sparams[:redirect_uri])
|
131
132
|
# start browser on login page
|
@@ -138,17 +139,17 @@ module Aspera
|
|
138
139
|
grant_type: 'authorization_code',
|
139
140
|
code: received_params['code'],
|
140
141
|
redirect_uri: oauth.sparams[:redirect_uri]))
|
141
|
-
},lambda { |_oauth|
|
142
|
+
}, lambda { |_oauth|
|
142
143
|
return []
|
143
144
|
}
|
144
145
|
|
145
146
|
# Authentication using private key
|
146
|
-
register_token_creator :jwt,lambda { |oauth|
|
147
|
+
register_token_creator :jwt, lambda { |oauth|
|
147
148
|
# https://tools.ietf.org/html/rfc7523
|
148
149
|
# https://tools.ietf.org/html/rfc7519
|
149
150
|
require 'jwt'
|
150
151
|
seconds_since_epoch = Time.new.to_i
|
151
|
-
Log.log.info
|
152
|
+
Log.log.info{"seconds=#{seconds_since_epoch}"}
|
152
153
|
raise 'missing JWT payload' unless oauth.sparams[:payload].is_a?(Hash)
|
153
154
|
jwt_payload = {
|
154
155
|
exp: seconds_since_epoch + JWT_EXPIRY_OFFSET_SEC, # expiration time
|
@@ -156,14 +157,14 @@ module Aspera
|
|
156
157
|
iat: seconds_since_epoch, # issued at
|
157
158
|
jti: SecureRandom.uuid # JWT id
|
158
159
|
}.merge(oauth.sparams[:payload])
|
159
|
-
Log.log.debug
|
160
|
+
Log.log.debug{"JWT jwt_payload=[#{jwt_payload}]"}
|
160
161
|
rsa_private = oauth.sparams[:private_key_obj] # type: OpenSSL::PKey::RSA
|
161
|
-
Log.log.debug
|
162
|
+
Log.log.debug{"private=[#{rsa_private}]"}
|
162
163
|
assertion = JWT.encode(jwt_payload, rsa_private, 'RS256', oauth.sparams[:headers] || {})
|
163
|
-
Log.log.debug
|
164
|
+
Log.log.debug{"assertion=[#{assertion}]"}
|
164
165
|
return oauth.create_token(oauth.optional_scope_client_id.merge(grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', assertion: assertion))
|
165
|
-
},lambda { |oauth|
|
166
|
-
return [oauth.sparams.dig(:payload
|
166
|
+
}, lambda { |oauth|
|
167
|
+
return [oauth.sparams.dig(:payload, :sub)]
|
167
168
|
}
|
168
169
|
|
169
170
|
attr_reader :gparams, :sparams, :api
|
@@ -186,14 +187,14 @@ module Aspera
|
|
186
187
|
# :web:path_authorize [D] for type :web
|
187
188
|
# :generic [M] for type :generic
|
188
189
|
def initialize(a_params)
|
189
|
-
Log.log.debug
|
190
|
+
Log.log.debug{"auth=#{a_params}"}
|
190
191
|
# replace default values
|
191
192
|
@gparams = DEFAULT_CREATE_PARAMS.deep_merge(a_params)
|
192
193
|
# check that type is known
|
193
194
|
self.class.token_creator(@gparams[:crtype])
|
194
195
|
# specific parameters for the creation type
|
195
|
-
@sparams
|
196
|
-
if @gparams[:crtype].eql?(:web) && @sparams.
|
196
|
+
@sparams = @gparams[@gparams[:crtype]]
|
197
|
+
if @gparams[:crtype].eql?(:web) && @sparams.key?(:redirect_uri)
|
197
198
|
uri = URI.parse(@sparams[:redirect_uri])
|
198
199
|
raise 'redirect_uri scheme must be http or https' unless %w[http https].include?(uri.scheme)
|
199
200
|
raise 'redirect_uri must have a port' if uri.port.nil?
|
@@ -203,14 +204,14 @@ module Aspera
|
|
203
204
|
base_url: @gparams[:base_url],
|
204
205
|
redirect_max: 2
|
205
206
|
}
|
206
|
-
rest_params[:auth] = a_params[:auth] if a_params.
|
207
|
+
rest_params[:auth] = a_params[:auth] if a_params.key?(:auth)
|
207
208
|
@api = Rest.new(rest_params)
|
208
209
|
# if needed use from api
|
209
210
|
@gparams.delete(:base_url)
|
210
211
|
@gparams.delete(:auth)
|
211
212
|
@gparams.delete(@gparams[:crtype])
|
212
|
-
Log.dump(:gparams
|
213
|
-
Log.dump(:sparams
|
213
|
+
Log.dump(:gparams, @gparams)
|
214
|
+
Log.dump(:sparams, @sparams)
|
214
215
|
end
|
215
216
|
|
216
217
|
public
|
@@ -226,7 +227,7 @@ module Aspera
|
|
226
227
|
|
227
228
|
# @return Hash with optional general parameters
|
228
229
|
def optional_scope_client_id(add_secret: false)
|
229
|
-
call_params={}
|
230
|
+
call_params = {}
|
230
231
|
call_params[:scope] = @gparams[:scope] unless @gparams[:scope].nil?
|
231
232
|
call_params[:client_id] = @gparams[:client_id] unless @gparams[:client_id].nil?
|
232
233
|
call_params[:client_secret] = @gparams[:client_secret] if add_secret && !@gparams[:client_id].nil?
|
@@ -254,7 +255,7 @@ module Aspera
|
|
254
255
|
# `direct` agent is equipped with refresh code
|
255
256
|
if !use_refresh_token && !token_data.nil?
|
256
257
|
decoded_token = self.class.decode_token(token_data[@gparams[:token_field]])
|
257
|
-
Log.dump('decoded_token',decoded_token) unless decoded_token.nil?
|
258
|
+
Log.dump('decoded_token', decoded_token) unless decoded_token.nil?
|
258
259
|
if decoded_token.is_a?(Hash)
|
259
260
|
expires_at_sec =
|
260
261
|
if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
|
@@ -262,13 +263,13 @@ module Aspera
|
|
262
263
|
end
|
263
264
|
# force refresh if we see a token too close from expiration
|
264
265
|
use_refresh_token = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < TOKEN_EXPIRATION_GUARD_SEC
|
265
|
-
Log.log.debug
|
266
|
+
Log.log.debug{"Expiration: #{expires_at_sec} / #{use_refresh_token}"}
|
266
267
|
end
|
267
268
|
end
|
268
269
|
|
269
270
|
# an API was already called, but failed, we need to regenerate or refresh
|
270
271
|
if use_refresh_token
|
271
|
-
if token_data.is_a?(Hash) && token_data.
|
272
|
+
if token_data.is_a?(Hash) && token_data.key?('refresh_token')
|
272
273
|
# save possible refresh token, before deleting the cache
|
273
274
|
refresh_token = token_data['refresh_token']
|
274
275
|
end
|
@@ -277,17 +278,17 @@ module Aspera
|
|
277
278
|
token_data = nil
|
278
279
|
# lets try the existing refresh token
|
279
280
|
if !refresh_token.nil?
|
280
|
-
Log.log.info
|
281
|
+
Log.log.info{"refresh=[#{refresh_token}]".bg_green}
|
281
282
|
# try to refresh
|
282
283
|
# note: AoC admin token has no refresh, and lives by default 1800secs
|
283
|
-
resp = create_token(optional_scope_client_id.merge(grant_type: 'refresh_token',refresh_token: refresh_token))
|
284
|
+
resp = create_token(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
|
284
285
|
if resp[:http].code.start_with?('2')
|
285
286
|
# save only if success
|
286
287
|
json_data = resp[:http].body
|
287
288
|
token_data = JSON.parse(json_data)
|
288
|
-
self.class.persist_mgr.put(token_id,json_data)
|
289
|
+
self.class.persist_mgr.put(token_id, json_data)
|
289
290
|
else
|
290
|
-
Log.log.debug
|
291
|
+
Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
|
291
292
|
end
|
292
293
|
end
|
293
294
|
end
|
@@ -297,9 +298,9 @@ module Aspera
|
|
297
298
|
resp = self.class.token_creator(@gparams[:crtype]).call(self)
|
298
299
|
json_data = resp[:http].body
|
299
300
|
token_data = JSON.parse(json_data)
|
300
|
-
self.class.persist_mgr.put(token_id,json_data)
|
301
|
+
self.class.persist_mgr.put(token_id, json_data)
|
301
302
|
end # if ! in_cache
|
302
|
-
raise "API error: No such field in answer: #{@gparams[:token_field]}" unless token_data.
|
303
|
+
raise "API error: No such field in answer: #{@gparams[:token_field]}" unless token_data.key?(@gparams[:token_field])
|
303
304
|
# ok we shall have a token here
|
304
305
|
return 'Bearer ' + token_data[@gparams[:token_field]]
|
305
306
|
end
|
@@ -12,30 +12,38 @@ module Aspera
|
|
12
12
|
class OpenApplication
|
13
13
|
include Singleton
|
14
14
|
class << self
|
15
|
+
USER_INTERFACES = %i[text graphical].freeze
|
15
16
|
# User Interfaces
|
16
|
-
def user_interfaces;
|
17
|
+
def user_interfaces; USER_INTERFACES; end
|
17
18
|
|
18
19
|
def default_gui_mode
|
19
|
-
return :graphical if [Aspera::Environment::OS_WINDOWS,Aspera::Environment::OS_X].include?(Aspera::Environment.os)
|
20
|
+
return :graphical if [Aspera::Environment::OS_WINDOWS, Aspera::Environment::OS_X].include?(Aspera::Environment.os)
|
20
21
|
# unix family
|
21
|
-
return :graphical if ENV.
|
22
|
+
return :graphical if ENV.key?('DISPLAY') && !ENV['DISPLAY'].empty?
|
22
23
|
return :text
|
23
24
|
end
|
24
25
|
|
25
26
|
# command must be non blocking
|
26
27
|
def uri_graphical(uri)
|
27
28
|
case Aspera::Environment.os
|
28
|
-
when Aspera::Environment::OS_X
|
29
|
-
|
30
|
-
when Aspera::Environment::
|
31
|
-
return system('start explorer "' + uri.to_s + '"')
|
32
|
-
when Aspera::Environment::OS_LINUX
|
33
|
-
return system("xdg-open '#{uri}'")
|
29
|
+
when Aspera::Environment::OS_X then return system('open', uri.to_s)
|
30
|
+
when Aspera::Environment::OS_WINDOWS then return system('start', 'explorer', '"' + uri.to_s + '"')
|
31
|
+
when Aspera::Environment::OS_LINUX then return system('xdg-open', uri.to_s)
|
34
32
|
else
|
35
33
|
raise "no graphical open method for #{Aspera::Environment.os}"
|
36
34
|
end
|
37
35
|
end
|
38
|
-
|
36
|
+
|
37
|
+
def editor(file_path)
|
38
|
+
if ENV.key?('EDITOR')
|
39
|
+
system(ENV['EDITOR'], file_path.to_s)
|
40
|
+
elsif Aspera::Environment.os.eql?(Aspera::Environment::OS_WINDOWS)
|
41
|
+
system('notepad.exe', '"' + file_path.to_s + '"')
|
42
|
+
else
|
43
|
+
uri_graphical(file_path.to_s)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end # self
|
39
47
|
|
40
48
|
attr_accessor :url_method
|
41
49
|
|
@@ -56,7 +64,7 @@ module Aspera
|
|
56
64
|
puts "USER ACTION: open this:\n" + the_url.to_s.red + "\n"
|
57
65
|
end
|
58
66
|
else
|
59
|
-
raise StandardError,"unsupported url open method: #{@url_method}"
|
67
|
+
raise StandardError, "unsupported url open method: #{@url_method}"
|
60
68
|
end
|
61
69
|
end
|
62
70
|
end # OpenApplication
|