aspera-cli 4.9.0 → 4.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +20 -0
- data/CHANGELOG.md +509 -0
- data/CONTRIBUTING.md +118 -0
- data/README.md +1241 -916
- data/bin/ascli +4 -4
- data/bin/asession +11 -11
- data/docs/test_env.conf +32 -21
- data/examples/aoc.rb +4 -4
- data/examples/dascli +16 -9
- data/examples/faspex4.rb +8 -8
- data/examples/node.rb +12 -12
- data/examples/server.rb +10 -10
- data/lib/aspera/aoc.rb +273 -266
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formater.rb +64 -64
- data/lib/aspera/cli/info.rb +2 -2
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +14 -19
- data/lib/aspera/cli/main.rb +66 -67
- data/lib/aspera/cli/manager.rb +112 -110
- data/lib/aspera/cli/plugin.rb +57 -36
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +309 -670
- data/lib/aspera/cli/plugins/ats.rb +44 -46
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +497 -378
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +112 -114
- data/lib/aspera/cli/plugins/faspex5.rb +71 -46
- data/lib/aspera/cli/plugins/node.rb +379 -283
- data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
- data/lib/aspera/cli/plugins/preview.rb +122 -114
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +30 -29
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +60 -59
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +27 -27
- data/lib/aspera/cos_node.rb +22 -20
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +35 -15
- data/lib/aspera/fasp/agent_base.rb +15 -15
- data/lib/aspera/fasp/agent_connect.rb +23 -21
- data/lib/aspera/fasp/agent_direct.rb +66 -64
- data/lib/aspera/fasp/agent_httpgw.rb +141 -78
- data/lib/aspera/fasp/agent_node.rb +23 -21
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +79 -79
- data/lib/aspera/fasp/listener.rb +1 -1
- data/lib/aspera/fasp/parameters.rb +86 -71
- data/lib/aspera/fasp/parameters.yaml +7 -4
- data/lib/aspera/fasp/resume_policy.rb +8 -8
- data/lib/aspera/fasp/transfer_spec.rb +35 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +38 -105
- data/lib/aspera/keychain/macos_security.rb +128 -57
- data/lib/aspera/log.rb +7 -7
- data/lib/aspera/nagios.rb +19 -18
- data/lib/aspera/node.rb +209 -35
- data/lib/aspera/oauth.rb +37 -36
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +16 -15
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +41 -41
- data/lib/aspera/proxy_auto_config.rb +21 -14
- data/lib/aspera/rest.rb +72 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +18 -17
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +15 -13
- data/lib/aspera/ssh.rb +11 -10
- data/lib/aspera/sync.rb +158 -44
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +14 -13
- data.tar.gz.sig +0 -0
- metadata +11 -36
- metadata.gz.sig +0 -0
data/lib/aspera/nagios.rb
CHANGED
@@ -10,32 +10,32 @@ module Aspera
|
|
10
10
|
# date offset levels
|
11
11
|
DATE_WARN_OFFSET = 2
|
12
12
|
DATE_CRIT_OFFSET = 5
|
13
|
-
private_constant :LEVELS
|
13
|
+
private_constant :LEVELS, :ADD_PREFIX, :DATE_WARN_OFFSET, :DATE_CRIT_OFFSET
|
14
14
|
|
15
15
|
# add methods to add nagios error levels, each take component name and message
|
16
16
|
LEVELS.each_index do |code|
|
17
17
|
name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
|
18
|
-
define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
|
18
|
+
define_method(name){|comp, msg|@data.push({code: code, comp: comp, msg: msg})}
|
19
19
|
end
|
20
20
|
|
21
21
|
class << self
|
22
22
|
# process results of a analysis and display status and exit with code
|
23
23
|
def process(data)
|
24
24
|
raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
|
25
|
-
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.
|
25
|
+
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.key?(c)}
|
26
26
|
res_errors = data.reject{|s|s['status'].eql?('ok')}
|
27
27
|
# keep only errors in case of problem, other ok are assumed so
|
28
28
|
data = res_errors unless res_errors.empty?
|
29
29
|
# first is most critical
|
30
|
-
data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
30
|
+
data.sort!{|a, b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
31
31
|
# build message: if multiple components: concatenate
|
32
|
-
#message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
33
|
-
message = data
|
34
|
-
map{|i|i['component']}
|
35
|
-
uniq
|
36
|
-
map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
|
37
|
-
join(', ')
|
38
|
-
tr("\n",' ')
|
32
|
+
# message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
33
|
+
message = data
|
34
|
+
.map{|i|i['component']}
|
35
|
+
.uniq
|
36
|
+
.map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
|
37
|
+
.join(', ')
|
38
|
+
.tr("\n", ' ')
|
39
39
|
status = data.first['status'].upcase
|
40
40
|
# display status for nagios
|
41
41
|
puts("#{status} - [#{message}]\n")
|
@@ -45,6 +45,7 @@ module Aspera
|
|
45
45
|
end
|
46
46
|
|
47
47
|
attr_reader :data
|
48
|
+
|
48
49
|
def initialize
|
49
50
|
@data = []
|
50
51
|
end
|
@@ -55,26 +56,26 @@ module Aspera
|
|
55
56
|
rtime = DateTime.strptime(remote_date)
|
56
57
|
diff_time = (rtime - DateTime.now).abs
|
57
58
|
diff_disp = diff_time.round(-2)
|
58
|
-
Log.log.debug
|
59
|
+
Log.log.debug{"DATE: #{remote_date} #{rtime} diff=#{diff_disp}"}
|
59
60
|
msg = "offset #{diff_disp} sec"
|
60
61
|
if diff_time >= DATE_CRIT_OFFSET
|
61
|
-
add_critical(component,msg)
|
62
|
+
add_critical(component, msg)
|
62
63
|
elsif diff_time >= DATE_WARN_OFFSET
|
63
|
-
add_warning(component,msg)
|
64
|
+
add_warning(component, msg)
|
64
65
|
else
|
65
|
-
add_ok(component,msg)
|
66
|
+
add_ok(component, msg)
|
66
67
|
end
|
67
68
|
end
|
68
69
|
|
69
70
|
def check_product_version(component, _product, version)
|
70
|
-
add_ok(component,"version #{version}")
|
71
|
-
# TODO check on database if latest version
|
71
|
+
add_ok(component, "version #{version}")
|
72
|
+
# TODO: check on database if latest version
|
72
73
|
end
|
73
74
|
|
74
75
|
# translate for display
|
75
76
|
def result
|
76
77
|
raise 'missing result' if @data.empty?
|
77
|
-
{type: :object_list,data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s,'component' => i[:comp],'message' => i[:msg]}}}
|
78
|
+
{type: :object_list, data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
|
78
79
|
end
|
79
80
|
end
|
80
81
|
end
|
data/lib/aspera/node.rb
CHANGED
@@ -9,22 +9,23 @@ require 'zlib'
|
|
9
9
|
require 'base64'
|
10
10
|
|
11
11
|
module Aspera
|
12
|
-
# Provides additional functions using node API
|
13
|
-
class Node < Rest
|
12
|
+
# Provides additional functions using node API with gen4 extensions (access keys)
|
13
|
+
class Node < Aspera::Rest
|
14
14
|
# permissions
|
15
15
|
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
16
16
|
# prefix for ruby code for filter
|
17
17
|
MATCH_EXEC_PREFIX = 'exec:'
|
18
|
+
X_ASPERA_ACCESSKEY = 'X-Aspera-AccessKey'
|
19
|
+
PATH_SEPARATOR = '/'
|
18
20
|
|
19
21
|
# register node special token decoder
|
20
22
|
Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
|
21
23
|
|
24
|
+
# class instance variable, access with accessors on class
|
25
|
+
@use_standard_ports = true
|
26
|
+
|
22
27
|
class << self
|
23
|
-
|
24
|
-
Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, "\
|
25
|
-
"but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
26
|
-
ts['token'] = Rest.basic_creds(ak,secret)
|
27
|
-
end
|
28
|
+
attr_accessor :use_standard_ports
|
28
29
|
|
29
30
|
# for access keys: provide expression to match entry in folder
|
30
31
|
# if no prefix: regex
|
@@ -39,49 +40,222 @@ module Aspera
|
|
39
40
|
end
|
40
41
|
end
|
41
42
|
|
42
|
-
|
43
|
-
|
44
|
-
|
43
|
+
REQUIRED_APP_INFO_FIELDS = %i[node_info app api plugin].freeze
|
44
|
+
REQUIRED_APP_API_METHODS = %i[node_id_to_api add_ts_tags].freeze
|
45
|
+
private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
|
46
|
+
|
47
|
+
attr_reader :app_info
|
48
|
+
|
49
|
+
# @param params [Hash] Rest parameters
|
50
|
+
# @param app_info [Hash,NilClass] special processing for AoC
|
51
|
+
def initialize(params:, app_info: nil, add_tspec: nil)
|
52
|
+
super(params)
|
53
|
+
@app_info = app_info
|
54
|
+
# this is added to transfer spec, for instance to add tags (COS)
|
55
|
+
@add_tspec = add_tspec
|
56
|
+
if !@app_info.nil?
|
57
|
+
REQUIRED_APP_INFO_FIELDS.each do |field|
|
58
|
+
raise "INTERNAL ERROR: app_info lacks field #{field}" unless @app_info.key?(field)
|
59
|
+
end
|
60
|
+
REQUIRED_APP_API_METHODS.each do |method|
|
61
|
+
raise "INTERNAL ERROR: #{@app_info[:api].class} lacks method #{method}" unless @app_info[:api].respond_to?(method)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# update transfer spec with special additional tags
|
67
|
+
def add_tspec_info(tspec)
|
68
|
+
tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
|
69
|
+
return tspec
|
70
|
+
end
|
71
|
+
|
72
|
+
# @returns [Aspera::Node] a Node or nil
|
73
|
+
def node_id_to_node(node_id)
|
74
|
+
return self if !@app_info.nil? && @app_info[:node_info]['id'].eql?(node_id)
|
75
|
+
return @app_info[:api].node_id_to_api(node_id) unless @app_info.nil?
|
76
|
+
Log.log.warn{"cannot resolve link with node id #{node_id}"}
|
77
|
+
return nil
|
78
|
+
end
|
45
79
|
|
46
|
-
# recursively
|
80
|
+
# recursively browse in a folder (with non-recursive method)
|
47
81
|
# subfolders a processed if the processing method returns true
|
48
|
-
# @param
|
49
|
-
# @param
|
50
|
-
#
|
51
|
-
#
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
opt[:method] ||= :process_entry
|
59
|
-
raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
|
60
|
-
Log.log.debug("crawl #{opt}")
|
61
|
-
#top_info=read("files/#{opt[:top_file_id]}")[:data]
|
62
|
-
folders_to_explore = [{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
|
63
|
-
Log.dump(:folders_to_explore,folders_to_explore)
|
64
|
-
while !folders_to_explore.empty?
|
82
|
+
# @param state [Object] state object sent to processing method
|
83
|
+
# @param method [Symbol] processing method name
|
84
|
+
# @param top_file_id [String] file id to start at (default = access key root file id)
|
85
|
+
# @param top_file_path [String] path of top folder (default = /)
|
86
|
+
def process_folder_tree(state:, method:, top_file_id:, top_file_path: '/')
|
87
|
+
raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
|
88
|
+
raise "INTERNAL ERROR: Missing method #{method}" unless respond_to?(method)
|
89
|
+
folders_to_explore = [{id: top_file_id, relpath: top_file_path}]
|
90
|
+
Log.dump(:folders_to_explore, folders_to_explore)
|
91
|
+
until folders_to_explore.empty?
|
65
92
|
current_item = folders_to_explore.shift
|
66
|
-
Log.log.debug
|
93
|
+
Log.log.debug{"searching #{current_item[:relpath]}".bg_green}
|
67
94
|
# get folder content
|
68
95
|
folder_contents =
|
69
96
|
begin
|
70
97
|
read("files/#{current_item[:id]}/files")[:data]
|
71
98
|
rescue StandardError => e
|
72
|
-
Log.log.warn
|
99
|
+
Log.log.warn{"#{current_item[:relpath]}: #{e.class} #{e.message}"}
|
73
100
|
[]
|
74
101
|
end
|
75
|
-
Log.dump(:folder_contents,folder_contents)
|
102
|
+
Log.dump(:folder_contents, folder_contents)
|
76
103
|
folder_contents.each do |entry|
|
77
|
-
relative_path = File.join(current_item[:relpath],entry['name'])
|
78
|
-
Log.log.debug
|
104
|
+
relative_path = File.join(current_item[:relpath], entry['name'])
|
105
|
+
Log.log.debug{"looking #{relative_path}".bg_green}
|
106
|
+
# continue only if method returns true
|
107
|
+
next unless send(method, entry, relative_path, state)
|
79
108
|
# entry type is file, folder or link
|
80
|
-
|
81
|
-
|
109
|
+
case entry['type']
|
110
|
+
when 'folder'
|
111
|
+
folders_to_explore.push({id: entry['id'], relpath: relative_path})
|
112
|
+
when 'link'
|
113
|
+
node_id_to_node(entry['target_node_id'])&.process_folder_tree(
|
114
|
+
state: state,
|
115
|
+
method: method,
|
116
|
+
top_file_id: entry['target_id'],
|
117
|
+
top_file_path: relative_path)
|
82
118
|
end
|
83
119
|
end
|
84
120
|
end
|
121
|
+
end # process_folder_tree
|
122
|
+
|
123
|
+
# processing method to resolve a file path to id
|
124
|
+
# @returns true if processing need to continue
|
125
|
+
def process_resolve_node_path(entry, _path, state)
|
126
|
+
# stop digging here if not in right path
|
127
|
+
return false unless entry['name'].eql?(state[:path].first)
|
128
|
+
# ok it matches, so we remove the match
|
129
|
+
state[:path].shift
|
130
|
+
case entry['type']
|
131
|
+
when 'file'
|
132
|
+
# file must be terminal
|
133
|
+
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
134
|
+
# it's terminal, we found it
|
135
|
+
state[:result] = {api: self, file_id: entry['id']}
|
136
|
+
return false
|
137
|
+
when 'folder'
|
138
|
+
if state[:path].empty?
|
139
|
+
# we found it
|
140
|
+
state[:result] = {api: self, file_id: entry['id']}
|
141
|
+
return false
|
142
|
+
end
|
143
|
+
when 'link'
|
144
|
+
if state[:path].empty?
|
145
|
+
# we found it
|
146
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
147
|
+
raise 'cannot resolve link' if other_node.nil?
|
148
|
+
state[:result] = {api: other_node, file_id: entry['target_id']}
|
149
|
+
return false
|
150
|
+
end
|
151
|
+
else
|
152
|
+
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
153
|
+
end
|
154
|
+
# continue to dig folder
|
155
|
+
return true
|
156
|
+
end
|
157
|
+
|
158
|
+
# Navigate the path from given file id
|
159
|
+
# @param id initial file id
|
160
|
+
# @param path file path
|
161
|
+
# @return {.api,.file_id}
|
162
|
+
def resolve_api_fid(top_file_id, path)
|
163
|
+
raise 'file id shall be String' unless top_file_id.is_a?(String)
|
164
|
+
path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
|
165
|
+
return {api: self, file_id: top_file_id} if path_elements.empty?
|
166
|
+
resolve_state = {path: path_elements, result: nil}
|
167
|
+
process_folder_tree(state: resolve_state, method: :process_resolve_node_path, top_file_id: top_file_id)
|
168
|
+
raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
|
169
|
+
return resolve_state[:result]
|
170
|
+
end
|
171
|
+
|
172
|
+
# add entry to list if test block is success
|
173
|
+
def process_find_files(entry, path, state)
|
174
|
+
begin
|
175
|
+
# add to result if match filter
|
176
|
+
state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
|
177
|
+
# process link
|
178
|
+
if entry[:type].eql?('link')
|
179
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
180
|
+
other_node.process_folder_tree(state: state, method: process_find_files, top_file_id: entry['target_id'], top_file_path: path)
|
181
|
+
end
|
182
|
+
rescue StandardError => e
|
183
|
+
Log.log.error{"#{path}: #{e.message}"}
|
184
|
+
end
|
185
|
+
# process all folders
|
186
|
+
return true
|
187
|
+
end
|
188
|
+
|
189
|
+
def find_files(top_file_id, test_block)
|
190
|
+
Log.log.debug{"find_files: fileid=#{top_file_id}"}
|
191
|
+
find_state = {found: [], test_block: test_block}
|
192
|
+
process_folder_tree(state: find_state, method: :process_find_files, top_file_id: top_file_id)
|
193
|
+
return find_state[:found]
|
194
|
+
end
|
195
|
+
|
196
|
+
# Create transfer spec for gen4
|
197
|
+
def transfer_spec_gen4(file_id, direction, ts_merge=nil)
|
198
|
+
ak_name = nil
|
199
|
+
ak_token = nil
|
200
|
+
case params[:auth][:type]
|
201
|
+
when :basic
|
202
|
+
ak_name = params[:auth][:username]
|
203
|
+
when :oauth2
|
204
|
+
ak_name = params[:headers][X_ASPERA_ACCESSKEY]
|
205
|
+
token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
|
206
|
+
ak_token = token_generation_lambda.call(false) # first time, use cache
|
207
|
+
@app_info[:plugin].transfer.token_regenerator = token_generation_lambda unless @app_info.nil?
|
208
|
+
else raise "Unsupported auth method for node gen4: #{params[:auth][:type]}"
|
209
|
+
end
|
210
|
+
transfer_spec = {
|
211
|
+
'direction' => direction,
|
212
|
+
'token' => ak_token,
|
213
|
+
'tags' => {
|
214
|
+
'aspera' => {
|
215
|
+
'node' => {
|
216
|
+
'access_key' => ak_name,
|
217
|
+
'file_id' => file_id
|
218
|
+
} # node
|
219
|
+
} # aspera
|
220
|
+
} # tags
|
221
|
+
}
|
222
|
+
# add specials tags (cos)
|
223
|
+
add_tspec_info(transfer_spec)
|
224
|
+
transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
|
225
|
+
# add application specific tags (AoC)
|
226
|
+
the_app = app_info
|
227
|
+
the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
|
228
|
+
# add basic token
|
229
|
+
if transfer_spec['token'].nil?
|
230
|
+
ts_basic_token(transfer_spec)
|
231
|
+
end
|
232
|
+
# add remote host info
|
233
|
+
if self.class.use_standard_ports
|
234
|
+
# get default TCP/UDP ports and transfer user
|
235
|
+
transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
|
236
|
+
# by default: same address as node API
|
237
|
+
transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
|
238
|
+
if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
|
239
|
+
transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
|
240
|
+
end
|
241
|
+
else
|
242
|
+
# retrieve values from API
|
243
|
+
std_t_spec = create(
|
244
|
+
'files/download_setup',
|
245
|
+
{transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
|
246
|
+
)[:data]['transfer_specs'].first['transfer_spec']
|
247
|
+
# copy some parts
|
248
|
+
%w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].each {|i| transfer_spec[i] = std_t_spec[i] if std_t_spec.key?(i)}
|
249
|
+
end
|
250
|
+
return transfer_spec
|
251
|
+
end
|
252
|
+
|
253
|
+
# set basic token in transfer spec
|
254
|
+
def ts_basic_token(ts)
|
255
|
+
Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
|
256
|
+
unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
257
|
+
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
258
|
+
ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
|
85
259
|
end
|
86
260
|
end
|
87
261
|
end
|
data/lib/aspera/oauth.rb
CHANGED
@@ -35,7 +35,7 @@ module Aspera
|
|
35
35
|
# a prefix for persistency of tokens (simplify garbage collect)
|
36
36
|
PERSIST_CATEGORY_TOKEN = 'token'
|
37
37
|
|
38
|
-
private_constant :JWT_NOTBEFORE_OFFSET_SEC
|
38
|
+
private_constant :JWT_NOTBEFORE_OFFSET_SEC, :JWT_EXPIRY_OFFSET_SEC, :TOKEN_CACHE_EXPIRY_SEC, :PERSIST_CATEGORY_TOKEN, :TOKEN_EXPIRATION_GUARD_SEC
|
39
39
|
|
40
40
|
# persistency manager
|
41
41
|
@persist = nil
|
@@ -48,7 +48,7 @@ module Aspera
|
|
48
48
|
def persist_mgr=(manager)
|
49
49
|
@persist = manager
|
50
50
|
# cleanup expired tokens
|
51
|
-
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_CACHE_EXPIRY_SEC)
|
51
|
+
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN, TOKEN_CACHE_EXPIRY_SEC)
|
52
52
|
end
|
53
53
|
|
54
54
|
def persist_mgr
|
@@ -56,7 +56,7 @@ module Aspera
|
|
56
56
|
Log.log.debug('Not using persistency') # (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)
|
57
57
|
# create NULL persistency class
|
58
58
|
@persist = Class.new do
|
59
|
-
def get(_x);nil;end;def delete(_x);nil;end;def put(_x,_y);nil;end;def garbage_collect(_x,_y);nil;end # rubocop:disable Layout/EmptyLineBetweenDefs
|
59
|
+
def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Layout/EmptyLineBetweenDefs, Style/Semicolon, Layout/LineLength
|
60
60
|
end.new
|
61
61
|
end
|
62
62
|
return @persist
|
@@ -64,7 +64,7 @@ module Aspera
|
|
64
64
|
|
65
65
|
# delete all existing tokens
|
66
66
|
def flush_tokens
|
67
|
-
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
|
67
|
+
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN, nil)
|
68
68
|
end
|
69
69
|
|
70
70
|
# register a bearer token decoder, mainly to inspect expiry date
|
@@ -94,24 +94,24 @@ module Aspera
|
|
94
94
|
|
95
95
|
# @return one of the registered creators for the given create type
|
96
96
|
def token_creator(id)
|
97
|
-
raise "token creator type unknown: #{id}/#{id.class}" unless @create_handlers.
|
97
|
+
raise "token creator type unknown: #{id}/#{id.class}" unless @create_handlers.key?(id)
|
98
98
|
@create_handlers[id]
|
99
99
|
end
|
100
100
|
|
101
101
|
# list of identifiers foundn in creation parameters that can be used to uniquely identify the token
|
102
102
|
def id_creator(id)
|
103
|
-
raise "id creator type unknown: #{id}/#{id.class}" unless @id_handlers.
|
103
|
+
raise "id creator type unknown: #{id}/#{id.class}" unless @id_handlers.key?(id)
|
104
104
|
@id_handlers[id]
|
105
105
|
end
|
106
106
|
end # self
|
107
107
|
|
108
108
|
# JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
|
109
|
-
register_decoder lambda { |token| parts = token.split('.'); raise 'not aoc token' unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))}
|
109
|
+
register_decoder lambda { |token| parts = token.split('.'); raise 'not aoc token' unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))} # rubocop:disable Style/Semicolon, Layout/LineLength
|
110
110
|
|
111
111
|
# generic token creation, parameters are provided in :generic
|
112
|
-
register_token_creator :generic,lambda { |oauth|
|
112
|
+
register_token_creator :generic, lambda { |oauth|
|
113
113
|
return oauth.create_token(oauth.sparams)
|
114
|
-
},lambda { |oauth|
|
114
|
+
}, lambda { |oauth|
|
115
115
|
return [
|
116
116
|
oauth.sparams[:grant_type]&.split(':')&.last,
|
117
117
|
oauth.sparams[:apikey],
|
@@ -120,12 +120,13 @@ module Aspera
|
|
120
120
|
}
|
121
121
|
|
122
122
|
# Authentication using Web browser
|
123
|
-
register_token_creator :web,lambda { |oauth|
|
123
|
+
register_token_creator :web, lambda { |oauth|
|
124
124
|
random_state = SecureRandom.uuid # used to check later
|
125
|
-
login_page_url = Rest.build_uri(
|
125
|
+
login_page_url = Rest.build_uri(
|
126
|
+
"#{oauth.api.params[:base_url]}/#{oauth.sparams[:path_authorize]}",
|
126
127
|
oauth.optional_scope_client_id.merge(response_type: 'code', redirect_uri: oauth.sparams[:redirect_uri], state: random_state))
|
127
128
|
# here, we need a human to authorize on a web page
|
128
|
-
Log.log.info
|
129
|
+
Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
|
129
130
|
# start a web server to receive request code
|
130
131
|
webserver = WebAuth.new(oauth.sparams[:redirect_uri])
|
131
132
|
# start browser on login page
|
@@ -138,17 +139,17 @@ module Aspera
|
|
138
139
|
grant_type: 'authorization_code',
|
139
140
|
code: received_params['code'],
|
140
141
|
redirect_uri: oauth.sparams[:redirect_uri]))
|
141
|
-
},lambda { |_oauth|
|
142
|
+
}, lambda { |_oauth|
|
142
143
|
return []
|
143
144
|
}
|
144
145
|
|
145
146
|
# Authentication using private key
|
146
|
-
register_token_creator :jwt,lambda { |oauth|
|
147
|
+
register_token_creator :jwt, lambda { |oauth|
|
147
148
|
# https://tools.ietf.org/html/rfc7523
|
148
149
|
# https://tools.ietf.org/html/rfc7519
|
149
150
|
require 'jwt'
|
150
151
|
seconds_since_epoch = Time.new.to_i
|
151
|
-
Log.log.info
|
152
|
+
Log.log.info{"seconds=#{seconds_since_epoch}"}
|
152
153
|
raise 'missing JWT payload' unless oauth.sparams[:payload].is_a?(Hash)
|
153
154
|
jwt_payload = {
|
154
155
|
exp: seconds_since_epoch + JWT_EXPIRY_OFFSET_SEC, # expiration time
|
@@ -156,14 +157,14 @@ module Aspera
|
|
156
157
|
iat: seconds_since_epoch, # issued at
|
157
158
|
jti: SecureRandom.uuid # JWT id
|
158
159
|
}.merge(oauth.sparams[:payload])
|
159
|
-
Log.log.debug
|
160
|
+
Log.log.debug{"JWT jwt_payload=[#{jwt_payload}]"}
|
160
161
|
rsa_private = oauth.sparams[:private_key_obj] # type: OpenSSL::PKey::RSA
|
161
|
-
Log.log.debug
|
162
|
+
Log.log.debug{"private=[#{rsa_private}]"}
|
162
163
|
assertion = JWT.encode(jwt_payload, rsa_private, 'RS256', oauth.sparams[:headers] || {})
|
163
|
-
Log.log.debug
|
164
|
+
Log.log.debug{"assertion=[#{assertion}]"}
|
164
165
|
return oauth.create_token(oauth.optional_scope_client_id.merge(grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', assertion: assertion))
|
165
|
-
},lambda { |oauth|
|
166
|
-
return [oauth.sparams.dig(:payload
|
166
|
+
}, lambda { |oauth|
|
167
|
+
return [oauth.sparams.dig(:payload, :sub)]
|
167
168
|
}
|
168
169
|
|
169
170
|
attr_reader :gparams, :sparams, :api
|
@@ -186,14 +187,14 @@ module Aspera
|
|
186
187
|
# :web:path_authorize [D] for type :web
|
187
188
|
# :generic [M] for type :generic
|
188
189
|
def initialize(a_params)
|
189
|
-
Log.log.debug
|
190
|
+
Log.log.debug{"auth=#{a_params}"}
|
190
191
|
# replace default values
|
191
192
|
@gparams = DEFAULT_CREATE_PARAMS.deep_merge(a_params)
|
192
193
|
# check that type is known
|
193
194
|
self.class.token_creator(@gparams[:crtype])
|
194
195
|
# specific parameters for the creation type
|
195
|
-
@sparams
|
196
|
-
if @gparams[:crtype].eql?(:web) && @sparams.
|
196
|
+
@sparams = @gparams[@gparams[:crtype]]
|
197
|
+
if @gparams[:crtype].eql?(:web) && @sparams.key?(:redirect_uri)
|
197
198
|
uri = URI.parse(@sparams[:redirect_uri])
|
198
199
|
raise 'redirect_uri scheme must be http or https' unless %w[http https].include?(uri.scheme)
|
199
200
|
raise 'redirect_uri must have a port' if uri.port.nil?
|
@@ -203,14 +204,14 @@ module Aspera
|
|
203
204
|
base_url: @gparams[:base_url],
|
204
205
|
redirect_max: 2
|
205
206
|
}
|
206
|
-
rest_params[:auth] = a_params[:auth] if a_params.
|
207
|
+
rest_params[:auth] = a_params[:auth] if a_params.key?(:auth)
|
207
208
|
@api = Rest.new(rest_params)
|
208
209
|
# if needed use from api
|
209
210
|
@gparams.delete(:base_url)
|
210
211
|
@gparams.delete(:auth)
|
211
212
|
@gparams.delete(@gparams[:crtype])
|
212
|
-
Log.dump(:gparams
|
213
|
-
Log.dump(:sparams
|
213
|
+
Log.dump(:gparams, @gparams)
|
214
|
+
Log.dump(:sparams, @sparams)
|
214
215
|
end
|
215
216
|
|
216
217
|
public
|
@@ -226,7 +227,7 @@ module Aspera
|
|
226
227
|
|
227
228
|
# @return Hash with optional general parameters
|
228
229
|
def optional_scope_client_id(add_secret: false)
|
229
|
-
call_params={}
|
230
|
+
call_params = {}
|
230
231
|
call_params[:scope] = @gparams[:scope] unless @gparams[:scope].nil?
|
231
232
|
call_params[:client_id] = @gparams[:client_id] unless @gparams[:client_id].nil?
|
232
233
|
call_params[:client_secret] = @gparams[:client_secret] if add_secret && !@gparams[:client_id].nil?
|
@@ -254,7 +255,7 @@ module Aspera
|
|
254
255
|
# `direct` agent is equipped with refresh code
|
255
256
|
if !use_refresh_token && !token_data.nil?
|
256
257
|
decoded_token = self.class.decode_token(token_data[@gparams[:token_field]])
|
257
|
-
Log.dump('decoded_token',decoded_token) unless decoded_token.nil?
|
258
|
+
Log.dump('decoded_token', decoded_token) unless decoded_token.nil?
|
258
259
|
if decoded_token.is_a?(Hash)
|
259
260
|
expires_at_sec =
|
260
261
|
if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
|
@@ -262,13 +263,13 @@ module Aspera
|
|
262
263
|
end
|
263
264
|
# force refresh if we see a token too close from expiration
|
264
265
|
use_refresh_token = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < TOKEN_EXPIRATION_GUARD_SEC
|
265
|
-
Log.log.debug
|
266
|
+
Log.log.debug{"Expiration: #{expires_at_sec} / #{use_refresh_token}"}
|
266
267
|
end
|
267
268
|
end
|
268
269
|
|
269
270
|
# an API was already called, but failed, we need to regenerate or refresh
|
270
271
|
if use_refresh_token
|
271
|
-
if token_data.is_a?(Hash) && token_data.
|
272
|
+
if token_data.is_a?(Hash) && token_data.key?('refresh_token')
|
272
273
|
# save possible refresh token, before deleting the cache
|
273
274
|
refresh_token = token_data['refresh_token']
|
274
275
|
end
|
@@ -277,17 +278,17 @@ module Aspera
|
|
277
278
|
token_data = nil
|
278
279
|
# lets try the existing refresh token
|
279
280
|
if !refresh_token.nil?
|
280
|
-
Log.log.info
|
281
|
+
Log.log.info{"refresh=[#{refresh_token}]".bg_green}
|
281
282
|
# try to refresh
|
282
283
|
# note: AoC admin token has no refresh, and lives by default 1800secs
|
283
|
-
resp = create_token(optional_scope_client_id.merge(grant_type: 'refresh_token',refresh_token: refresh_token))
|
284
|
+
resp = create_token(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
|
284
285
|
if resp[:http].code.start_with?('2')
|
285
286
|
# save only if success
|
286
287
|
json_data = resp[:http].body
|
287
288
|
token_data = JSON.parse(json_data)
|
288
|
-
self.class.persist_mgr.put(token_id,json_data)
|
289
|
+
self.class.persist_mgr.put(token_id, json_data)
|
289
290
|
else
|
290
|
-
Log.log.debug
|
291
|
+
Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
|
291
292
|
end
|
292
293
|
end
|
293
294
|
end
|
@@ -297,9 +298,9 @@ module Aspera
|
|
297
298
|
resp = self.class.token_creator(@gparams[:crtype]).call(self)
|
298
299
|
json_data = resp[:http].body
|
299
300
|
token_data = JSON.parse(json_data)
|
300
|
-
self.class.persist_mgr.put(token_id,json_data)
|
301
|
+
self.class.persist_mgr.put(token_id, json_data)
|
301
302
|
end # if ! in_cache
|
302
|
-
raise "API error: No such field in answer: #{@gparams[:token_field]}" unless token_data.
|
303
|
+
raise "API error: No such field in answer: #{@gparams[:token_field]}" unless token_data.key?(@gparams[:token_field])
|
303
304
|
# ok we shall have a token here
|
304
305
|
return 'Bearer ' + token_data[@gparams[:token_field]]
|
305
306
|
end
|
@@ -12,30 +12,38 @@ module Aspera
|
|
12
12
|
class OpenApplication
|
13
13
|
include Singleton
|
14
14
|
class << self
|
15
|
+
USER_INTERFACES = %i[text graphical].freeze
|
15
16
|
# User Interfaces
|
16
|
-
def user_interfaces;
|
17
|
+
def user_interfaces; USER_INTERFACES; end
|
17
18
|
|
18
19
|
def default_gui_mode
|
19
|
-
return :graphical if [Aspera::Environment::OS_WINDOWS,Aspera::Environment::OS_X].include?(Aspera::Environment.os)
|
20
|
+
return :graphical if [Aspera::Environment::OS_WINDOWS, Aspera::Environment::OS_X].include?(Aspera::Environment.os)
|
20
21
|
# unix family
|
21
|
-
return :graphical if ENV.
|
22
|
+
return :graphical if ENV.key?('DISPLAY') && !ENV['DISPLAY'].empty?
|
22
23
|
return :text
|
23
24
|
end
|
24
25
|
|
25
26
|
# command must be non blocking
|
26
27
|
def uri_graphical(uri)
|
27
28
|
case Aspera::Environment.os
|
28
|
-
when Aspera::Environment::OS_X
|
29
|
-
|
30
|
-
when Aspera::Environment::
|
31
|
-
return system('start explorer "' + uri.to_s + '"')
|
32
|
-
when Aspera::Environment::OS_LINUX
|
33
|
-
return system("xdg-open '#{uri}'")
|
29
|
+
when Aspera::Environment::OS_X then return system('open', uri.to_s)
|
30
|
+
when Aspera::Environment::OS_WINDOWS then return system('start', 'explorer', '"' + uri.to_s + '"')
|
31
|
+
when Aspera::Environment::OS_LINUX then return system('xdg-open', uri.to_s)
|
34
32
|
else
|
35
33
|
raise "no graphical open method for #{Aspera::Environment.os}"
|
36
34
|
end
|
37
35
|
end
|
38
|
-
|
36
|
+
|
37
|
+
def editor(file_path)
|
38
|
+
if ENV.key?('EDITOR')
|
39
|
+
system(ENV['EDITOR'], file_path.to_s)
|
40
|
+
elsif Aspera::Environment.os.eql?(Aspera::Environment::OS_WINDOWS)
|
41
|
+
system('notepad.exe', '"' + file_path.to_s + '"')
|
42
|
+
else
|
43
|
+
uri_graphical(file_path.to_s)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end # self
|
39
47
|
|
40
48
|
attr_accessor :url_method
|
41
49
|
|
@@ -56,7 +64,7 @@ module Aspera
|
|
56
64
|
puts "USER ACTION: open this:\n" + the_url.to_s.red + "\n"
|
57
65
|
end
|
58
66
|
else
|
59
|
-
raise StandardError,"unsupported url open method: #{@url_method}"
|
67
|
+
raise StandardError, "unsupported url open method: #{@url_method}"
|
60
68
|
end
|
61
69
|
end
|
62
70
|
end # OpenApplication
|