aspera-cli 4.10.0 → 4.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +19 -0
- data/CHANGELOG.md +528 -0
- data/CONTRIBUTING.md +143 -0
- data/README.md +977 -589
- data/bin/ascli +4 -4
- data/bin/asession +12 -12
- data/docs/test_env.conf +29 -19
- data/examples/aoc.rb +6 -6
- data/examples/dascli +18 -16
- data/examples/faspex4.rb +15 -15
- data/examples/node.rb +12 -12
- data/examples/proxy.pac +2 -2
- data/examples/server.rb +12 -12
- data/lib/aspera/aoc.rb +344 -272
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +9 -9
- data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +16 -21
- data/lib/aspera/cli/main.rb +72 -73
- data/lib/aspera/cli/manager.rb +112 -112
- data/lib/aspera/cli/plugin.rb +68 -48
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +322 -720
- data/lib/aspera/cli/plugins/ats.rb +50 -52
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +514 -410
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +134 -136
- data/lib/aspera/cli/plugins/faspex5.rb +235 -70
- data/lib/aspera/cli/plugins/node.rb +378 -309
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
- data/lib/aspera/cli/plugins/preview.rb +129 -120
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +77 -52
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +61 -61
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +78 -74
- data/lib/aspera/cos_node.rb +31 -29
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +17 -15
- data/lib/aspera/fasp/agent_connect.rb +34 -32
- data/lib/aspera/fasp/agent_direct.rb +70 -73
- data/lib/aspera/fasp/agent_httpgw.rb +79 -74
- data/lib/aspera/fasp/agent_node.rb +26 -26
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +80 -80
- data/lib/aspera/fasp/listener.rb +2 -2
- data/lib/aspera/fasp/parameters.rb +103 -92
- data/lib/aspera/fasp/parameters.yaml +313 -214
- data/lib/aspera/fasp/resume_policy.rb +10 -10
- data/lib/aspera/fasp/transfer_spec.rb +22 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +80 -159
- data/lib/aspera/faspex_postproc.rb +77 -0
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/nagios.rb +24 -23
- data/lib/aspera/node.rb +217 -38
- data/lib/aspera/oauth.rb +78 -74
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +63 -63
- data/lib/aspera/proxy_auto_config.rb +19 -19
- data/lib/aspera/rest.rb +65 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +22 -21
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +17 -14
- data/lib/aspera/ssh.rb +15 -14
- data/lib/aspera/sync.rb +177 -62
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +13 -64
- data/lib/aspera/web_server_simple.rb +76 -0
- data.tar.gz.sig +0 -0
- metadata +11 -6
- metadata.gz.sig +0 -0
@@ -10,10 +10,10 @@ module Aspera
|
|
10
10
|
# keychain based on macOS keychain, using `security` cmmand line
|
11
11
|
class Keychain
|
12
12
|
DOMAINS = %i[user system common dynamic].freeze
|
13
|
-
LIST_OPTIONS={
|
13
|
+
LIST_OPTIONS = {
|
14
14
|
domain: :c
|
15
15
|
}
|
16
|
-
ADD_PASS_OPTIONS={
|
16
|
+
ADD_PASS_OPTIONS = {
|
17
17
|
account: :a,
|
18
18
|
creator: :c,
|
19
19
|
type: :C,
|
@@ -32,7 +32,7 @@ module Aspera
|
|
32
32
|
getpass: :g
|
33
33
|
}.freeze
|
34
34
|
class << self
|
35
|
-
def execute(command,options=nil,supported=nil,lastopt=nil)
|
35
|
+
def execute(command, options=nil, supported=nil, lastopt=nil)
|
36
36
|
url = options&.delete(:url)
|
37
37
|
if !url.nil?
|
38
38
|
uri = URI.parse(url)
|
@@ -40,20 +40,20 @@ module Aspera
|
|
40
40
|
options[:protocol] = 'htps'
|
41
41
|
raise 'host required in URL' if uri.host.nil?
|
42
42
|
options[:server] = uri.host
|
43
|
-
options[:path] = uri.path unless ['','/'].include?(uri.path)
|
43
|
+
options[:path] = uri.path unless ['', '/'].include?(uri.path)
|
44
44
|
options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
|
45
45
|
end
|
46
|
-
cmd=['security',command]
|
47
|
-
options&.each do |k,v|
|
48
|
-
raise "unknown option: #{k}" unless supported.
|
46
|
+
cmd = ['security', command]
|
47
|
+
options&.each do |k, v|
|
48
|
+
raise "unknown option: #{k}" unless supported.key?(k)
|
49
49
|
next if v.nil?
|
50
50
|
cmd.push("-#{supported[k]}")
|
51
51
|
cmd.push(v.shellescape) unless v.empty?
|
52
52
|
end
|
53
53
|
cmd.push(lastopt) unless lastopt.nil?
|
54
|
-
Log.log.debug
|
55
|
-
result
|
56
|
-
Log.log.debug
|
54
|
+
Log.log.debug{"executing>>#{cmd.join(' ')}"}
|
55
|
+
result = %x(#{cmd.join(' ')} 2>&1)
|
56
|
+
Log.log.debug{"result>>[#{result}]"}
|
57
57
|
return result
|
58
58
|
end
|
59
59
|
|
@@ -70,8 +70,8 @@ module Aspera
|
|
70
70
|
end
|
71
71
|
|
72
72
|
def list(options={})
|
73
|
-
raise ArgumentError,"Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
|
74
|
-
keychains(execute('list-keychains',options,LIST_OPTIONS))
|
73
|
+
raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
|
74
|
+
keychains(execute('list-keychains', options, LIST_OPTIONS))
|
75
75
|
end
|
76
76
|
|
77
77
|
def by_name(name)
|
@@ -88,15 +88,15 @@ module Aspera
|
|
88
88
|
[string].pack('H*').force_encoding('UTF-8')
|
89
89
|
end
|
90
90
|
|
91
|
-
def password(operation,passtype,options)
|
91
|
+
def password(operation, passtype, options)
|
92
92
|
raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
|
93
93
|
raise "wrong passtype: #{passtype}" unless %i[generic internet].include?(passtype)
|
94
94
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
95
|
-
missing=(operation.eql?(:add) ? %i[account service password] : %i[label])-options.keys
|
95
|
+
missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
|
96
96
|
raise "missing options: #{missing}" unless missing.empty?
|
97
|
-
options[:getpass]='' if operation.eql?(:find)
|
98
|
-
output=self.class.execute("#{operation}-#{passtype}-password",options,ADD_PASS_OPTIONS
|
99
|
-
raise output.gsub(/^.*: /,'') if output.start_with?('security: ')
|
97
|
+
options[:getpass] = '' if operation.eql?(:find)
|
98
|
+
output = self.class.execute("#{operation}-#{passtype}-password", options, ADD_PASS_OPTIONS, @path)
|
99
|
+
raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
|
100
100
|
return nil unless operation.eql?(:find)
|
101
101
|
attributes = {}
|
102
102
|
output.split("\n").each do |line|
|
@@ -121,7 +121,7 @@ module Aspera
|
|
121
121
|
end
|
122
122
|
|
123
123
|
class MacosSystem
|
124
|
-
def initialize(name=nil,
|
124
|
+
def initialize(name=nil, _password=nil)
|
125
125
|
@keychain = name.nil? ? MacosSecurity::Keychain.default_keychain : MacosSecurity::Keychain.by_name(name)
|
126
126
|
raise "no such keychain #{name}" if @keychain.nil?
|
127
127
|
end
|
@@ -130,7 +130,8 @@ module Aspera
|
|
130
130
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
131
131
|
unsupported = options.keys - %i[label username password url description]
|
132
132
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
133
|
-
@keychain.password(
|
133
|
+
@keychain.password(
|
134
|
+
:add, :generic, service: options[:label],
|
134
135
|
account: options[:username] || 'none', password: options[:password], comment: options[:description])
|
135
136
|
end
|
136
137
|
|
@@ -138,7 +139,7 @@ module Aspera
|
|
138
139
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
139
140
|
unsupported = options.keys - %i[label]
|
140
141
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
141
|
-
info = @keychain.password(:find
|
142
|
+
info = @keychain.password(:find, :generic, label: options[:label])
|
142
143
|
raise 'not found' if info.nil?
|
143
144
|
result = options.clone
|
144
145
|
result[:secret] = info['password']
|
data/lib/aspera/log.rb
CHANGED
@@ -11,28 +11,27 @@ module Aspera
|
|
11
11
|
# Singleton object for logging
|
12
12
|
class Log
|
13
13
|
include Singleton
|
14
|
+
# where logs are sent to
|
15
|
+
LOG_TYPES = %i[stderr stdout syslog].freeze
|
14
16
|
# class methods
|
15
17
|
class << self
|
16
18
|
# levels are :debug,:info,:warn,:error,fatal,:unknown
|
17
|
-
def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
|
18
|
-
|
19
|
-
# where logs are sent to
|
20
|
-
def logtypes; %i[stderr stdout syslog];end
|
19
|
+
def levels; Logger::Severity.constants.sort{|a, b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym}; end
|
21
20
|
|
22
21
|
# get the logger object of singleton
|
23
|
-
def log; instance.logger;end
|
22
|
+
def log; instance.logger; end
|
24
23
|
|
25
24
|
# dump object in debug mode
|
26
25
|
# @param name string or symbol
|
27
26
|
# @param format either pp or json format
|
28
|
-
def dump(name,object,format=:json)
|
27
|
+
def dump(name, object, format=:json)
|
29
28
|
log.debug do
|
30
29
|
result =
|
31
30
|
case format
|
32
31
|
when :json
|
33
|
-
JSON.pretty_generate(object) rescue PP.pp(object
|
32
|
+
JSON.pretty_generate(object) rescue PP.pp(object, +'')
|
34
33
|
when :ruby
|
35
|
-
PP.pp(object
|
34
|
+
PP.pp(object, +'')
|
36
35
|
else
|
37
36
|
raise 'wrong parameter, expect pp or json'
|
38
37
|
end
|
@@ -68,12 +67,13 @@ module Aspera
|
|
68
67
|
end
|
69
68
|
|
70
69
|
# change underlying logger, but keep log level
|
71
|
-
def logger_type=(
|
70
|
+
def logger_type=(new_log_type)
|
72
71
|
current_severity_integer = @logger.level unless @logger.nil?
|
73
|
-
current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.
|
72
|
+
current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
|
74
73
|
current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
|
75
|
-
case
|
74
|
+
case new_log_type
|
76
75
|
when :stderr
|
76
|
+
# typed: Logger
|
77
77
|
@logger = Logger.new($stderr)
|
78
78
|
when :stdout
|
79
79
|
@logger = Logger.new($stdout)
|
@@ -81,10 +81,10 @@ module Aspera
|
|
81
81
|
require 'syslog/logger'
|
82
82
|
@logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
|
83
83
|
else
|
84
|
-
raise "unknown log type: #{
|
84
|
+
raise "unknown log type: #{new_log_type.class} #{new_log_type}"
|
85
85
|
end
|
86
86
|
@logger.level = current_severity_integer
|
87
|
-
@logger_type =
|
87
|
+
@logger_type = new_log_type
|
88
88
|
# update formatter with password hiding
|
89
89
|
@logger.formatter = SecretHider.log_formatter(@logger.formatter)
|
90
90
|
end
|
data/lib/aspera/nagios.rb
CHANGED
@@ -10,32 +10,32 @@ module Aspera
|
|
10
10
|
# date offset levels
|
11
11
|
DATE_WARN_OFFSET = 2
|
12
12
|
DATE_CRIT_OFFSET = 5
|
13
|
-
private_constant :LEVELS
|
13
|
+
private_constant :LEVELS, :ADD_PREFIX, :DATE_WARN_OFFSET, :DATE_CRIT_OFFSET
|
14
14
|
|
15
15
|
# add methods to add nagios error levels, each take component name and message
|
16
16
|
LEVELS.each_index do |code|
|
17
17
|
name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
|
18
|
-
define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
|
18
|
+
define_method(name){|comp, msg|@data.push({code: code, comp: comp, msg: msg})}
|
19
19
|
end
|
20
20
|
|
21
21
|
class << self
|
22
22
|
# process results of a analysis and display status and exit with code
|
23
23
|
def process(data)
|
24
24
|
raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
|
25
|
-
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.
|
25
|
+
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.key?(c)}
|
26
26
|
res_errors = data.reject{|s|s['status'].eql?('ok')}
|
27
27
|
# keep only errors in case of problem, other ok are assumed so
|
28
28
|
data = res_errors unless res_errors.empty?
|
29
29
|
# first is most critical
|
30
|
-
data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
30
|
+
data.sort!{|a, b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
31
31
|
# build message: if multiple components: concatenate
|
32
|
-
#message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
33
|
-
message = data
|
34
|
-
map{|i|i['component']}
|
35
|
-
uniq
|
36
|
-
map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
|
37
|
-
join(', ')
|
38
|
-
tr("\n",' ')
|
32
|
+
# message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
33
|
+
message = data
|
34
|
+
.map{|i|i['component']}
|
35
|
+
.uniq
|
36
|
+
.map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
|
37
|
+
.join(', ')
|
38
|
+
.tr("\n", ' ')
|
39
39
|
status = data.first['status'].upcase
|
40
40
|
# display status for nagios
|
41
41
|
puts("#{status} - [#{message}]\n")
|
@@ -45,36 +45,37 @@ module Aspera
|
|
45
45
|
end
|
46
46
|
|
47
47
|
attr_reader :data
|
48
|
+
|
48
49
|
def initialize
|
49
50
|
@data = []
|
50
51
|
end
|
51
52
|
|
52
|
-
#
|
53
|
+
# compare remote time with local time
|
53
54
|
def check_time_offset(remote_date, component)
|
54
55
|
# check date if specified : 2015-10-13T07:32:01Z
|
55
|
-
|
56
|
-
diff_time = (
|
57
|
-
|
58
|
-
Log.log.debug
|
59
|
-
msg = "offset #{
|
56
|
+
remote_time = DateTime.strptime(remote_date)
|
57
|
+
diff_time = (remote_time - DateTime.now).abs
|
58
|
+
diff_rounded = diff_time.round(-2)
|
59
|
+
Log.log.debug{"DATE: #{remote_date} #{remote_time} diff=#{diff_rounded}"}
|
60
|
+
msg = "offset #{diff_rounded} sec"
|
60
61
|
if diff_time >= DATE_CRIT_OFFSET
|
61
|
-
add_critical(component,msg)
|
62
|
+
add_critical(component, msg)
|
62
63
|
elsif diff_time >= DATE_WARN_OFFSET
|
63
|
-
add_warning(component,msg)
|
64
|
+
add_warning(component, msg)
|
64
65
|
else
|
65
|
-
add_ok(component,msg)
|
66
|
+
add_ok(component, msg)
|
66
67
|
end
|
67
68
|
end
|
68
69
|
|
69
70
|
def check_product_version(component, _product, version)
|
70
|
-
add_ok(component,"version #{version}")
|
71
|
-
# TODO check on database if latest version
|
71
|
+
add_ok(component, "version #{version}")
|
72
|
+
# TODO: check on database if latest version
|
72
73
|
end
|
73
74
|
|
74
75
|
# translate for display
|
75
76
|
def result
|
76
77
|
raise 'missing result' if @data.empty?
|
77
|
-
{type: :object_list,data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s,'component' => i[:comp],'message' => i[:msg]}}}
|
78
|
+
{type: :object_list, data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
|
78
79
|
end
|
79
80
|
end
|
80
81
|
end
|
data/lib/aspera/node.rb
CHANGED
@@ -9,27 +9,28 @@ require 'zlib'
|
|
9
9
|
require 'base64'
|
10
10
|
|
11
11
|
module Aspera
|
12
|
-
# Provides additional functions using node API
|
13
|
-
class Node < Rest
|
12
|
+
# Provides additional functions using node API with gen4 extensions (access keys)
|
13
|
+
class Node < Aspera::Rest
|
14
14
|
# permissions
|
15
15
|
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
16
16
|
# prefix for ruby code for filter
|
17
17
|
MATCH_EXEC_PREFIX = 'exec:'
|
18
|
+
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
19
|
+
PATH_SEPARATOR = '/'
|
18
20
|
|
19
21
|
# register node special token decoder
|
20
22
|
Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
|
21
23
|
|
24
|
+
# class instance variable, access with accessors on class
|
25
|
+
@use_standard_ports = true
|
26
|
+
|
22
27
|
class << self
|
23
|
-
|
24
|
-
Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, "\
|
25
|
-
"but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
26
|
-
ts['token'] = Rest.basic_creds(ak,secret)
|
27
|
-
end
|
28
|
+
attr_accessor :use_standard_ports
|
28
29
|
|
29
30
|
# for access keys: provide expression to match entry in folder
|
30
31
|
# if no prefix: regex
|
31
32
|
# if prefix: ruby code
|
32
|
-
# if
|
33
|
+
# if expression is nil, then always match
|
33
34
|
def file_matcher(match_expression)
|
34
35
|
match_expression ||= "#{MATCH_EXEC_PREFIX}true"
|
35
36
|
if match_expression.start_with?(MATCH_EXEC_PREFIX)
|
@@ -39,49 +40,227 @@ module Aspera
|
|
39
40
|
end
|
40
41
|
end
|
41
42
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
# @param
|
49
|
-
# @param
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
43
|
+
REQUIRED_APP_INFO_FIELDS = %i[node_info app api workspace_info].freeze
|
44
|
+
REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
|
45
|
+
private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
|
46
|
+
|
47
|
+
attr_reader :app_info
|
48
|
+
|
49
|
+
# @param params [Hash] Rest parameters
|
50
|
+
# @param app_info [Hash,NilClass] special processing for AoC
|
51
|
+
def initialize(params:, app_info: nil, add_tspec: nil)
|
52
|
+
super(params)
|
53
|
+
@app_info = app_info
|
54
|
+
# this is added to transfer spec, for instance to add tags (COS)
|
55
|
+
@add_tspec = add_tspec
|
56
|
+
if !@app_info.nil?
|
57
|
+
REQUIRED_APP_INFO_FIELDS.each do |field|
|
58
|
+
raise "INTERNAL ERROR: app_info lacks field #{field}" unless @app_info.key?(field)
|
59
|
+
end
|
60
|
+
REQUIRED_APP_API_METHODS.each do |method|
|
61
|
+
raise "INTERNAL ERROR: #{@app_info[:api].class} lacks method #{method}" unless @app_info[:api].respond_to?(method)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# update transfer spec with special additional tags
|
67
|
+
def add_tspec_info(tspec)
|
68
|
+
tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
|
69
|
+
return tspec
|
70
|
+
end
|
71
|
+
|
72
|
+
# @returns [Aspera::Node] a Node or nil
|
73
|
+
def node_id_to_node(node_id)
|
74
|
+
return self if !@app_info.nil? && @app_info[:node_info]['id'].eql?(node_id)
|
75
|
+
return @app_info[:api].node_api_from(node_id: node_id, workspace_info: @app_info[workspace_info]) unless @app_info.nil?
|
76
|
+
Log.log.warn{"cannot resolve link with node id #{node_id}"}
|
77
|
+
return nil
|
78
|
+
end
|
79
|
+
|
80
|
+
# recursively browse in a folder (with non-recursive method)
|
81
|
+
# sub folders are processed if the processing method returns true
|
82
|
+
# @param state [Object] state object sent to processing method
|
83
|
+
# @param method [Symbol] processing method name
|
84
|
+
# @param top_file_id [String] file id to start at (default = access key root file id)
|
85
|
+
# @param top_file_path [String] path of top folder (default = /)
|
86
|
+
def process_folder_tree(state:, method:, top_file_id:, top_file_path: '/')
|
87
|
+
raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
|
88
|
+
raise "INTERNAL ERROR: Missing method #{method}" unless respond_to?(method)
|
89
|
+
folders_to_explore = [{id: top_file_id, path: top_file_path}]
|
90
|
+
Log.dump(:folders_to_explore, folders_to_explore)
|
91
|
+
until folders_to_explore.empty?
|
65
92
|
current_item = folders_to_explore.shift
|
66
|
-
Log.log.debug
|
93
|
+
Log.log.debug{"searching #{current_item[:path]}".bg_green}
|
67
94
|
# get folder content
|
68
95
|
folder_contents =
|
69
96
|
begin
|
70
97
|
read("files/#{current_item[:id]}/files")[:data]
|
71
98
|
rescue StandardError => e
|
72
|
-
Log.log.warn
|
99
|
+
Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
|
73
100
|
[]
|
74
101
|
end
|
75
|
-
Log.dump(:folder_contents,folder_contents)
|
102
|
+
Log.dump(:folder_contents, folder_contents)
|
76
103
|
folder_contents.each do |entry|
|
77
|
-
relative_path = File.join(current_item[:
|
78
|
-
Log.log.debug
|
104
|
+
relative_path = File.join(current_item[:path], entry['name'])
|
105
|
+
Log.log.debug{"looking #{relative_path}".bg_green}
|
106
|
+
# continue only if method returns true
|
107
|
+
next unless send(method, entry, relative_path, state)
|
79
108
|
# entry type is file, folder or link
|
80
|
-
|
81
|
-
|
109
|
+
case entry['type']
|
110
|
+
when 'folder'
|
111
|
+
folders_to_explore.push({id: entry['id'], path: relative_path})
|
112
|
+
when 'link'
|
113
|
+
node_id_to_node(entry['target_node_id'])&.process_folder_tree(
|
114
|
+
state: state,
|
115
|
+
method: method,
|
116
|
+
top_file_id: entry['target_id'],
|
117
|
+
top_file_path: relative_path)
|
82
118
|
end
|
83
119
|
end
|
84
120
|
end
|
121
|
+
end # process_folder_tree
|
122
|
+
|
123
|
+
# processing method to resolve a file path to id
|
124
|
+
# @returns true if processing need to continue
|
125
|
+
def process_resolve_node_path(entry, _path, state)
|
126
|
+
# stop digging here if not in right path
|
127
|
+
return false unless entry['name'].eql?(state[:path].first)
|
128
|
+
# ok it matches, so we remove the match
|
129
|
+
state[:path].shift
|
130
|
+
case entry['type']
|
131
|
+
when 'file'
|
132
|
+
# file must be terminal
|
133
|
+
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
134
|
+
# it's terminal, we found it
|
135
|
+
state[:result] = {api: self, file_id: entry['id']}
|
136
|
+
return false
|
137
|
+
when 'folder'
|
138
|
+
if state[:path].empty?
|
139
|
+
# we found it
|
140
|
+
state[:result] = {api: self, file_id: entry['id']}
|
141
|
+
return false
|
142
|
+
end
|
143
|
+
when 'link'
|
144
|
+
if state[:path].empty?
|
145
|
+
# we found it
|
146
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
147
|
+
raise 'cannot resolve link' if other_node.nil?
|
148
|
+
state[:result] = {api: other_node, file_id: entry['target_id']}
|
149
|
+
return false
|
150
|
+
end
|
151
|
+
else
|
152
|
+
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
153
|
+
end
|
154
|
+
# continue to dig folder
|
155
|
+
return true
|
156
|
+
end
|
157
|
+
|
158
|
+
# Navigate the path from given file id
|
159
|
+
# @param top_file_id [String] id initial file id
|
160
|
+
# @param path [String] file path
|
161
|
+
# @return [Hash] {.api,.file_id}
|
162
|
+
def resolve_api_fid(top_file_id, path)
|
163
|
+
raise 'file id shall be String' unless top_file_id.is_a?(String)
|
164
|
+
path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
|
165
|
+
return {api: self, file_id: top_file_id} if path_elements.empty?
|
166
|
+
resolve_state = {path: path_elements, result: nil}
|
167
|
+
process_folder_tree(state: resolve_state, method: :process_resolve_node_path, top_file_id: top_file_id)
|
168
|
+
raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
|
169
|
+
return resolve_state[:result]
|
170
|
+
end
|
171
|
+
|
172
|
+
# add entry to list if test block is success
|
173
|
+
# @return [TrueClass,FalseClass]
|
174
|
+
def process_find_files(entry, path, state)
|
175
|
+
begin
|
176
|
+
# add to result if match filter
|
177
|
+
state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
|
178
|
+
# process link
|
179
|
+
if entry[:type].eql?('link')
|
180
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
181
|
+
other_node.process_folder_tree(state: state, method: process_find_files, top_file_id: entry['target_id'], top_file_path: path)
|
182
|
+
end
|
183
|
+
rescue StandardError => e
|
184
|
+
Log.log.error{"#{path}: #{e.message}"}
|
185
|
+
end
|
186
|
+
# process all folders
|
187
|
+
return true
|
188
|
+
end
|
189
|
+
|
190
|
+
def find_files(top_file_id, test_block)
|
191
|
+
Log.log.debug{"find_files: file id=#{top_file_id}"}
|
192
|
+
find_state = {found: [], test_block: test_block}
|
193
|
+
process_folder_tree(state: find_state, method: :process_find_files, top_file_id: top_file_id)
|
194
|
+
return find_state[:found]
|
195
|
+
end
|
196
|
+
|
197
|
+
def refreshed_transfer_token
|
198
|
+
return oauth_token(force_refresh: true)
|
199
|
+
end
|
200
|
+
|
201
|
+
# Create transfer spec for gen4
|
202
|
+
def transfer_spec_gen4(file_id, direction, ts_merge=nil)
|
203
|
+
ak_name = nil
|
204
|
+
ak_token = nil
|
205
|
+
case params[:auth][:type]
|
206
|
+
when :basic
|
207
|
+
ak_name = params[:auth][:username]
|
208
|
+
when :oauth2
|
209
|
+
ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
|
210
|
+
# TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
|
211
|
+
# get bearer token, possibly use cache
|
212
|
+
ak_token = oauth_token(force_refresh: false)
|
213
|
+
else raise "Unsupported auth method for node gen4: #{params[:auth][:type]}"
|
214
|
+
end
|
215
|
+
transfer_spec = {
|
216
|
+
'direction' => direction,
|
217
|
+
'token' => ak_token,
|
218
|
+
'tags' => {
|
219
|
+
'aspera' => {
|
220
|
+
'node' => {
|
221
|
+
'access_key' => ak_name,
|
222
|
+
'file_id' => file_id
|
223
|
+
} # node
|
224
|
+
} # aspera
|
225
|
+
} # tags
|
226
|
+
}
|
227
|
+
# add specials tags (cos)
|
228
|
+
add_tspec_info(transfer_spec)
|
229
|
+
transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
|
230
|
+
# add application specific tags (AoC)
|
231
|
+
the_app = app_info
|
232
|
+
the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
|
233
|
+
# add basic token
|
234
|
+
if transfer_spec['token'].nil?
|
235
|
+
ts_basic_token(transfer_spec)
|
236
|
+
end
|
237
|
+
# add remote host info
|
238
|
+
if self.class.use_standard_ports
|
239
|
+
# get default TCP/UDP ports and transfer user
|
240
|
+
transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
|
241
|
+
# by default: same address as node API
|
242
|
+
transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
|
243
|
+
if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
|
244
|
+
transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
|
245
|
+
end
|
246
|
+
else
|
247
|
+
# retrieve values from API
|
248
|
+
std_t_spec = create(
|
249
|
+
'files/download_setup',
|
250
|
+
{transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
|
251
|
+
)[:data]['transfer_specs'].first['transfer_spec']
|
252
|
+
# copy some parts
|
253
|
+
%w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].each {|i| transfer_spec[i] = std_t_spec[i] if std_t_spec.key?(i)}
|
254
|
+
end
|
255
|
+
return transfer_spec
|
256
|
+
end
|
257
|
+
|
258
|
+
# set basic token in transfer spec
|
259
|
+
def ts_basic_token(ts)
|
260
|
+
Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
|
261
|
+
unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
262
|
+
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
263
|
+
ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
|
85
264
|
end
|
86
265
|
end
|
87
266
|
end
|