aspera-cli 4.10.0 → 4.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +19 -0
- data/CHANGELOG.md +528 -0
- data/CONTRIBUTING.md +143 -0
- data/README.md +977 -589
- data/bin/ascli +4 -4
- data/bin/asession +12 -12
- data/docs/test_env.conf +29 -19
- data/examples/aoc.rb +6 -6
- data/examples/dascli +18 -16
- data/examples/faspex4.rb +15 -15
- data/examples/node.rb +12 -12
- data/examples/proxy.pac +2 -2
- data/examples/server.rb +12 -12
- data/lib/aspera/aoc.rb +344 -272
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +9 -9
- data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +16 -21
- data/lib/aspera/cli/main.rb +72 -73
- data/lib/aspera/cli/manager.rb +112 -112
- data/lib/aspera/cli/plugin.rb +68 -48
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +322 -720
- data/lib/aspera/cli/plugins/ats.rb +50 -52
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +514 -410
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +134 -136
- data/lib/aspera/cli/plugins/faspex5.rb +235 -70
- data/lib/aspera/cli/plugins/node.rb +378 -309
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
- data/lib/aspera/cli/plugins/preview.rb +129 -120
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +77 -52
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +61 -61
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +78 -74
- data/lib/aspera/cos_node.rb +31 -29
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +17 -15
- data/lib/aspera/fasp/agent_connect.rb +34 -32
- data/lib/aspera/fasp/agent_direct.rb +70 -73
- data/lib/aspera/fasp/agent_httpgw.rb +79 -74
- data/lib/aspera/fasp/agent_node.rb +26 -26
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +80 -80
- data/lib/aspera/fasp/listener.rb +2 -2
- data/lib/aspera/fasp/parameters.rb +103 -92
- data/lib/aspera/fasp/parameters.yaml +313 -214
- data/lib/aspera/fasp/resume_policy.rb +10 -10
- data/lib/aspera/fasp/transfer_spec.rb +22 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +80 -159
- data/lib/aspera/faspex_postproc.rb +77 -0
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/nagios.rb +24 -23
- data/lib/aspera/node.rb +217 -38
- data/lib/aspera/oauth.rb +78 -74
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +63 -63
- data/lib/aspera/proxy_auto_config.rb +19 -19
- data/lib/aspera/rest.rb +65 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +22 -21
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +17 -14
- data/lib/aspera/ssh.rb +15 -14
- data/lib/aspera/sync.rb +177 -62
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +13 -64
- data/lib/aspera/web_server_simple.rb +76 -0
- data.tar.gz.sig +0 -0
- metadata +11 -6
- metadata.gz.sig +0 -0
@@ -10,10 +10,10 @@ module Aspera
|
|
10
10
|
# keychain based on macOS keychain, using `security` cmmand line
|
11
11
|
class Keychain
|
12
12
|
DOMAINS = %i[user system common dynamic].freeze
|
13
|
-
LIST_OPTIONS={
|
13
|
+
LIST_OPTIONS = {
|
14
14
|
domain: :c
|
15
15
|
}
|
16
|
-
ADD_PASS_OPTIONS={
|
16
|
+
ADD_PASS_OPTIONS = {
|
17
17
|
account: :a,
|
18
18
|
creator: :c,
|
19
19
|
type: :C,
|
@@ -32,7 +32,7 @@ module Aspera
|
|
32
32
|
getpass: :g
|
33
33
|
}.freeze
|
34
34
|
class << self
|
35
|
-
def execute(command,options=nil,supported=nil,lastopt=nil)
|
35
|
+
def execute(command, options=nil, supported=nil, lastopt=nil)
|
36
36
|
url = options&.delete(:url)
|
37
37
|
if !url.nil?
|
38
38
|
uri = URI.parse(url)
|
@@ -40,20 +40,20 @@ module Aspera
|
|
40
40
|
options[:protocol] = 'htps'
|
41
41
|
raise 'host required in URL' if uri.host.nil?
|
42
42
|
options[:server] = uri.host
|
43
|
-
options[:path] = uri.path unless ['','/'].include?(uri.path)
|
43
|
+
options[:path] = uri.path unless ['', '/'].include?(uri.path)
|
44
44
|
options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
|
45
45
|
end
|
46
|
-
cmd=['security',command]
|
47
|
-
options&.each do |k,v|
|
48
|
-
raise "unknown option: #{k}" unless supported.
|
46
|
+
cmd = ['security', command]
|
47
|
+
options&.each do |k, v|
|
48
|
+
raise "unknown option: #{k}" unless supported.key?(k)
|
49
49
|
next if v.nil?
|
50
50
|
cmd.push("-#{supported[k]}")
|
51
51
|
cmd.push(v.shellescape) unless v.empty?
|
52
52
|
end
|
53
53
|
cmd.push(lastopt) unless lastopt.nil?
|
54
|
-
Log.log.debug
|
55
|
-
result
|
56
|
-
Log.log.debug
|
54
|
+
Log.log.debug{"executing>>#{cmd.join(' ')}"}
|
55
|
+
result = %x(#{cmd.join(' ')} 2>&1)
|
56
|
+
Log.log.debug{"result>>[#{result}]"}
|
57
57
|
return result
|
58
58
|
end
|
59
59
|
|
@@ -70,8 +70,8 @@ module Aspera
|
|
70
70
|
end
|
71
71
|
|
72
72
|
def list(options={})
|
73
|
-
raise ArgumentError,"Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
|
74
|
-
keychains(execute('list-keychains',options,LIST_OPTIONS))
|
73
|
+
raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
|
74
|
+
keychains(execute('list-keychains', options, LIST_OPTIONS))
|
75
75
|
end
|
76
76
|
|
77
77
|
def by_name(name)
|
@@ -88,15 +88,15 @@ module Aspera
|
|
88
88
|
[string].pack('H*').force_encoding('UTF-8')
|
89
89
|
end
|
90
90
|
|
91
|
-
def password(operation,passtype,options)
|
91
|
+
def password(operation, passtype, options)
|
92
92
|
raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
|
93
93
|
raise "wrong passtype: #{passtype}" unless %i[generic internet].include?(passtype)
|
94
94
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
95
|
-
missing=(operation.eql?(:add) ? %i[account service password] : %i[label])-options.keys
|
95
|
+
missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
|
96
96
|
raise "missing options: #{missing}" unless missing.empty?
|
97
|
-
options[:getpass]='' if operation.eql?(:find)
|
98
|
-
output=self.class.execute("#{operation}-#{passtype}-password",options,ADD_PASS_OPTIONS
|
99
|
-
raise output.gsub(/^.*: /,'') if output.start_with?('security: ')
|
97
|
+
options[:getpass] = '' if operation.eql?(:find)
|
98
|
+
output = self.class.execute("#{operation}-#{passtype}-password", options, ADD_PASS_OPTIONS, @path)
|
99
|
+
raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
|
100
100
|
return nil unless operation.eql?(:find)
|
101
101
|
attributes = {}
|
102
102
|
output.split("\n").each do |line|
|
@@ -121,7 +121,7 @@ module Aspera
|
|
121
121
|
end
|
122
122
|
|
123
123
|
class MacosSystem
|
124
|
-
def initialize(name=nil,
|
124
|
+
def initialize(name=nil, _password=nil)
|
125
125
|
@keychain = name.nil? ? MacosSecurity::Keychain.default_keychain : MacosSecurity::Keychain.by_name(name)
|
126
126
|
raise "no such keychain #{name}" if @keychain.nil?
|
127
127
|
end
|
@@ -130,7 +130,8 @@ module Aspera
|
|
130
130
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
131
131
|
unsupported = options.keys - %i[label username password url description]
|
132
132
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
133
|
-
@keychain.password(
|
133
|
+
@keychain.password(
|
134
|
+
:add, :generic, service: options[:label],
|
134
135
|
account: options[:username] || 'none', password: options[:password], comment: options[:description])
|
135
136
|
end
|
136
137
|
|
@@ -138,7 +139,7 @@ module Aspera
|
|
138
139
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
139
140
|
unsupported = options.keys - %i[label]
|
140
141
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
141
|
-
info = @keychain.password(:find
|
142
|
+
info = @keychain.password(:find, :generic, label: options[:label])
|
142
143
|
raise 'not found' if info.nil?
|
143
144
|
result = options.clone
|
144
145
|
result[:secret] = info['password']
|
data/lib/aspera/log.rb
CHANGED
@@ -11,28 +11,27 @@ module Aspera
|
|
11
11
|
# Singleton object for logging
|
12
12
|
class Log
|
13
13
|
include Singleton
|
14
|
+
# where logs are sent to
|
15
|
+
LOG_TYPES = %i[stderr stdout syslog].freeze
|
14
16
|
# class methods
|
15
17
|
class << self
|
16
18
|
# levels are :debug,:info,:warn,:error,fatal,:unknown
|
17
|
-
def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
|
18
|
-
|
19
|
-
# where logs are sent to
|
20
|
-
def logtypes; %i[stderr stdout syslog];end
|
19
|
+
def levels; Logger::Severity.constants.sort{|a, b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym}; end
|
21
20
|
|
22
21
|
# get the logger object of singleton
|
23
|
-
def log; instance.logger;end
|
22
|
+
def log; instance.logger; end
|
24
23
|
|
25
24
|
# dump object in debug mode
|
26
25
|
# @param name string or symbol
|
27
26
|
# @param format either pp or json format
|
28
|
-
def dump(name,object,format=:json)
|
27
|
+
def dump(name, object, format=:json)
|
29
28
|
log.debug do
|
30
29
|
result =
|
31
30
|
case format
|
32
31
|
when :json
|
33
|
-
JSON.pretty_generate(object) rescue PP.pp(object
|
32
|
+
JSON.pretty_generate(object) rescue PP.pp(object, +'')
|
34
33
|
when :ruby
|
35
|
-
PP.pp(object
|
34
|
+
PP.pp(object, +'')
|
36
35
|
else
|
37
36
|
raise 'wrong parameter, expect pp or json'
|
38
37
|
end
|
@@ -68,12 +67,13 @@ module Aspera
|
|
68
67
|
end
|
69
68
|
|
70
69
|
# change underlying logger, but keep log level
|
71
|
-
def logger_type=(
|
70
|
+
def logger_type=(new_log_type)
|
72
71
|
current_severity_integer = @logger.level unless @logger.nil?
|
73
|
-
current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.
|
72
|
+
current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
|
74
73
|
current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
|
75
|
-
case
|
74
|
+
case new_log_type
|
76
75
|
when :stderr
|
76
|
+
# typed: Logger
|
77
77
|
@logger = Logger.new($stderr)
|
78
78
|
when :stdout
|
79
79
|
@logger = Logger.new($stdout)
|
@@ -81,10 +81,10 @@ module Aspera
|
|
81
81
|
require 'syslog/logger'
|
82
82
|
@logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
|
83
83
|
else
|
84
|
-
raise "unknown log type: #{
|
84
|
+
raise "unknown log type: #{new_log_type.class} #{new_log_type}"
|
85
85
|
end
|
86
86
|
@logger.level = current_severity_integer
|
87
|
-
@logger_type =
|
87
|
+
@logger_type = new_log_type
|
88
88
|
# update formatter with password hiding
|
89
89
|
@logger.formatter = SecretHider.log_formatter(@logger.formatter)
|
90
90
|
end
|
data/lib/aspera/nagios.rb
CHANGED
@@ -10,32 +10,32 @@ module Aspera
|
|
10
10
|
# date offset levels
|
11
11
|
DATE_WARN_OFFSET = 2
|
12
12
|
DATE_CRIT_OFFSET = 5
|
13
|
-
private_constant :LEVELS
|
13
|
+
private_constant :LEVELS, :ADD_PREFIX, :DATE_WARN_OFFSET, :DATE_CRIT_OFFSET
|
14
14
|
|
15
15
|
# add methods to add nagios error levels, each take component name and message
|
16
16
|
LEVELS.each_index do |code|
|
17
17
|
name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
|
18
|
-
define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
|
18
|
+
define_method(name){|comp, msg|@data.push({code: code, comp: comp, msg: msg})}
|
19
19
|
end
|
20
20
|
|
21
21
|
class << self
|
22
22
|
# process results of a analysis and display status and exit with code
|
23
23
|
def process(data)
|
24
24
|
raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
|
25
|
-
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.
|
25
|
+
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.key?(c)}
|
26
26
|
res_errors = data.reject{|s|s['status'].eql?('ok')}
|
27
27
|
# keep only errors in case of problem, other ok are assumed so
|
28
28
|
data = res_errors unless res_errors.empty?
|
29
29
|
# first is most critical
|
30
|
-
data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
30
|
+
data.sort!{|a, b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
31
31
|
# build message: if multiple components: concatenate
|
32
|
-
#message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
33
|
-
message = data
|
34
|
-
map{|i|i['component']}
|
35
|
-
uniq
|
36
|
-
map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
|
37
|
-
join(', ')
|
38
|
-
tr("\n",' ')
|
32
|
+
# message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
33
|
+
message = data
|
34
|
+
.map{|i|i['component']}
|
35
|
+
.uniq
|
36
|
+
.map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
|
37
|
+
.join(', ')
|
38
|
+
.tr("\n", ' ')
|
39
39
|
status = data.first['status'].upcase
|
40
40
|
# display status for nagios
|
41
41
|
puts("#{status} - [#{message}]\n")
|
@@ -45,36 +45,37 @@ module Aspera
|
|
45
45
|
end
|
46
46
|
|
47
47
|
attr_reader :data
|
48
|
+
|
48
49
|
def initialize
|
49
50
|
@data = []
|
50
51
|
end
|
51
52
|
|
52
|
-
#
|
53
|
+
# compare remote time with local time
|
53
54
|
def check_time_offset(remote_date, component)
|
54
55
|
# check date if specified : 2015-10-13T07:32:01Z
|
55
|
-
|
56
|
-
diff_time = (
|
57
|
-
|
58
|
-
Log.log.debug
|
59
|
-
msg = "offset #{
|
56
|
+
remote_time = DateTime.strptime(remote_date)
|
57
|
+
diff_time = (remote_time - DateTime.now).abs
|
58
|
+
diff_rounded = diff_time.round(-2)
|
59
|
+
Log.log.debug{"DATE: #{remote_date} #{remote_time} diff=#{diff_rounded}"}
|
60
|
+
msg = "offset #{diff_rounded} sec"
|
60
61
|
if diff_time >= DATE_CRIT_OFFSET
|
61
|
-
add_critical(component,msg)
|
62
|
+
add_critical(component, msg)
|
62
63
|
elsif diff_time >= DATE_WARN_OFFSET
|
63
|
-
add_warning(component,msg)
|
64
|
+
add_warning(component, msg)
|
64
65
|
else
|
65
|
-
add_ok(component,msg)
|
66
|
+
add_ok(component, msg)
|
66
67
|
end
|
67
68
|
end
|
68
69
|
|
69
70
|
def check_product_version(component, _product, version)
|
70
|
-
add_ok(component,"version #{version}")
|
71
|
-
# TODO check on database if latest version
|
71
|
+
add_ok(component, "version #{version}")
|
72
|
+
# TODO: check on database if latest version
|
72
73
|
end
|
73
74
|
|
74
75
|
# translate for display
|
75
76
|
def result
|
76
77
|
raise 'missing result' if @data.empty?
|
77
|
-
{type: :object_list,data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s,'component' => i[:comp],'message' => i[:msg]}}}
|
78
|
+
{type: :object_list, data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
|
78
79
|
end
|
79
80
|
end
|
80
81
|
end
|
data/lib/aspera/node.rb
CHANGED
@@ -9,27 +9,28 @@ require 'zlib'
|
|
9
9
|
require 'base64'
|
10
10
|
|
11
11
|
module Aspera
|
12
|
-
# Provides additional functions using node API
|
13
|
-
class Node < Rest
|
12
|
+
# Provides additional functions using node API with gen4 extensions (access keys)
|
13
|
+
class Node < Aspera::Rest
|
14
14
|
# permissions
|
15
15
|
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
16
16
|
# prefix for ruby code for filter
|
17
17
|
MATCH_EXEC_PREFIX = 'exec:'
|
18
|
+
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
19
|
+
PATH_SEPARATOR = '/'
|
18
20
|
|
19
21
|
# register node special token decoder
|
20
22
|
Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
|
21
23
|
|
24
|
+
# class instance variable, access with accessors on class
|
25
|
+
@use_standard_ports = true
|
26
|
+
|
22
27
|
class << self
|
23
|
-
|
24
|
-
Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, "\
|
25
|
-
"but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
26
|
-
ts['token'] = Rest.basic_creds(ak,secret)
|
27
|
-
end
|
28
|
+
attr_accessor :use_standard_ports
|
28
29
|
|
29
30
|
# for access keys: provide expression to match entry in folder
|
30
31
|
# if no prefix: regex
|
31
32
|
# if prefix: ruby code
|
32
|
-
# if
|
33
|
+
# if expression is nil, then always match
|
33
34
|
def file_matcher(match_expression)
|
34
35
|
match_expression ||= "#{MATCH_EXEC_PREFIX}true"
|
35
36
|
if match_expression.start_with?(MATCH_EXEC_PREFIX)
|
@@ -39,49 +40,227 @@ module Aspera
|
|
39
40
|
end
|
40
41
|
end
|
41
42
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
# @param
|
49
|
-
# @param
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
43
|
+
REQUIRED_APP_INFO_FIELDS = %i[node_info app api workspace_info].freeze
|
44
|
+
REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
|
45
|
+
private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
|
46
|
+
|
47
|
+
attr_reader :app_info
|
48
|
+
|
49
|
+
# @param params [Hash] Rest parameters
|
50
|
+
# @param app_info [Hash,NilClass] special processing for AoC
|
51
|
+
def initialize(params:, app_info: nil, add_tspec: nil)
|
52
|
+
super(params)
|
53
|
+
@app_info = app_info
|
54
|
+
# this is added to transfer spec, for instance to add tags (COS)
|
55
|
+
@add_tspec = add_tspec
|
56
|
+
if !@app_info.nil?
|
57
|
+
REQUIRED_APP_INFO_FIELDS.each do |field|
|
58
|
+
raise "INTERNAL ERROR: app_info lacks field #{field}" unless @app_info.key?(field)
|
59
|
+
end
|
60
|
+
REQUIRED_APP_API_METHODS.each do |method|
|
61
|
+
raise "INTERNAL ERROR: #{@app_info[:api].class} lacks method #{method}" unless @app_info[:api].respond_to?(method)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# update transfer spec with special additional tags
|
67
|
+
def add_tspec_info(tspec)
|
68
|
+
tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
|
69
|
+
return tspec
|
70
|
+
end
|
71
|
+
|
72
|
+
# @returns [Aspera::Node] a Node or nil
|
73
|
+
def node_id_to_node(node_id)
|
74
|
+
return self if !@app_info.nil? && @app_info[:node_info]['id'].eql?(node_id)
|
75
|
+
return @app_info[:api].node_api_from(node_id: node_id, workspace_info: @app_info[workspace_info]) unless @app_info.nil?
|
76
|
+
Log.log.warn{"cannot resolve link with node id #{node_id}"}
|
77
|
+
return nil
|
78
|
+
end
|
79
|
+
|
80
|
+
# recursively browse in a folder (with non-recursive method)
|
81
|
+
# sub folders are processed if the processing method returns true
|
82
|
+
# @param state [Object] state object sent to processing method
|
83
|
+
# @param method [Symbol] processing method name
|
84
|
+
# @param top_file_id [String] file id to start at (default = access key root file id)
|
85
|
+
# @param top_file_path [String] path of top folder (default = /)
|
86
|
+
def process_folder_tree(state:, method:, top_file_id:, top_file_path: '/')
|
87
|
+
raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
|
88
|
+
raise "INTERNAL ERROR: Missing method #{method}" unless respond_to?(method)
|
89
|
+
folders_to_explore = [{id: top_file_id, path: top_file_path}]
|
90
|
+
Log.dump(:folders_to_explore, folders_to_explore)
|
91
|
+
until folders_to_explore.empty?
|
65
92
|
current_item = folders_to_explore.shift
|
66
|
-
Log.log.debug
|
93
|
+
Log.log.debug{"searching #{current_item[:path]}".bg_green}
|
67
94
|
# get folder content
|
68
95
|
folder_contents =
|
69
96
|
begin
|
70
97
|
read("files/#{current_item[:id]}/files")[:data]
|
71
98
|
rescue StandardError => e
|
72
|
-
Log.log.warn
|
99
|
+
Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
|
73
100
|
[]
|
74
101
|
end
|
75
|
-
Log.dump(:folder_contents,folder_contents)
|
102
|
+
Log.dump(:folder_contents, folder_contents)
|
76
103
|
folder_contents.each do |entry|
|
77
|
-
relative_path = File.join(current_item[:
|
78
|
-
Log.log.debug
|
104
|
+
relative_path = File.join(current_item[:path], entry['name'])
|
105
|
+
Log.log.debug{"looking #{relative_path}".bg_green}
|
106
|
+
# continue only if method returns true
|
107
|
+
next unless send(method, entry, relative_path, state)
|
79
108
|
# entry type is file, folder or link
|
80
|
-
|
81
|
-
|
109
|
+
case entry['type']
|
110
|
+
when 'folder'
|
111
|
+
folders_to_explore.push({id: entry['id'], path: relative_path})
|
112
|
+
when 'link'
|
113
|
+
node_id_to_node(entry['target_node_id'])&.process_folder_tree(
|
114
|
+
state: state,
|
115
|
+
method: method,
|
116
|
+
top_file_id: entry['target_id'],
|
117
|
+
top_file_path: relative_path)
|
82
118
|
end
|
83
119
|
end
|
84
120
|
end
|
121
|
+
end # process_folder_tree
|
122
|
+
|
123
|
+
# processing method to resolve a file path to id
|
124
|
+
# @returns true if processing need to continue
|
125
|
+
def process_resolve_node_path(entry, _path, state)
|
126
|
+
# stop digging here if not in right path
|
127
|
+
return false unless entry['name'].eql?(state[:path].first)
|
128
|
+
# ok it matches, so we remove the match
|
129
|
+
state[:path].shift
|
130
|
+
case entry['type']
|
131
|
+
when 'file'
|
132
|
+
# file must be terminal
|
133
|
+
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
134
|
+
# it's terminal, we found it
|
135
|
+
state[:result] = {api: self, file_id: entry['id']}
|
136
|
+
return false
|
137
|
+
when 'folder'
|
138
|
+
if state[:path].empty?
|
139
|
+
# we found it
|
140
|
+
state[:result] = {api: self, file_id: entry['id']}
|
141
|
+
return false
|
142
|
+
end
|
143
|
+
when 'link'
|
144
|
+
if state[:path].empty?
|
145
|
+
# we found it
|
146
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
147
|
+
raise 'cannot resolve link' if other_node.nil?
|
148
|
+
state[:result] = {api: other_node, file_id: entry['target_id']}
|
149
|
+
return false
|
150
|
+
end
|
151
|
+
else
|
152
|
+
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
153
|
+
end
|
154
|
+
# continue to dig folder
|
155
|
+
return true
|
156
|
+
end
|
157
|
+
|
158
|
+
# Navigate the path from given file id
|
159
|
+
# @param top_file_id [String] id initial file id
|
160
|
+
# @param path [String] file path
|
161
|
+
# @return [Hash] {.api,.file_id}
|
162
|
+
def resolve_api_fid(top_file_id, path)
|
163
|
+
raise 'file id shall be String' unless top_file_id.is_a?(String)
|
164
|
+
path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
|
165
|
+
return {api: self, file_id: top_file_id} if path_elements.empty?
|
166
|
+
resolve_state = {path: path_elements, result: nil}
|
167
|
+
process_folder_tree(state: resolve_state, method: :process_resolve_node_path, top_file_id: top_file_id)
|
168
|
+
raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
|
169
|
+
return resolve_state[:result]
|
170
|
+
end
|
171
|
+
|
172
|
+
# add entry to list if test block is success
|
173
|
+
# @return [TrueClass,FalseClass]
|
174
|
+
def process_find_files(entry, path, state)
|
175
|
+
begin
|
176
|
+
# add to result if match filter
|
177
|
+
state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
|
178
|
+
# process link
|
179
|
+
if entry[:type].eql?('link')
|
180
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
181
|
+
other_node.process_folder_tree(state: state, method: process_find_files, top_file_id: entry['target_id'], top_file_path: path)
|
182
|
+
end
|
183
|
+
rescue StandardError => e
|
184
|
+
Log.log.error{"#{path}: #{e.message}"}
|
185
|
+
end
|
186
|
+
# process all folders
|
187
|
+
return true
|
188
|
+
end
|
189
|
+
|
190
|
+
def find_files(top_file_id, test_block)
|
191
|
+
Log.log.debug{"find_files: file id=#{top_file_id}"}
|
192
|
+
find_state = {found: [], test_block: test_block}
|
193
|
+
process_folder_tree(state: find_state, method: :process_find_files, top_file_id: top_file_id)
|
194
|
+
return find_state[:found]
|
195
|
+
end
|
196
|
+
|
197
|
+
def refreshed_transfer_token
|
198
|
+
return oauth_token(force_refresh: true)
|
199
|
+
end
|
200
|
+
|
201
|
+
# Create transfer spec for gen4
|
202
|
+
def transfer_spec_gen4(file_id, direction, ts_merge=nil)
|
203
|
+
ak_name = nil
|
204
|
+
ak_token = nil
|
205
|
+
case params[:auth][:type]
|
206
|
+
when :basic
|
207
|
+
ak_name = params[:auth][:username]
|
208
|
+
when :oauth2
|
209
|
+
ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
|
210
|
+
# TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
|
211
|
+
# get bearer token, possibly use cache
|
212
|
+
ak_token = oauth_token(force_refresh: false)
|
213
|
+
else raise "Unsupported auth method for node gen4: #{params[:auth][:type]}"
|
214
|
+
end
|
215
|
+
transfer_spec = {
|
216
|
+
'direction' => direction,
|
217
|
+
'token' => ak_token,
|
218
|
+
'tags' => {
|
219
|
+
'aspera' => {
|
220
|
+
'node' => {
|
221
|
+
'access_key' => ak_name,
|
222
|
+
'file_id' => file_id
|
223
|
+
} # node
|
224
|
+
} # aspera
|
225
|
+
} # tags
|
226
|
+
}
|
227
|
+
# add specials tags (cos)
|
228
|
+
add_tspec_info(transfer_spec)
|
229
|
+
transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
|
230
|
+
# add application specific tags (AoC)
|
231
|
+
the_app = app_info
|
232
|
+
the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
|
233
|
+
# add basic token
|
234
|
+
if transfer_spec['token'].nil?
|
235
|
+
ts_basic_token(transfer_spec)
|
236
|
+
end
|
237
|
+
# add remote host info
|
238
|
+
if self.class.use_standard_ports
|
239
|
+
# get default TCP/UDP ports and transfer user
|
240
|
+
transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
|
241
|
+
# by default: same address as node API
|
242
|
+
transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
|
243
|
+
if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
|
244
|
+
transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
|
245
|
+
end
|
246
|
+
else
|
247
|
+
# retrieve values from API
|
248
|
+
std_t_spec = create(
|
249
|
+
'files/download_setup',
|
250
|
+
{transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
|
251
|
+
)[:data]['transfer_specs'].first['transfer_spec']
|
252
|
+
# copy some parts
|
253
|
+
%w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].each {|i| transfer_spec[i] = std_t_spec[i] if std_t_spec.key?(i)}
|
254
|
+
end
|
255
|
+
return transfer_spec
|
256
|
+
end
|
257
|
+
|
258
|
+
# set basic token in transfer spec
|
259
|
+
def ts_basic_token(ts)
|
260
|
+
Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
|
261
|
+
unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
262
|
+
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
263
|
+
ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
|
85
264
|
end
|
86
265
|
end
|
87
266
|
end
|