aspera-cli 4.13.0 → 4.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +81 -7
- data/CONTRIBUTING.md +22 -6
- data/README.md +2038 -1080
- data/bin/ascli +18 -9
- data/bin/asession +12 -14
- data/examples/dascli +1 -1
- data/examples/proxy.pac +1 -1
- data/examples/rubyc +24 -0
- data/lib/aspera/aoc.rb +219 -159
- data/lib/aspera/ascmd.rb +25 -14
- data/lib/aspera/cli/basic_auth_plugin.rb +12 -9
- data/lib/aspera/cli/error.rb +17 -0
- data/lib/aspera/cli/extended_value.rb +47 -12
- data/lib/aspera/cli/formatter.rb +260 -179
- data/lib/aspera/cli/hints.rb +80 -0
- data/lib/aspera/cli/main.rb +104 -156
- data/lib/aspera/cli/manager.rb +259 -209
- data/lib/aspera/cli/plugin.rb +123 -63
- data/lib/aspera/cli/plugins/alee.rb +2 -3
- data/lib/aspera/cli/plugins/aoc.rb +341 -261
- data/lib/aspera/cli/plugins/ats.rb +22 -21
- data/lib/aspera/cli/plugins/bss.rb +5 -5
- data/lib/aspera/cli/plugins/config.rb +578 -627
- data/lib/aspera/cli/plugins/console.rb +44 -6
- data/lib/aspera/cli/plugins/cos.rb +15 -17
- data/lib/aspera/cli/plugins/faspex.rb +114 -100
- data/lib/aspera/cli/plugins/faspex5.rb +411 -264
- data/lib/aspera/cli/plugins/node.rb +354 -259
- data/lib/aspera/cli/plugins/orchestrator.rb +61 -29
- data/lib/aspera/cli/plugins/preview.rb +82 -90
- data/lib/aspera/cli/plugins/server.rb +79 -32
- data/lib/aspera/cli/plugins/shares.rb +55 -42
- data/lib/aspera/cli/sync_actions.rb +68 -0
- data/lib/aspera/cli/transfer_agent.rb +66 -73
- data/lib/aspera/cli/transfer_progress.rb +74 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +12 -8
- data/lib/aspera/command_line_builder.rb +14 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/data/6 +0 -0
- data/lib/aspera/environment.rb +24 -9
- data/lib/aspera/fasp/agent_aspera.rb +126 -0
- data/lib/aspera/fasp/agent_base.rb +31 -77
- data/lib/aspera/fasp/agent_connect.rb +25 -21
- data/lib/aspera/fasp/agent_direct.rb +89 -103
- data/lib/aspera/fasp/agent_httpgw.rb +231 -149
- data/lib/aspera/fasp/agent_node.rb +41 -34
- data/lib/aspera/fasp/agent_trsdk.rb +75 -32
- data/lib/aspera/fasp/error_info.rb +4 -2
- data/lib/aspera/fasp/faux_file.rb +52 -0
- data/lib/aspera/fasp/installation.rb +53 -195
- data/lib/aspera/fasp/management.rb +244 -0
- data/lib/aspera/fasp/parameters.rb +71 -37
- data/lib/aspera/fasp/parameters.yaml +76 -8
- data/lib/aspera/fasp/products.rb +162 -0
- data/lib/aspera/fasp/resume_policy.rb +3 -3
- data/lib/aspera/fasp/transfer_spec.rb +7 -6
- data/lib/aspera/fasp/uri.rb +26 -24
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/hash_ext.rb +14 -4
- data/lib/aspera/json_rpc.rb +49 -0
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/line_logger.rb +23 -0
- data/lib/aspera/log.rb +58 -16
- data/lib/aspera/node.rb +157 -92
- data/lib/aspera/oauth.rb +37 -19
- data/lib/aspera/open_application.rb +4 -4
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +4 -2
- data/lib/aspera/preview/generator.rb +22 -35
- data/lib/aspera/preview/options.rb +2 -0
- data/lib/aspera/preview/terminal.rb +73 -16
- data/lib/aspera/preview/utils.rb +21 -28
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/rest.rb +136 -68
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +15 -14
- data/lib/aspera/rest_errors_aspera.rb +37 -34
- data/lib/aspera/secret_hider.rb +18 -15
- data/lib/aspera/ssh.rb +5 -2
- data/lib/aspera/sync.rb +127 -119
- data/lib/aspera/temp_file_manager.rb +10 -3
- data/lib/aspera/web_auth.rb +10 -7
- data/lib/aspera/web_server_simple.rb +9 -4
- data.tar.gz.sig +0 -0
- metadata +34 -17
- metadata.gz.sig +0 -0
- data/docs/test_env.conf +0 -186
- data/lib/aspera/cli/listener/line_dump.rb +0 -19
- data/lib/aspera/cli/listener/logger.rb +0 -22
- data/lib/aspera/cli/listener/progress.rb +0 -50
- data/lib/aspera/cli/listener/progress_multi.rb +0 -84
- data/lib/aspera/cli/plugins/sync.rb +0 -44
- data/lib/aspera/data/7 +0 -0
- data/lib/aspera/fasp/listener.rb +0 -13
@@ -3,11 +3,11 @@
|
|
3
3
|
# https://github.com/fastlane-community/security
|
4
4
|
require 'aspera/cli/info'
|
5
5
|
|
6
|
-
# enhance the gem to support other
|
6
|
+
# enhance the gem to support other key chains
|
7
7
|
module Aspera
|
8
8
|
module Keychain
|
9
9
|
module MacosSecurity
|
10
|
-
# keychain based on macOS keychain, using `security`
|
10
|
+
# keychain based on macOS keychain, using `security` command line
|
11
11
|
class Keychain
|
12
12
|
DOMAINS = %i[user system common dynamic].freeze
|
13
13
|
LIST_OPTIONS = {
|
@@ -32,12 +32,12 @@ module Aspera
|
|
32
32
|
getpass: :g
|
33
33
|
}.freeze
|
34
34
|
class << self
|
35
|
-
def execute(command, options=nil, supported=nil,
|
35
|
+
def execute(command, options=nil, supported=nil, last_opt=nil)
|
36
36
|
url = options&.delete(:url)
|
37
37
|
if !url.nil?
|
38
38
|
uri = URI.parse(url)
|
39
39
|
raise 'only https' unless uri.scheme.eql?('https')
|
40
|
-
options[:protocol] = 'htps'
|
40
|
+
options[:protocol] = 'htps' # cspell: disable-line
|
41
41
|
raise 'host required in URL' if uri.host.nil?
|
42
42
|
options[:server] = uri.host
|
43
43
|
options[:path] = uri.path unless ['', '/'].include?(uri.path)
|
@@ -50,28 +50,28 @@ module Aspera
|
|
50
50
|
cmd.push("-#{supported[k]}")
|
51
51
|
cmd.push(v.shellescape) unless v.empty?
|
52
52
|
end
|
53
|
-
cmd.push(
|
53
|
+
cmd.push(last_opt) unless last_opt.nil?
|
54
54
|
Log.log.debug{"executing>>#{cmd.join(' ')}"}
|
55
55
|
result = %x(#{cmd.join(' ')} 2>&1)
|
56
56
|
Log.log.debug{"result>>[#{result}]"}
|
57
57
|
return result
|
58
58
|
end
|
59
59
|
|
60
|
-
def
|
60
|
+
def key_chains(output)
|
61
61
|
output.split("\n").collect { |line| new(line.strip.gsub(/^"|"$/, '')) }
|
62
62
|
end
|
63
63
|
|
64
64
|
def default
|
65
|
-
|
65
|
+
key_chains(execute('default-keychain')).first
|
66
66
|
end
|
67
67
|
|
68
68
|
def login
|
69
|
-
|
69
|
+
key_chains(execute('login-keychain')).first
|
70
70
|
end
|
71
71
|
|
72
72
|
def list(options={})
|
73
73
|
raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
|
74
|
-
|
74
|
+
key_chains(execute('list-key_chains', options, LIST_OPTIONS))
|
75
75
|
end
|
76
76
|
|
77
77
|
def by_name(name)
|
@@ -88,14 +88,14 @@ module Aspera
|
|
88
88
|
[string].pack('H*').force_encoding('UTF-8')
|
89
89
|
end
|
90
90
|
|
91
|
-
def password(operation,
|
91
|
+
def password(operation, pass_type, options)
|
92
92
|
raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
|
93
|
-
raise "wrong
|
93
|
+
raise "wrong pass_type: #{pass_type}" unless %i[generic internet].include?(pass_type)
|
94
94
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
95
95
|
missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
|
96
96
|
raise "missing options: #{missing}" unless missing.empty?
|
97
97
|
options[:getpass] = '' if operation.eql?(:find)
|
98
|
-
output = self.class.execute("#{operation}-#{
|
98
|
+
output = self.class.execute("#{operation}-#{pass_type}-password", options, ADD_PASS_OPTIONS, @path)
|
99
99
|
raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
|
100
100
|
return nil unless operation.eql?(:find)
|
101
101
|
attributes = {}
|
@@ -143,7 +143,7 @@ module Aspera
|
|
143
143
|
raise 'not found' if info.nil?
|
144
144
|
result = options.clone
|
145
145
|
result[:secret] = info['password']
|
146
|
-
result[:description] = info['icmt']
|
146
|
+
result[:description] = info['icmt'] # cspell: disable-line
|
147
147
|
return result
|
148
148
|
end
|
149
149
|
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/log'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
# used for logging http
|
7
|
+
class LineLogger
|
8
|
+
def initialize(level)
|
9
|
+
@level = level
|
10
|
+
@buffer = []
|
11
|
+
end
|
12
|
+
|
13
|
+
def <<(string)
|
14
|
+
return if string.nil? || string.empty?
|
15
|
+
if !string.end_with?("\n")
|
16
|
+
@buffer.push(string)
|
17
|
+
return
|
18
|
+
end
|
19
|
+
Log.log.send(@level, @buffer.join('') + string.chomp)
|
20
|
+
@buffer.clear
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/aspera/log.rb
CHANGED
@@ -2,17 +2,53 @@
|
|
2
2
|
|
3
3
|
require 'aspera/colors'
|
4
4
|
require 'aspera/secret_hider'
|
5
|
+
require 'aspera/environment'
|
5
6
|
require 'logger'
|
6
7
|
require 'pp'
|
7
8
|
require 'json'
|
8
9
|
require 'singleton'
|
9
10
|
|
11
|
+
# extend Ruby logger with trace levels
|
12
|
+
class Logger
|
13
|
+
TRACE_MAX = 2
|
14
|
+
# add custom level to logger severity
|
15
|
+
module Severity
|
16
|
+
1.upto(TRACE_MAX).each { |level| const_set("TRACE#{level}", - level)}
|
17
|
+
end
|
18
|
+
# quick access to label
|
19
|
+
SEVERITY_LABEL = Severity.constants.each_with_object({}) { |name, hash| hash[Severity.const_get(name)] = name}
|
20
|
+
def format_severity(severity)
|
21
|
+
SEVERITY_LABEL[severity] || 'ANY'
|
22
|
+
end
|
23
|
+
|
24
|
+
# define methods for a given level
|
25
|
+
def self.make_methods(str_level) # rubocop:disable Style/ClassMethodsDefinitions
|
26
|
+
int_level = ::Logger.const_get(str_level.upcase)
|
27
|
+
str_level = str_level.downcase
|
28
|
+
Kernel.send('lave'.reverse, <<-EOM, nil, __FILE__, __LINE__ + 1)
|
29
|
+
def #{str_level}(message = nil, &block)
|
30
|
+
add(#{int_level}, message, &block)
|
31
|
+
end
|
32
|
+
|
33
|
+
def #{str_level}?
|
34
|
+
level <= #{int_level}
|
35
|
+
end
|
36
|
+
|
37
|
+
def #{str_level}!
|
38
|
+
self.level = #{int_level}
|
39
|
+
end
|
40
|
+
EOM
|
41
|
+
end
|
42
|
+
Logger::Severity.constants.each { |severity| make_methods(severity) }
|
43
|
+
end
|
44
|
+
|
10
45
|
module Aspera
|
11
46
|
# Singleton object for logging
|
12
47
|
class Log
|
13
48
|
include Singleton
|
14
49
|
# where logs are sent to
|
15
50
|
LOG_TYPES = %i[stderr stdout syslog].freeze
|
51
|
+
@@format = :json # rubocop:disable Style/ClassVars
|
16
52
|
# class methods
|
17
53
|
class << self
|
18
54
|
# levels are :debug,:info,:warn,:error,fatal,:unknown
|
@@ -21,24 +57,23 @@ module Aspera
|
|
21
57
|
# get the logger object of singleton
|
22
58
|
def log; instance.logger; end
|
23
59
|
|
24
|
-
# dump object
|
60
|
+
# dump object suitable for Log.log.debug
|
25
61
|
# @param name string or symbol
|
26
62
|
# @param format either pp or json format
|
27
|
-
def dump(name, object
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
"#{name.to_s.green} (#{format})=\n#{result}"
|
39
|
-
end
|
63
|
+
def dump(name, object)
|
64
|
+
result =
|
65
|
+
case @@format
|
66
|
+
when :json
|
67
|
+
JSON.pretty_generate(object) rescue PP.pp(object, +'')
|
68
|
+
when :ruby
|
69
|
+
PP.pp(object, +'')
|
70
|
+
else
|
71
|
+
raise 'wrong parameter, expect ruby or json'
|
72
|
+
end
|
73
|
+
"#{name.to_s.green} (#{@@format})=\n#{result}"
|
40
74
|
end
|
41
75
|
|
76
|
+
# Capture the output of $stderr and log it at debug level
|
42
77
|
def capture_stderr
|
43
78
|
real_stderr = $stderr
|
44
79
|
$stderr = StringIO.new
|
@@ -69,16 +104,23 @@ module Aspera
|
|
69
104
|
# change underlying logger, but keep log level
|
70
105
|
def logger_type=(new_log_type)
|
71
106
|
current_severity_integer = @logger.level unless @logger.nil?
|
72
|
-
current_severity_integer = ENV
|
107
|
+
current_severity_integer = ENV.fetch('AS_LOG_LEVEL', nil) if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
|
73
108
|
current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
|
74
109
|
case new_log_type
|
75
110
|
when :stderr
|
76
|
-
# typed: Logger
|
77
111
|
@logger = Logger.new($stderr)
|
78
112
|
when :stdout
|
79
113
|
@logger = Logger.new($stdout)
|
80
114
|
when :syslog
|
81
115
|
require 'syslog/logger'
|
116
|
+
# the syslog class automatically creates methods from the severity names
|
117
|
+
# we just need to add the mapping (but syslog lowest is DEBUG)
|
118
|
+
1.upto(Logger::TRACE_MAX).each do |level|
|
119
|
+
Syslog::Logger.const_get(:LEVEL_MAP)[Logger.const_get("TRACE#{level}")] = Syslog::LOG_DEBUG
|
120
|
+
end
|
121
|
+
Logger::Severity.constants.each do |severity|
|
122
|
+
Syslog::Logger.make_methods(severity.downcase)
|
123
|
+
end
|
82
124
|
@logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
|
83
125
|
else
|
84
126
|
raise "unknown log type: #{new_log_type.class} #{new_log_type}"
|
data/lib/aspera/node.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/cli/error'
|
3
4
|
require 'aspera/fasp/transfer_spec'
|
4
5
|
require 'aspera/rest'
|
5
6
|
require 'aspera/oauth'
|
@@ -13,13 +14,22 @@ module Aspera
|
|
13
14
|
class Node < Aspera::Rest
|
14
15
|
# permissions
|
15
16
|
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
16
|
-
# prefix for ruby code for filter
|
17
|
+
# prefix for ruby code for filter (deprecated)
|
17
18
|
MATCH_EXEC_PREFIX = 'exec:'
|
19
|
+
MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
|
18
20
|
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
19
21
|
PATH_SEPARATOR = '/'
|
22
|
+
TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
|
23
|
+
SCOPE_USER = 'user:all'
|
24
|
+
SCOPE_ADMIN = 'admin:all'
|
25
|
+
SCOPE_PREFIX = 'node.'
|
26
|
+
SCOPE_SEPARATOR = ':'
|
27
|
+
SIGNATURE_DELIMITER = '==SIGNATURE=='
|
28
|
+
BEARER_TOKEN_VALIDITY_DEFAULT = 86400
|
29
|
+
BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
|
20
30
|
|
21
31
|
# register node special token decoder
|
22
|
-
Oauth.register_decoder(lambda{|token|
|
32
|
+
Oauth.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
|
23
33
|
|
24
34
|
# class instance variable, access with accessors on class
|
25
35
|
@use_standard_ports = true
|
@@ -27,16 +37,79 @@ module Aspera
|
|
27
37
|
class << self
|
28
38
|
attr_accessor :use_standard_ports
|
29
39
|
|
30
|
-
#
|
31
|
-
# if no prefix: regex
|
32
|
-
# if prefix: ruby code
|
33
|
-
# if expression is nil, then always match
|
40
|
+
# For access keys: provide expression to match entry in folder
|
34
41
|
def file_matcher(match_expression)
|
35
|
-
match_expression
|
36
|
-
|
37
|
-
|
42
|
+
case match_expression
|
43
|
+
when Proc then return match_expression
|
44
|
+
when Regexp then return ->(f){f['name'].match?(match_expression)}
|
45
|
+
when String
|
46
|
+
if match_expression.start_with?(MATCH_EXEC_PREFIX)
|
47
|
+
code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
|
48
|
+
Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
|
49
|
+
return Environment.secure_eval(code, __FILE__, __LINE__)
|
50
|
+
end
|
51
|
+
return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
|
52
|
+
when NilClass then return ->(_){true}
|
53
|
+
else raise Cli::BadArgument, "Invalid match expression type: #{match_expression.class}"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def file_matcher_from_argument(options)
|
58
|
+
return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
|
59
|
+
end
|
60
|
+
|
61
|
+
# node API scopes
|
62
|
+
def token_scope(access_key, scope)
|
63
|
+
return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
|
64
|
+
end
|
65
|
+
|
66
|
+
def decode_scope(scope)
|
67
|
+
items = scope.split(SCOPE_SEPARATOR, 2)
|
68
|
+
raise "invalid scope: #{scope}" unless items.length.eql?(2)
|
69
|
+
raise "invalid scope: #{scope}" unless items[0].start_with?(SCOPE_PREFIX)
|
70
|
+
return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
|
71
|
+
end
|
72
|
+
|
73
|
+
# Create an Aspera Node bearer token
|
74
|
+
# @param payload [String] JSON payload to be included in the token
|
75
|
+
# @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
|
76
|
+
def bearer_token(access_key:, payload:, private_key:)
|
77
|
+
raise 'payload shall be Hash' unless payload.is_a?(Hash)
|
78
|
+
raise 'missing user_id' unless payload.key?('user_id')
|
79
|
+
raise 'user_id must be a String' unless payload['user_id'].is_a?(String)
|
80
|
+
raise 'user_id must not be empty' if payload['user_id'].empty?
|
81
|
+
raise 'private_key shall be OpenSSL::PKey::RSA' unless private_key.is_a?(OpenSSL::PKey::RSA)
|
82
|
+
# manage convenience parameters
|
83
|
+
expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
|
84
|
+
payload.delete('_validity')
|
85
|
+
scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
|
86
|
+
payload.delete('_scope')
|
87
|
+
payload['scope'] ||= token_scope(access_key, scope)
|
88
|
+
payload['auth_type'] ||= 'access_key'
|
89
|
+
payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
|
90
|
+
payload_json = JSON.generate(payload)
|
91
|
+
return Base64.strict_encode64(Zlib::Deflate.deflate([
|
92
|
+
payload_json,
|
93
|
+
SIGNATURE_DELIMITER,
|
94
|
+
Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
|
95
|
+
''
|
96
|
+
].join("\n")))
|
97
|
+
end
|
98
|
+
|
99
|
+
def decode_bearer_token(token)
|
100
|
+
return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
|
101
|
+
end
|
102
|
+
|
103
|
+
def bearer_headers(bearer_auth, access_key: nil)
|
104
|
+
# if username is not provided, use the access key from the token
|
105
|
+
if access_key.nil?
|
106
|
+
access_key = Aspera::Node.decode_scope(Aspera::Node.decode_bearer_token(Oauth.bearer_extract(bearer_auth))['scope'])[:access_key]
|
107
|
+
raise "internal error #{access_key}" if access_key.nil?
|
38
108
|
end
|
39
|
-
return
|
109
|
+
return {
|
110
|
+
Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
|
111
|
+
'Authorization' => bearer_auth
|
112
|
+
}
|
40
113
|
end
|
41
114
|
end
|
42
115
|
|
@@ -51,6 +124,7 @@ module Aspera
|
|
51
124
|
# @param params [Hash] Rest parameters
|
52
125
|
# @param app_info [Hash,NilClass] special processing for AoC
|
53
126
|
def initialize(params:, app_info: nil, add_tspec: nil)
|
127
|
+
# init Rest
|
54
128
|
super(params)
|
55
129
|
@app_info = app_info
|
56
130
|
# this is added to transfer spec, for instance to add tags (COS)
|
@@ -84,17 +158,18 @@ module Aspera
|
|
84
158
|
return nil
|
85
159
|
end
|
86
160
|
|
87
|
-
#
|
161
|
+
# Recursively browse in a folder (with non-recursive method)
|
88
162
|
# sub folders are processed if the processing method returns true
|
89
163
|
# @param state [Object] state object sent to processing method
|
90
|
-
# @param method [Symbol] processing method name
|
91
164
|
# @param top_file_id [String] file id to start at (default = access key root file id)
|
92
165
|
# @param top_file_path [String] path of top folder (default = /)
|
93
|
-
|
166
|
+
# @param block [Proc] processing method, arguments: entry, path, state
|
167
|
+
def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
|
94
168
|
raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
|
95
|
-
raise
|
169
|
+
raise 'INTERNAL ERROR: Missing block' unless block
|
170
|
+
# start at top folder
|
96
171
|
folders_to_explore = [{id: top_file_id, path: top_file_path}]
|
97
|
-
Log.dump(:folders_to_explore, folders_to_explore)
|
172
|
+
Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
|
98
173
|
until folders_to_explore.empty?
|
99
174
|
current_item = folders_to_explore.shift
|
100
175
|
Log.log.debug{"searching #{current_item[:path]}".bg_green}
|
@@ -106,12 +181,12 @@ module Aspera
|
|
106
181
|
Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
|
107
182
|
[]
|
108
183
|
end
|
109
|
-
Log.dump(:folder_contents, folder_contents)
|
184
|
+
Log.log.debug{Log.dump(:folder_contents, folder_contents)}
|
110
185
|
folder_contents.each do |entry|
|
111
186
|
relative_path = File.join(current_item[:path], entry['name'])
|
112
|
-
Log.log.debug{"
|
187
|
+
Log.log.debug{"process_folder_tree checking #{relative_path}"}
|
113
188
|
# continue only if method returns true
|
114
|
-
next unless
|
189
|
+
next unless yield(entry, relative_path, state)
|
115
190
|
# entry type is file, folder or link
|
116
191
|
case entry['type']
|
117
192
|
when 'folder'
|
@@ -119,85 +194,74 @@ module Aspera
|
|
119
194
|
when 'link'
|
120
195
|
node_id_to_node(entry['target_node_id'])&.process_folder_tree(
|
121
196
|
state: state,
|
122
|
-
method: method,
|
123
197
|
top_file_id: entry['target_id'],
|
124
|
-
top_file_path: relative_path
|
198
|
+
top_file_path: relative_path,
|
199
|
+
&block)
|
125
200
|
end
|
126
201
|
end
|
127
202
|
end
|
128
203
|
end # process_folder_tree
|
129
204
|
|
130
|
-
# processing method to resolve a file path to id
|
131
|
-
# @returns true if processing need to continue
|
132
|
-
def process_resolve_node_path(entry, _path, state)
|
133
|
-
# stop digging here if not in right path
|
134
|
-
return false unless entry['name'].eql?(state[:path].first)
|
135
|
-
# ok it matches, so we remove the match
|
136
|
-
state[:path].shift
|
137
|
-
case entry['type']
|
138
|
-
when 'file'
|
139
|
-
# file must be terminal
|
140
|
-
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
141
|
-
# it's terminal, we found it
|
142
|
-
state[:result] = {api: self, file_id: entry['id']}
|
143
|
-
return false
|
144
|
-
when 'folder'
|
145
|
-
if state[:path].empty?
|
146
|
-
# we found it
|
147
|
-
state[:result] = {api: self, file_id: entry['id']}
|
148
|
-
return false
|
149
|
-
end
|
150
|
-
when 'link'
|
151
|
-
if state[:path].empty?
|
152
|
-
# we found it
|
153
|
-
other_node = node_id_to_node(entry['target_node_id'])
|
154
|
-
raise 'cannot resolve link' if other_node.nil?
|
155
|
-
state[:result] = {api: other_node, file_id: entry['target_id']}
|
156
|
-
return false
|
157
|
-
end
|
158
|
-
else
|
159
|
-
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
160
|
-
end
|
161
|
-
# continue to dig folder
|
162
|
-
return true
|
163
|
-
end
|
164
|
-
|
165
205
|
# Navigate the path from given file id
|
166
206
|
# @param top_file_id [String] id initial file id
|
167
207
|
# @param path [String] file path
|
168
208
|
# @return [Hash] {.api,.file_id}
|
169
209
|
def resolve_api_fid(top_file_id, path)
|
170
210
|
raise 'file id shall be String' unless top_file_id.is_a?(String)
|
211
|
+
process_last_link = path.end_with?(PATH_SEPARATOR)
|
171
212
|
path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
|
172
213
|
return {api: self, file_id: top_file_id} if path_elements.empty?
|
173
214
|
resolve_state = {path: path_elements, result: nil}
|
174
|
-
process_folder_tree(state: resolve_state,
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
215
|
+
process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
|
216
|
+
# this block is called recursively for each entry in folder
|
217
|
+
# stop digging here if not in right path
|
218
|
+
next false unless entry['name'].eql?(state[:path].first)
|
219
|
+
# ok it matches, so we remove the match
|
220
|
+
state[:path].shift
|
221
|
+
case entry['type']
|
222
|
+
when 'file'
|
223
|
+
# file must be terminal
|
224
|
+
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
225
|
+
# it's terminal, we found it
|
226
|
+
state[:result] = {api: self, file_id: entry['id']}
|
227
|
+
next false
|
228
|
+
when 'folder'
|
229
|
+
if state[:path].empty?
|
230
|
+
# we found it
|
231
|
+
state[:result] = {api: self, file_id: entry['id']}
|
232
|
+
next false
|
233
|
+
end
|
234
|
+
when 'link'
|
235
|
+
if state[:path].empty?
|
236
|
+
if process_last_link
|
237
|
+
# we found it
|
238
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
239
|
+
raise 'cannot resolve link' if other_node.nil?
|
240
|
+
state[:result] = {api: other_node, file_id: entry['target_id']}
|
241
|
+
else
|
242
|
+
# we found it but we do not process the link
|
243
|
+
state[:result] = {api: self, file_id: entry['id']}
|
244
|
+
end
|
245
|
+
next false
|
246
|
+
end
|
247
|
+
else
|
248
|
+
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
189
249
|
end
|
190
|
-
|
191
|
-
|
250
|
+
# continue to dig folder
|
251
|
+
next true
|
192
252
|
end
|
193
|
-
#
|
194
|
-
return
|
253
|
+
raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
|
254
|
+
return resolve_state[:result]
|
195
255
|
end
|
196
256
|
|
197
257
|
def find_files(top_file_id, test_block)
|
198
258
|
Log.log.debug{"find_files: file id=#{top_file_id}"}
|
199
259
|
find_state = {found: [], test_block: test_block}
|
200
|
-
process_folder_tree(state: find_state,
|
260
|
+
process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
|
261
|
+
state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
|
262
|
+
# test all files deeply
|
263
|
+
true
|
264
|
+
end
|
201
265
|
return find_state[:found]
|
202
266
|
end
|
203
267
|
|
@@ -212,6 +276,8 @@ module Aspera
|
|
212
276
|
case params[:auth][:type]
|
213
277
|
when :basic
|
214
278
|
ak_name = params[:auth][:username]
|
279
|
+
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
280
|
+
ak_token = Rest.basic_token(params[:auth][:username], params[:auth][:password])
|
215
281
|
when :oauth2
|
216
282
|
ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
|
217
283
|
# TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
|
@@ -235,39 +301,38 @@ module Aspera
|
|
235
301
|
add_tspec_info(transfer_spec)
|
236
302
|
transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
|
237
303
|
# add application specific tags (AoC)
|
238
|
-
|
239
|
-
the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
|
240
|
-
# add basic token
|
241
|
-
if transfer_spec['token'].nil?
|
242
|
-
ts_basic_token(transfer_spec)
|
243
|
-
end
|
304
|
+
app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
|
244
305
|
# add remote host info
|
245
306
|
if self.class.use_standard_ports
|
246
307
|
# get default TCP/UDP ports and transfer user
|
247
308
|
transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
|
248
309
|
# by default: same address as node API
|
249
310
|
transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
|
311
|
+
# AoC allows specification of other url
|
250
312
|
if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
|
251
313
|
transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
|
252
314
|
end
|
315
|
+
info = read('info')[:data]
|
316
|
+
# get the transfer user from info on access key
|
317
|
+
transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
|
318
|
+
# get settings from name.value array to hash key.value
|
319
|
+
settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
|
320
|
+
# check WSS ports
|
321
|
+
%w[wss_enabled wss_port].each do |i|
|
322
|
+
transfer_spec[i] = settings[i] if settings.key?(i)
|
323
|
+
end if settings.is_a?(Hash)
|
253
324
|
else
|
254
|
-
# retrieve values from API
|
255
|
-
|
325
|
+
# retrieve values from API (and keep a copy/cache)
|
326
|
+
@std_t_spec_cache ||= create(
|
256
327
|
'files/download_setup',
|
257
328
|
{transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
|
258
329
|
)[:data]['transfer_specs'].first['transfer_spec']
|
259
330
|
# copy some parts
|
260
|
-
|
331
|
+
TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
|
261
332
|
end
|
333
|
+
Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
|
334
|
+
unless transfer_spec['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
262
335
|
return transfer_spec
|
263
336
|
end
|
264
|
-
|
265
|
-
# set basic token in transfer spec
|
266
|
-
def ts_basic_token(ts)
|
267
|
-
Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
|
268
|
-
unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
269
|
-
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
270
|
-
ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
|
271
|
-
end
|
272
337
|
end
|
273
338
|
end
|