aspera-cli 4.13.0 → 4.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +81 -7
- data/CONTRIBUTING.md +22 -6
- data/README.md +2038 -1080
- data/bin/ascli +18 -9
- data/bin/asession +12 -14
- data/examples/dascli +1 -1
- data/examples/proxy.pac +1 -1
- data/examples/rubyc +24 -0
- data/lib/aspera/aoc.rb +219 -159
- data/lib/aspera/ascmd.rb +25 -14
- data/lib/aspera/cli/basic_auth_plugin.rb +12 -9
- data/lib/aspera/cli/error.rb +17 -0
- data/lib/aspera/cli/extended_value.rb +47 -12
- data/lib/aspera/cli/formatter.rb +260 -179
- data/lib/aspera/cli/hints.rb +80 -0
- data/lib/aspera/cli/main.rb +104 -156
- data/lib/aspera/cli/manager.rb +259 -209
- data/lib/aspera/cli/plugin.rb +123 -63
- data/lib/aspera/cli/plugins/alee.rb +2 -3
- data/lib/aspera/cli/plugins/aoc.rb +341 -261
- data/lib/aspera/cli/plugins/ats.rb +22 -21
- data/lib/aspera/cli/plugins/bss.rb +5 -5
- data/lib/aspera/cli/plugins/config.rb +578 -627
- data/lib/aspera/cli/plugins/console.rb +44 -6
- data/lib/aspera/cli/plugins/cos.rb +15 -17
- data/lib/aspera/cli/plugins/faspex.rb +114 -100
- data/lib/aspera/cli/plugins/faspex5.rb +411 -264
- data/lib/aspera/cli/plugins/node.rb +354 -259
- data/lib/aspera/cli/plugins/orchestrator.rb +61 -29
- data/lib/aspera/cli/plugins/preview.rb +82 -90
- data/lib/aspera/cli/plugins/server.rb +79 -32
- data/lib/aspera/cli/plugins/shares.rb +55 -42
- data/lib/aspera/cli/sync_actions.rb +68 -0
- data/lib/aspera/cli/transfer_agent.rb +66 -73
- data/lib/aspera/cli/transfer_progress.rb +74 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +12 -8
- data/lib/aspera/command_line_builder.rb +14 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/data/6 +0 -0
- data/lib/aspera/environment.rb +24 -9
- data/lib/aspera/fasp/agent_aspera.rb +126 -0
- data/lib/aspera/fasp/agent_base.rb +31 -77
- data/lib/aspera/fasp/agent_connect.rb +25 -21
- data/lib/aspera/fasp/agent_direct.rb +89 -103
- data/lib/aspera/fasp/agent_httpgw.rb +231 -149
- data/lib/aspera/fasp/agent_node.rb +41 -34
- data/lib/aspera/fasp/agent_trsdk.rb +75 -32
- data/lib/aspera/fasp/error_info.rb +4 -2
- data/lib/aspera/fasp/faux_file.rb +52 -0
- data/lib/aspera/fasp/installation.rb +53 -195
- data/lib/aspera/fasp/management.rb +244 -0
- data/lib/aspera/fasp/parameters.rb +71 -37
- data/lib/aspera/fasp/parameters.yaml +76 -8
- data/lib/aspera/fasp/products.rb +162 -0
- data/lib/aspera/fasp/resume_policy.rb +3 -3
- data/lib/aspera/fasp/transfer_spec.rb +7 -6
- data/lib/aspera/fasp/uri.rb +26 -24
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/hash_ext.rb +14 -4
- data/lib/aspera/json_rpc.rb +49 -0
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/line_logger.rb +23 -0
- data/lib/aspera/log.rb +58 -16
- data/lib/aspera/node.rb +157 -92
- data/lib/aspera/oauth.rb +37 -19
- data/lib/aspera/open_application.rb +4 -4
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +4 -2
- data/lib/aspera/preview/generator.rb +22 -35
- data/lib/aspera/preview/options.rb +2 -0
- data/lib/aspera/preview/terminal.rb +73 -16
- data/lib/aspera/preview/utils.rb +21 -28
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/rest.rb +136 -68
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +15 -14
- data/lib/aspera/rest_errors_aspera.rb +37 -34
- data/lib/aspera/secret_hider.rb +18 -15
- data/lib/aspera/ssh.rb +5 -2
- data/lib/aspera/sync.rb +127 -119
- data/lib/aspera/temp_file_manager.rb +10 -3
- data/lib/aspera/web_auth.rb +10 -7
- data/lib/aspera/web_server_simple.rb +9 -4
- data.tar.gz.sig +0 -0
- metadata +34 -17
- metadata.gz.sig +0 -0
- data/docs/test_env.conf +0 -186
- data/lib/aspera/cli/listener/line_dump.rb +0 -19
- data/lib/aspera/cli/listener/logger.rb +0 -22
- data/lib/aspera/cli/listener/progress.rb +0 -50
- data/lib/aspera/cli/listener/progress_multi.rb +0 -84
- data/lib/aspera/cli/plugins/sync.rb +0 -44
- data/lib/aspera/data/7 +0 -0
- data/lib/aspera/fasp/listener.rb +0 -13
@@ -3,11 +3,11 @@
|
|
3
3
|
# https://github.com/fastlane-community/security
|
4
4
|
require 'aspera/cli/info'
|
5
5
|
|
6
|
-
# enhance the gem to support other
|
6
|
+
# enhance the gem to support other key chains
|
7
7
|
module Aspera
|
8
8
|
module Keychain
|
9
9
|
module MacosSecurity
|
10
|
-
# keychain based on macOS keychain, using `security`
|
10
|
+
# keychain based on macOS keychain, using `security` command line
|
11
11
|
class Keychain
|
12
12
|
DOMAINS = %i[user system common dynamic].freeze
|
13
13
|
LIST_OPTIONS = {
|
@@ -32,12 +32,12 @@ module Aspera
|
|
32
32
|
getpass: :g
|
33
33
|
}.freeze
|
34
34
|
class << self
|
35
|
-
def execute(command, options=nil, supported=nil,
|
35
|
+
def execute(command, options=nil, supported=nil, last_opt=nil)
|
36
36
|
url = options&.delete(:url)
|
37
37
|
if !url.nil?
|
38
38
|
uri = URI.parse(url)
|
39
39
|
raise 'only https' unless uri.scheme.eql?('https')
|
40
|
-
options[:protocol] = 'htps'
|
40
|
+
options[:protocol] = 'htps' # cspell: disable-line
|
41
41
|
raise 'host required in URL' if uri.host.nil?
|
42
42
|
options[:server] = uri.host
|
43
43
|
options[:path] = uri.path unless ['', '/'].include?(uri.path)
|
@@ -50,28 +50,28 @@ module Aspera
|
|
50
50
|
cmd.push("-#{supported[k]}")
|
51
51
|
cmd.push(v.shellescape) unless v.empty?
|
52
52
|
end
|
53
|
-
cmd.push(
|
53
|
+
cmd.push(last_opt) unless last_opt.nil?
|
54
54
|
Log.log.debug{"executing>>#{cmd.join(' ')}"}
|
55
55
|
result = %x(#{cmd.join(' ')} 2>&1)
|
56
56
|
Log.log.debug{"result>>[#{result}]"}
|
57
57
|
return result
|
58
58
|
end
|
59
59
|
|
60
|
-
def
|
60
|
+
def key_chains(output)
|
61
61
|
output.split("\n").collect { |line| new(line.strip.gsub(/^"|"$/, '')) }
|
62
62
|
end
|
63
63
|
|
64
64
|
def default
|
65
|
-
|
65
|
+
key_chains(execute('default-keychain')).first
|
66
66
|
end
|
67
67
|
|
68
68
|
def login
|
69
|
-
|
69
|
+
key_chains(execute('login-keychain')).first
|
70
70
|
end
|
71
71
|
|
72
72
|
def list(options={})
|
73
73
|
raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
|
74
|
-
|
74
|
+
key_chains(execute('list-key_chains', options, LIST_OPTIONS))
|
75
75
|
end
|
76
76
|
|
77
77
|
def by_name(name)
|
@@ -88,14 +88,14 @@ module Aspera
|
|
88
88
|
[string].pack('H*').force_encoding('UTF-8')
|
89
89
|
end
|
90
90
|
|
91
|
-
def password(operation,
|
91
|
+
def password(operation, pass_type, options)
|
92
92
|
raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
|
93
|
-
raise "wrong
|
93
|
+
raise "wrong pass_type: #{pass_type}" unless %i[generic internet].include?(pass_type)
|
94
94
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
95
95
|
missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
|
96
96
|
raise "missing options: #{missing}" unless missing.empty?
|
97
97
|
options[:getpass] = '' if operation.eql?(:find)
|
98
|
-
output = self.class.execute("#{operation}-#{
|
98
|
+
output = self.class.execute("#{operation}-#{pass_type}-password", options, ADD_PASS_OPTIONS, @path)
|
99
99
|
raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
|
100
100
|
return nil unless operation.eql?(:find)
|
101
101
|
attributes = {}
|
@@ -143,7 +143,7 @@ module Aspera
|
|
143
143
|
raise 'not found' if info.nil?
|
144
144
|
result = options.clone
|
145
145
|
result[:secret] = info['password']
|
146
|
-
result[:description] = info['icmt']
|
146
|
+
result[:description] = info['icmt'] # cspell: disable-line
|
147
147
|
return result
|
148
148
|
end
|
149
149
|
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/log'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
# used for logging http
|
7
|
+
class LineLogger
|
8
|
+
def initialize(level)
|
9
|
+
@level = level
|
10
|
+
@buffer = []
|
11
|
+
end
|
12
|
+
|
13
|
+
def <<(string)
|
14
|
+
return if string.nil? || string.empty?
|
15
|
+
if !string.end_with?("\n")
|
16
|
+
@buffer.push(string)
|
17
|
+
return
|
18
|
+
end
|
19
|
+
Log.log.send(@level, @buffer.join('') + string.chomp)
|
20
|
+
@buffer.clear
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/aspera/log.rb
CHANGED
@@ -2,17 +2,53 @@
|
|
2
2
|
|
3
3
|
require 'aspera/colors'
|
4
4
|
require 'aspera/secret_hider'
|
5
|
+
require 'aspera/environment'
|
5
6
|
require 'logger'
|
6
7
|
require 'pp'
|
7
8
|
require 'json'
|
8
9
|
require 'singleton'
|
9
10
|
|
11
|
+
# extend Ruby logger with trace levels
|
12
|
+
class Logger
|
13
|
+
TRACE_MAX = 2
|
14
|
+
# add custom level to logger severity
|
15
|
+
module Severity
|
16
|
+
1.upto(TRACE_MAX).each { |level| const_set("TRACE#{level}", - level)}
|
17
|
+
end
|
18
|
+
# quick access to label
|
19
|
+
SEVERITY_LABEL = Severity.constants.each_with_object({}) { |name, hash| hash[Severity.const_get(name)] = name}
|
20
|
+
def format_severity(severity)
|
21
|
+
SEVERITY_LABEL[severity] || 'ANY'
|
22
|
+
end
|
23
|
+
|
24
|
+
# define methods for a given level
|
25
|
+
def self.make_methods(str_level) # rubocop:disable Style/ClassMethodsDefinitions
|
26
|
+
int_level = ::Logger.const_get(str_level.upcase)
|
27
|
+
str_level = str_level.downcase
|
28
|
+
Kernel.send('lave'.reverse, <<-EOM, nil, __FILE__, __LINE__ + 1)
|
29
|
+
def #{str_level}(message = nil, &block)
|
30
|
+
add(#{int_level}, message, &block)
|
31
|
+
end
|
32
|
+
|
33
|
+
def #{str_level}?
|
34
|
+
level <= #{int_level}
|
35
|
+
end
|
36
|
+
|
37
|
+
def #{str_level}!
|
38
|
+
self.level = #{int_level}
|
39
|
+
end
|
40
|
+
EOM
|
41
|
+
end
|
42
|
+
Logger::Severity.constants.each { |severity| make_methods(severity) }
|
43
|
+
end
|
44
|
+
|
10
45
|
module Aspera
|
11
46
|
# Singleton object for logging
|
12
47
|
class Log
|
13
48
|
include Singleton
|
14
49
|
# where logs are sent to
|
15
50
|
LOG_TYPES = %i[stderr stdout syslog].freeze
|
51
|
+
@@format = :json # rubocop:disable Style/ClassVars
|
16
52
|
# class methods
|
17
53
|
class << self
|
18
54
|
# levels are :debug,:info,:warn,:error,fatal,:unknown
|
@@ -21,24 +57,23 @@ module Aspera
|
|
21
57
|
# get the logger object of singleton
|
22
58
|
def log; instance.logger; end
|
23
59
|
|
24
|
-
# dump object
|
60
|
+
# dump object suitable for Log.log.debug
|
25
61
|
# @param name string or symbol
|
26
62
|
# @param format either pp or json format
|
27
|
-
def dump(name, object
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
"#{name.to_s.green} (#{format})=\n#{result}"
|
39
|
-
end
|
63
|
+
def dump(name, object)
|
64
|
+
result =
|
65
|
+
case @@format
|
66
|
+
when :json
|
67
|
+
JSON.pretty_generate(object) rescue PP.pp(object, +'')
|
68
|
+
when :ruby
|
69
|
+
PP.pp(object, +'')
|
70
|
+
else
|
71
|
+
raise 'wrong parameter, expect ruby or json'
|
72
|
+
end
|
73
|
+
"#{name.to_s.green} (#{@@format})=\n#{result}"
|
40
74
|
end
|
41
75
|
|
76
|
+
# Capture the output of $stderr and log it at debug level
|
42
77
|
def capture_stderr
|
43
78
|
real_stderr = $stderr
|
44
79
|
$stderr = StringIO.new
|
@@ -69,16 +104,23 @@ module Aspera
|
|
69
104
|
# change underlying logger, but keep log level
|
70
105
|
def logger_type=(new_log_type)
|
71
106
|
current_severity_integer = @logger.level unless @logger.nil?
|
72
|
-
current_severity_integer = ENV
|
107
|
+
current_severity_integer = ENV.fetch('AS_LOG_LEVEL', nil) if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
|
73
108
|
current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
|
74
109
|
case new_log_type
|
75
110
|
when :stderr
|
76
|
-
# typed: Logger
|
77
111
|
@logger = Logger.new($stderr)
|
78
112
|
when :stdout
|
79
113
|
@logger = Logger.new($stdout)
|
80
114
|
when :syslog
|
81
115
|
require 'syslog/logger'
|
116
|
+
# the syslog class automatically creates methods from the severity names
|
117
|
+
# we just need to add the mapping (but syslog lowest is DEBUG)
|
118
|
+
1.upto(Logger::TRACE_MAX).each do |level|
|
119
|
+
Syslog::Logger.const_get(:LEVEL_MAP)[Logger.const_get("TRACE#{level}")] = Syslog::LOG_DEBUG
|
120
|
+
end
|
121
|
+
Logger::Severity.constants.each do |severity|
|
122
|
+
Syslog::Logger.make_methods(severity.downcase)
|
123
|
+
end
|
82
124
|
@logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
|
83
125
|
else
|
84
126
|
raise "unknown log type: #{new_log_type.class} #{new_log_type}"
|
data/lib/aspera/node.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/cli/error'
|
3
4
|
require 'aspera/fasp/transfer_spec'
|
4
5
|
require 'aspera/rest'
|
5
6
|
require 'aspera/oauth'
|
@@ -13,13 +14,22 @@ module Aspera
|
|
13
14
|
class Node < Aspera::Rest
|
14
15
|
# permissions
|
15
16
|
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
16
|
-
# prefix for ruby code for filter
|
17
|
+
# prefix for ruby code for filter (deprecated)
|
17
18
|
MATCH_EXEC_PREFIX = 'exec:'
|
19
|
+
MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
|
18
20
|
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
19
21
|
PATH_SEPARATOR = '/'
|
22
|
+
TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
|
23
|
+
SCOPE_USER = 'user:all'
|
24
|
+
SCOPE_ADMIN = 'admin:all'
|
25
|
+
SCOPE_PREFIX = 'node.'
|
26
|
+
SCOPE_SEPARATOR = ':'
|
27
|
+
SIGNATURE_DELIMITER = '==SIGNATURE=='
|
28
|
+
BEARER_TOKEN_VALIDITY_DEFAULT = 86400
|
29
|
+
BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
|
20
30
|
|
21
31
|
# register node special token decoder
|
22
|
-
Oauth.register_decoder(lambda{|token|
|
32
|
+
Oauth.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
|
23
33
|
|
24
34
|
# class instance variable, access with accessors on class
|
25
35
|
@use_standard_ports = true
|
@@ -27,16 +37,79 @@ module Aspera
|
|
27
37
|
class << self
|
28
38
|
attr_accessor :use_standard_ports
|
29
39
|
|
30
|
-
#
|
31
|
-
# if no prefix: regex
|
32
|
-
# if prefix: ruby code
|
33
|
-
# if expression is nil, then always match
|
40
|
+
# For access keys: provide expression to match entry in folder
|
34
41
|
def file_matcher(match_expression)
|
35
|
-
match_expression
|
36
|
-
|
37
|
-
|
42
|
+
case match_expression
|
43
|
+
when Proc then return match_expression
|
44
|
+
when Regexp then return ->(f){f['name'].match?(match_expression)}
|
45
|
+
when String
|
46
|
+
if match_expression.start_with?(MATCH_EXEC_PREFIX)
|
47
|
+
code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
|
48
|
+
Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
|
49
|
+
return Environment.secure_eval(code, __FILE__, __LINE__)
|
50
|
+
end
|
51
|
+
return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
|
52
|
+
when NilClass then return ->(_){true}
|
53
|
+
else raise Cli::BadArgument, "Invalid match expression type: #{match_expression.class}"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def file_matcher_from_argument(options)
|
58
|
+
return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
|
59
|
+
end
|
60
|
+
|
61
|
+
# node API scopes
|
62
|
+
def token_scope(access_key, scope)
|
63
|
+
return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
|
64
|
+
end
|
65
|
+
|
66
|
+
def decode_scope(scope)
|
67
|
+
items = scope.split(SCOPE_SEPARATOR, 2)
|
68
|
+
raise "invalid scope: #{scope}" unless items.length.eql?(2)
|
69
|
+
raise "invalid scope: #{scope}" unless items[0].start_with?(SCOPE_PREFIX)
|
70
|
+
return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
|
71
|
+
end
|
72
|
+
|
73
|
+
# Create an Aspera Node bearer token
|
74
|
+
# @param payload [String] JSON payload to be included in the token
|
75
|
+
# @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
|
76
|
+
def bearer_token(access_key:, payload:, private_key:)
|
77
|
+
raise 'payload shall be Hash' unless payload.is_a?(Hash)
|
78
|
+
raise 'missing user_id' unless payload.key?('user_id')
|
79
|
+
raise 'user_id must be a String' unless payload['user_id'].is_a?(String)
|
80
|
+
raise 'user_id must not be empty' if payload['user_id'].empty?
|
81
|
+
raise 'private_key shall be OpenSSL::PKey::RSA' unless private_key.is_a?(OpenSSL::PKey::RSA)
|
82
|
+
# manage convenience parameters
|
83
|
+
expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
|
84
|
+
payload.delete('_validity')
|
85
|
+
scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
|
86
|
+
payload.delete('_scope')
|
87
|
+
payload['scope'] ||= token_scope(access_key, scope)
|
88
|
+
payload['auth_type'] ||= 'access_key'
|
89
|
+
payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
|
90
|
+
payload_json = JSON.generate(payload)
|
91
|
+
return Base64.strict_encode64(Zlib::Deflate.deflate([
|
92
|
+
payload_json,
|
93
|
+
SIGNATURE_DELIMITER,
|
94
|
+
Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
|
95
|
+
''
|
96
|
+
].join("\n")))
|
97
|
+
end
|
98
|
+
|
99
|
+
def decode_bearer_token(token)
|
100
|
+
return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
|
101
|
+
end
|
102
|
+
|
103
|
+
def bearer_headers(bearer_auth, access_key: nil)
|
104
|
+
# if username is not provided, use the access key from the token
|
105
|
+
if access_key.nil?
|
106
|
+
access_key = Aspera::Node.decode_scope(Aspera::Node.decode_bearer_token(Oauth.bearer_extract(bearer_auth))['scope'])[:access_key]
|
107
|
+
raise "internal error #{access_key}" if access_key.nil?
|
38
108
|
end
|
39
|
-
return
|
109
|
+
return {
|
110
|
+
Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
|
111
|
+
'Authorization' => bearer_auth
|
112
|
+
}
|
40
113
|
end
|
41
114
|
end
|
42
115
|
|
@@ -51,6 +124,7 @@ module Aspera
|
|
51
124
|
# @param params [Hash] Rest parameters
|
52
125
|
# @param app_info [Hash,NilClass] special processing for AoC
|
53
126
|
def initialize(params:, app_info: nil, add_tspec: nil)
|
127
|
+
# init Rest
|
54
128
|
super(params)
|
55
129
|
@app_info = app_info
|
56
130
|
# this is added to transfer spec, for instance to add tags (COS)
|
@@ -84,17 +158,18 @@ module Aspera
|
|
84
158
|
return nil
|
85
159
|
end
|
86
160
|
|
87
|
-
#
|
161
|
+
# Recursively browse in a folder (with non-recursive method)
|
88
162
|
# sub folders are processed if the processing method returns true
|
89
163
|
# @param state [Object] state object sent to processing method
|
90
|
-
# @param method [Symbol] processing method name
|
91
164
|
# @param top_file_id [String] file id to start at (default = access key root file id)
|
92
165
|
# @param top_file_path [String] path of top folder (default = /)
|
93
|
-
|
166
|
+
# @param block [Proc] processing method, arguments: entry, path, state
|
167
|
+
def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
|
94
168
|
raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
|
95
|
-
raise
|
169
|
+
raise 'INTERNAL ERROR: Missing block' unless block
|
170
|
+
# start at top folder
|
96
171
|
folders_to_explore = [{id: top_file_id, path: top_file_path}]
|
97
|
-
Log.dump(:folders_to_explore, folders_to_explore)
|
172
|
+
Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
|
98
173
|
until folders_to_explore.empty?
|
99
174
|
current_item = folders_to_explore.shift
|
100
175
|
Log.log.debug{"searching #{current_item[:path]}".bg_green}
|
@@ -106,12 +181,12 @@ module Aspera
|
|
106
181
|
Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
|
107
182
|
[]
|
108
183
|
end
|
109
|
-
Log.dump(:folder_contents, folder_contents)
|
184
|
+
Log.log.debug{Log.dump(:folder_contents, folder_contents)}
|
110
185
|
folder_contents.each do |entry|
|
111
186
|
relative_path = File.join(current_item[:path], entry['name'])
|
112
|
-
Log.log.debug{"
|
187
|
+
Log.log.debug{"process_folder_tree checking #{relative_path}"}
|
113
188
|
# continue only if method returns true
|
114
|
-
next unless
|
189
|
+
next unless yield(entry, relative_path, state)
|
115
190
|
# entry type is file, folder or link
|
116
191
|
case entry['type']
|
117
192
|
when 'folder'
|
@@ -119,85 +194,74 @@ module Aspera
|
|
119
194
|
when 'link'
|
120
195
|
node_id_to_node(entry['target_node_id'])&.process_folder_tree(
|
121
196
|
state: state,
|
122
|
-
method: method,
|
123
197
|
top_file_id: entry['target_id'],
|
124
|
-
top_file_path: relative_path
|
198
|
+
top_file_path: relative_path,
|
199
|
+
&block)
|
125
200
|
end
|
126
201
|
end
|
127
202
|
end
|
128
203
|
end # process_folder_tree
|
129
204
|
|
130
|
-
# processing method to resolve a file path to id
|
131
|
-
# @returns true if processing need to continue
|
132
|
-
def process_resolve_node_path(entry, _path, state)
|
133
|
-
# stop digging here if not in right path
|
134
|
-
return false unless entry['name'].eql?(state[:path].first)
|
135
|
-
# ok it matches, so we remove the match
|
136
|
-
state[:path].shift
|
137
|
-
case entry['type']
|
138
|
-
when 'file'
|
139
|
-
# file must be terminal
|
140
|
-
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
141
|
-
# it's terminal, we found it
|
142
|
-
state[:result] = {api: self, file_id: entry['id']}
|
143
|
-
return false
|
144
|
-
when 'folder'
|
145
|
-
if state[:path].empty?
|
146
|
-
# we found it
|
147
|
-
state[:result] = {api: self, file_id: entry['id']}
|
148
|
-
return false
|
149
|
-
end
|
150
|
-
when 'link'
|
151
|
-
if state[:path].empty?
|
152
|
-
# we found it
|
153
|
-
other_node = node_id_to_node(entry['target_node_id'])
|
154
|
-
raise 'cannot resolve link' if other_node.nil?
|
155
|
-
state[:result] = {api: other_node, file_id: entry['target_id']}
|
156
|
-
return false
|
157
|
-
end
|
158
|
-
else
|
159
|
-
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
160
|
-
end
|
161
|
-
# continue to dig folder
|
162
|
-
return true
|
163
|
-
end
|
164
|
-
|
165
205
|
# Navigate the path from given file id
|
166
206
|
# @param top_file_id [String] id initial file id
|
167
207
|
# @param path [String] file path
|
168
208
|
# @return [Hash] {.api,.file_id}
|
169
209
|
def resolve_api_fid(top_file_id, path)
|
170
210
|
raise 'file id shall be String' unless top_file_id.is_a?(String)
|
211
|
+
process_last_link = path.end_with?(PATH_SEPARATOR)
|
171
212
|
path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
|
172
213
|
return {api: self, file_id: top_file_id} if path_elements.empty?
|
173
214
|
resolve_state = {path: path_elements, result: nil}
|
174
|
-
process_folder_tree(state: resolve_state,
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
215
|
+
process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
|
216
|
+
# this block is called recursively for each entry in folder
|
217
|
+
# stop digging here if not in right path
|
218
|
+
next false unless entry['name'].eql?(state[:path].first)
|
219
|
+
# ok it matches, so we remove the match
|
220
|
+
state[:path].shift
|
221
|
+
case entry['type']
|
222
|
+
when 'file'
|
223
|
+
# file must be terminal
|
224
|
+
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
225
|
+
# it's terminal, we found it
|
226
|
+
state[:result] = {api: self, file_id: entry['id']}
|
227
|
+
next false
|
228
|
+
when 'folder'
|
229
|
+
if state[:path].empty?
|
230
|
+
# we found it
|
231
|
+
state[:result] = {api: self, file_id: entry['id']}
|
232
|
+
next false
|
233
|
+
end
|
234
|
+
when 'link'
|
235
|
+
if state[:path].empty?
|
236
|
+
if process_last_link
|
237
|
+
# we found it
|
238
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
239
|
+
raise 'cannot resolve link' if other_node.nil?
|
240
|
+
state[:result] = {api: other_node, file_id: entry['target_id']}
|
241
|
+
else
|
242
|
+
# we found it but we do not process the link
|
243
|
+
state[:result] = {api: self, file_id: entry['id']}
|
244
|
+
end
|
245
|
+
next false
|
246
|
+
end
|
247
|
+
else
|
248
|
+
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
189
249
|
end
|
190
|
-
|
191
|
-
|
250
|
+
# continue to dig folder
|
251
|
+
next true
|
192
252
|
end
|
193
|
-
#
|
194
|
-
return
|
253
|
+
raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
|
254
|
+
return resolve_state[:result]
|
195
255
|
end
|
196
256
|
|
197
257
|
def find_files(top_file_id, test_block)
|
198
258
|
Log.log.debug{"find_files: file id=#{top_file_id}"}
|
199
259
|
find_state = {found: [], test_block: test_block}
|
200
|
-
process_folder_tree(state: find_state,
|
260
|
+
process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
|
261
|
+
state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
|
262
|
+
# test all files deeply
|
263
|
+
true
|
264
|
+
end
|
201
265
|
return find_state[:found]
|
202
266
|
end
|
203
267
|
|
@@ -212,6 +276,8 @@ module Aspera
|
|
212
276
|
case params[:auth][:type]
|
213
277
|
when :basic
|
214
278
|
ak_name = params[:auth][:username]
|
279
|
+
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
280
|
+
ak_token = Rest.basic_token(params[:auth][:username], params[:auth][:password])
|
215
281
|
when :oauth2
|
216
282
|
ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
|
217
283
|
# TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
|
@@ -235,39 +301,38 @@ module Aspera
|
|
235
301
|
add_tspec_info(transfer_spec)
|
236
302
|
transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
|
237
303
|
# add application specific tags (AoC)
|
238
|
-
|
239
|
-
the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
|
240
|
-
# add basic token
|
241
|
-
if transfer_spec['token'].nil?
|
242
|
-
ts_basic_token(transfer_spec)
|
243
|
-
end
|
304
|
+
app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
|
244
305
|
# add remote host info
|
245
306
|
if self.class.use_standard_ports
|
246
307
|
# get default TCP/UDP ports and transfer user
|
247
308
|
transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
|
248
309
|
# by default: same address as node API
|
249
310
|
transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
|
311
|
+
# AoC allows specification of other url
|
250
312
|
if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
|
251
313
|
transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
|
252
314
|
end
|
315
|
+
info = read('info')[:data]
|
316
|
+
# get the transfer user from info on access key
|
317
|
+
transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
|
318
|
+
# get settings from name.value array to hash key.value
|
319
|
+
settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
|
320
|
+
# check WSS ports
|
321
|
+
%w[wss_enabled wss_port].each do |i|
|
322
|
+
transfer_spec[i] = settings[i] if settings.key?(i)
|
323
|
+
end if settings.is_a?(Hash)
|
253
324
|
else
|
254
|
-
# retrieve values from API
|
255
|
-
|
325
|
+
# retrieve values from API (and keep a copy/cache)
|
326
|
+
@std_t_spec_cache ||= create(
|
256
327
|
'files/download_setup',
|
257
328
|
{transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
|
258
329
|
)[:data]['transfer_specs'].first['transfer_spec']
|
259
330
|
# copy some parts
|
260
|
-
|
331
|
+
TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
|
261
332
|
end
|
333
|
+
Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
|
334
|
+
unless transfer_spec['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
262
335
|
return transfer_spec
|
263
336
|
end
|
264
|
-
|
265
|
-
# set basic token in transfer spec
|
266
|
-
def ts_basic_token(ts)
|
267
|
-
Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
|
268
|
-
unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
269
|
-
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
270
|
-
ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
|
271
|
-
end
|
272
337
|
end
|
273
338
|
end
|