aspera-cli 4.12.0 → 4.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +45 -5
- data/CONTRIBUTING.md +113 -22
- data/README.md +1289 -754
- data/bin/ascli +3 -3
- data/examples/dascli +1 -1
- data/examples/rubyc +24 -0
- data/lib/aspera/aoc.rb +63 -74
- data/lib/aspera/ascmd.rb +5 -3
- data/lib/aspera/cli/basic_auth_plugin.rb +6 -6
- data/lib/aspera/cli/extended_value.rb +24 -37
- data/lib/aspera/cli/formatter.rb +23 -25
- data/lib/aspera/cli/info.rb +2 -4
- data/lib/aspera/cli/main.rb +27 -27
- data/lib/aspera/cli/manager.rb +143 -120
- data/lib/aspera/cli/plugin.rb +88 -43
- data/lib/aspera/cli/plugins/alee.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +235 -104
- data/lib/aspera/cli/plugins/ats.rb +16 -18
- data/lib/aspera/cli/plugins/bss.rb +3 -3
- data/lib/aspera/cli/plugins/config.rb +190 -373
- data/lib/aspera/cli/plugins/console.rb +4 -6
- data/lib/aspera/cli/plugins/cos.rb +12 -13
- data/lib/aspera/cli/plugins/faspex.rb +21 -21
- data/lib/aspera/cli/plugins/faspex5.rb +399 -150
- data/lib/aspera/cli/plugins/node.rb +260 -174
- data/lib/aspera/cli/plugins/orchestrator.rb +15 -18
- data/lib/aspera/cli/plugins/preview.rb +40 -62
- data/lib/aspera/cli/plugins/server.rb +33 -16
- data/lib/aspera/cli/plugins/shares.rb +24 -33
- data/lib/aspera/cli/plugins/sync.rb +6 -6
- data/lib/aspera/cli/transfer_agent.rb +47 -30
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +9 -7
- data/lib/aspera/command_line_builder.rb +2 -1
- data/lib/aspera/cos_node.rb +1 -1
- data/lib/aspera/data/6 +0 -0
- data/lib/aspera/environment.rb +7 -3
- data/lib/aspera/fasp/agent_connect.rb +6 -1
- data/lib/aspera/fasp/agent_direct.rb +17 -17
- data/lib/aspera/fasp/agent_httpgw.rb +138 -60
- data/lib/aspera/fasp/agent_node.rb +14 -4
- data/lib/aspera/fasp/agent_trsdk.rb +2 -0
- data/lib/aspera/fasp/error_info.rb +2 -0
- data/lib/aspera/fasp/installation.rb +19 -19
- data/lib/aspera/fasp/parameters.rb +29 -20
- data/lib/aspera/fasp/parameters.yaml +5 -2
- data/lib/aspera/fasp/resume_policy.rb +3 -3
- data/lib/aspera/fasp/transfer_spec.rb +8 -5
- data/lib/aspera/fasp/uri.rb +23 -21
- data/lib/aspera/faspex_gw.rb +1 -0
- data/lib/aspera/faspex_postproc.rb +3 -3
- data/lib/aspera/hash_ext.rb +12 -2
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/log.rb +1 -0
- data/lib/aspera/node.rb +73 -84
- data/lib/aspera/oauth.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/preview/file_types.rb +8 -6
- data/lib/aspera/preview/generator.rb +23 -11
- data/lib/aspera/preview/options.rb +3 -2
- data/lib/aspera/preview/terminal.rb +80 -0
- data/lib/aspera/preview/utils.rb +11 -11
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/rest.rb +42 -4
- data/lib/aspera/rest_call_error.rb +3 -1
- data/lib/aspera/secret_hider.rb +10 -5
- data/lib/aspera/ssh.rb +1 -1
- data/lib/aspera/sync.rb +41 -33
- data/lib/aspera/web_server_simple.rb +22 -18
- data.tar.gz.sig +0 -0
- metadata +40 -48
- metadata.gz.sig +0 -0
- data/docs/test_env.conf +0 -179
- data/examples/aoc.rb +0 -30
- data/examples/faspex4.rb +0 -94
- data/examples/node.rb +0 -96
- data/examples/server.rb +0 -93
- data/lib/aspera/data/7 +0 -0
data/lib/aspera/fasp/uri.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# cspell:words httpport targetrate minrate bwcap createpath lockpolicy lockminrate
|
4
|
+
|
3
5
|
require 'aspera/log'
|
4
6
|
require 'aspera/command_line_builder'
|
5
7
|
|
@@ -7,8 +9,8 @@ module Aspera
|
|
7
9
|
module Fasp
|
8
10
|
# translates a "faspe:" URI (used in Faspex 4) into transfer spec hash
|
9
11
|
class Uri
|
10
|
-
def initialize(
|
11
|
-
@fasp_uri = URI.parse(
|
12
|
+
def initialize(fasp_link)
|
13
|
+
@fasp_uri = URI.parse(fasp_link.gsub(' ', '%20'))
|
12
14
|
# TODO: check scheme is faspe
|
13
15
|
end
|
14
16
|
|
@@ -18,32 +20,32 @@ module Aspera
|
|
18
20
|
result_ts['remote_user'] = @fasp_uri.user
|
19
21
|
result_ts['ssh_port'] = @fasp_uri.port
|
20
22
|
result_ts['paths'] = [{'source' => URI.decode_www_form_component(@fasp_uri.path)}]
|
21
|
-
# faspex does not encode trailing base64
|
23
|
+
# faspex does not encode trailing base64 padding, fix that to be able to decode properly
|
22
24
|
fixed_query = @fasp_uri.query.gsub(/(=+)$/){|x|'%3D' * x.length}
|
23
25
|
|
24
26
|
URI.decode_www_form(fixed_query).each do |i|
|
25
27
|
name = i[0]
|
26
28
|
value = i[1]
|
27
29
|
case name
|
28
|
-
when 'cookie'
|
29
|
-
when 'token'
|
30
|
-
when 'sshfp'
|
31
|
-
when 'policy'
|
32
|
-
when 'httpport'
|
33
|
-
when 'targetrate'
|
34
|
-
when 'minrate'
|
35
|
-
when 'port'
|
36
|
-
when 'bwcap'
|
37
|
-
when 'enc'
|
38
|
-
when 'tags64'
|
39
|
-
when 'createpath'
|
40
|
-
when 'fallback'
|
41
|
-
when 'lockpolicy'
|
30
|
+
when 'cookie' then result_ts['cookie'] = value
|
31
|
+
when 'token' then result_ts['token'] = value
|
32
|
+
when 'sshfp' then result_ts['sshfp'] = value
|
33
|
+
when 'policy' then result_ts['rate_policy'] = value
|
34
|
+
when 'httpport' then result_ts['http_fallback_port'] = value.to_i
|
35
|
+
when 'targetrate' then result_ts['target_rate_kbps'] = value.to_i
|
36
|
+
when 'minrate' then result_ts['min_rate_kbps'] = value.to_i
|
37
|
+
when 'port' then result_ts['fasp_port'] = value.to_i
|
38
|
+
when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
|
39
|
+
when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub(/--/, '-')
|
40
|
+
when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
|
41
|
+
when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
|
42
|
+
when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
|
43
|
+
when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
|
42
44
|
when 'lockminrate' then result_ts['lock_min_rate'] = CommandLineBuilder.yes_to_true(value)
|
43
|
-
when 'auth'
|
44
|
-
when 'v'
|
45
|
-
when 'protect'
|
46
|
-
else
|
45
|
+
when 'auth' then Log.log.debug{"ignoring #{name}=#{value}"} # Not used (yes/no)
|
46
|
+
when 'v' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch Not used (2)
|
47
|
+
when 'protect' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch TODO: what is this ?
|
48
|
+
else Log.log.warn{"URI parameter ignored: #{name} = #{value}"}
|
47
49
|
end
|
48
50
|
end
|
49
51
|
return result_ts
|
data/lib/aspera/faspex_gw.rb
CHANGED
@@ -13,13 +13,13 @@ module Aspera
|
|
13
13
|
def initialize(server, parameters)
|
14
14
|
raise 'parameters must be Hash' unless parameters.is_a?(Hash)
|
15
15
|
@parameters = parameters.symbolize_keys
|
16
|
-
Log.dump(:
|
16
|
+
Log.dump(:post_proc_parameters, @parameters)
|
17
17
|
raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
|
18
18
|
@parameters[:script_folder] ||= '.'
|
19
19
|
@parameters[:fail_on_error] ||= false
|
20
20
|
@parameters[:timeout_seconds] ||= 60
|
21
21
|
super(server)
|
22
|
-
Log.log.debug{
|
22
|
+
Log.log.debug{'Faspex4PostProcServlet initialized'}
|
23
23
|
end
|
24
24
|
|
25
25
|
def do_POST(request, response)
|
@@ -39,7 +39,7 @@ module Aspera
|
|
39
39
|
return
|
40
40
|
end
|
41
41
|
# build script path by removing domain, and adding script folder
|
42
|
-
script_file = request.path[@parameters[:root].size
|
42
|
+
script_file = request.path[@parameters[:root].size..]
|
43
43
|
Log.log.debug{"script file=#{script_file}"}
|
44
44
|
script_path = File.join(@parameters[:script_folder], script_file)
|
45
45
|
Log.log.debug{"script=#{script_path}"}
|
data/lib/aspera/hash_ext.rb
CHANGED
@@ -2,11 +2,21 @@
|
|
2
2
|
|
3
3
|
class ::Hash
|
4
4
|
def deep_merge(second)
|
5
|
-
merge(second){|_key, v1, v2|Hash
|
5
|
+
merge(second){|_key, v1, v2|v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge(v2) : v2}
|
6
6
|
end
|
7
7
|
|
8
8
|
def deep_merge!(second)
|
9
|
-
merge!(second){|_key, v1, v2|Hash
|
9
|
+
merge!(second){|_key, v1, v2|v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge!(v2) : v2}
|
10
|
+
end
|
11
|
+
|
12
|
+
def deep_do(memory=nil, &block)
|
13
|
+
each do |key, value|
|
14
|
+
if value.is_a?(Hash)
|
15
|
+
value.deep_do(memory, &block)
|
16
|
+
else
|
17
|
+
yield(self, key, value, memory)
|
18
|
+
end
|
19
|
+
end
|
10
20
|
end
|
11
21
|
end
|
12
22
|
|
@@ -3,11 +3,11 @@
|
|
3
3
|
# https://github.com/fastlane-community/security
|
4
4
|
require 'aspera/cli/info'
|
5
5
|
|
6
|
-
# enhance the gem to support other
|
6
|
+
# enhance the gem to support other key chains
|
7
7
|
module Aspera
|
8
8
|
module Keychain
|
9
9
|
module MacosSecurity
|
10
|
-
# keychain based on macOS keychain, using `security`
|
10
|
+
# keychain based on macOS keychain, using `security` command line
|
11
11
|
class Keychain
|
12
12
|
DOMAINS = %i[user system common dynamic].freeze
|
13
13
|
LIST_OPTIONS = {
|
@@ -32,12 +32,12 @@ module Aspera
|
|
32
32
|
getpass: :g
|
33
33
|
}.freeze
|
34
34
|
class << self
|
35
|
-
def execute(command, options=nil, supported=nil,
|
35
|
+
def execute(command, options=nil, supported=nil, last_opt=nil)
|
36
36
|
url = options&.delete(:url)
|
37
37
|
if !url.nil?
|
38
38
|
uri = URI.parse(url)
|
39
39
|
raise 'only https' unless uri.scheme.eql?('https')
|
40
|
-
options[:protocol] = 'htps'
|
40
|
+
options[:protocol] = 'htps' # cspell: disable-line
|
41
41
|
raise 'host required in URL' if uri.host.nil?
|
42
42
|
options[:server] = uri.host
|
43
43
|
options[:path] = uri.path unless ['', '/'].include?(uri.path)
|
@@ -50,28 +50,28 @@ module Aspera
|
|
50
50
|
cmd.push("-#{supported[k]}")
|
51
51
|
cmd.push(v.shellescape) unless v.empty?
|
52
52
|
end
|
53
|
-
cmd.push(
|
53
|
+
cmd.push(last_opt) unless last_opt.nil?
|
54
54
|
Log.log.debug{"executing>>#{cmd.join(' ')}"}
|
55
55
|
result = %x(#{cmd.join(' ')} 2>&1)
|
56
56
|
Log.log.debug{"result>>[#{result}]"}
|
57
57
|
return result
|
58
58
|
end
|
59
59
|
|
60
|
-
def
|
60
|
+
def key_chains(output)
|
61
61
|
output.split("\n").collect { |line| new(line.strip.gsub(/^"|"$/, '')) }
|
62
62
|
end
|
63
63
|
|
64
64
|
def default
|
65
|
-
|
65
|
+
key_chains(execute('default-keychain')).first
|
66
66
|
end
|
67
67
|
|
68
68
|
def login
|
69
|
-
|
69
|
+
key_chains(execute('login-keychain')).first
|
70
70
|
end
|
71
71
|
|
72
72
|
def list(options={})
|
73
73
|
raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
|
74
|
-
|
74
|
+
key_chains(execute('list-key_chains', options, LIST_OPTIONS))
|
75
75
|
end
|
76
76
|
|
77
77
|
def by_name(name)
|
@@ -88,14 +88,14 @@ module Aspera
|
|
88
88
|
[string].pack('H*').force_encoding('UTF-8')
|
89
89
|
end
|
90
90
|
|
91
|
-
def password(operation,
|
91
|
+
def password(operation, pass_type, options)
|
92
92
|
raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
|
93
|
-
raise "wrong
|
93
|
+
raise "wrong pass_type: #{pass_type}" unless %i[generic internet].include?(pass_type)
|
94
94
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
95
95
|
missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
|
96
96
|
raise "missing options: #{missing}" unless missing.empty?
|
97
97
|
options[:getpass] = '' if operation.eql?(:find)
|
98
|
-
output = self.class.execute("#{operation}-#{
|
98
|
+
output = self.class.execute("#{operation}-#{pass_type}-password", options, ADD_PASS_OPTIONS, @path)
|
99
99
|
raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
|
100
100
|
return nil unless operation.eql?(:find)
|
101
101
|
attributes = {}
|
@@ -143,7 +143,7 @@ module Aspera
|
|
143
143
|
raise 'not found' if info.nil?
|
144
144
|
result = options.clone
|
145
145
|
result[:secret] = info['password']
|
146
|
-
result[:description] = info['icmt']
|
146
|
+
result[:description] = info['icmt'] # cspell: disable-line
|
147
147
|
return result
|
148
148
|
end
|
149
149
|
|
data/lib/aspera/log.rb
CHANGED
data/lib/aspera/node.rb
CHANGED
@@ -17,6 +17,7 @@ module Aspera
|
|
17
17
|
MATCH_EXEC_PREFIX = 'exec:'
|
18
18
|
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
19
19
|
PATH_SEPARATOR = '/'
|
20
|
+
TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
|
20
21
|
|
21
22
|
# register node special token decoder
|
22
23
|
Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
|
@@ -40,7 +41,9 @@ module Aspera
|
|
40
41
|
end
|
41
42
|
end
|
42
43
|
|
43
|
-
|
44
|
+
# fields in @app_info
|
45
|
+
REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
|
46
|
+
# methods of @app_info[:api]
|
44
47
|
REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
|
45
48
|
private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
|
46
49
|
|
@@ -71,21 +74,27 @@ module Aspera
|
|
71
74
|
|
72
75
|
# @returns [Aspera::Node] a Node or nil
|
73
76
|
def node_id_to_node(node_id)
|
74
|
-
|
75
|
-
|
77
|
+
if !@app_info.nil?
|
78
|
+
return self if node_id.eql?(@app_info[:node_info]['id'])
|
79
|
+
return @app_info[:api].node_api_from(
|
80
|
+
node_id: node_id,
|
81
|
+
workspace_id: @app_info[:workspace_id],
|
82
|
+
workspace_name: @app_info[:workspace_name])
|
83
|
+
end
|
76
84
|
Log.log.warn{"cannot resolve link with node id #{node_id}"}
|
77
85
|
return nil
|
78
86
|
end
|
79
87
|
|
80
|
-
#
|
88
|
+
# Recursively browse in a folder (with non-recursive method)
|
81
89
|
# sub folders are processed if the processing method returns true
|
82
90
|
# @param state [Object] state object sent to processing method
|
83
|
-
# @param method [Symbol] processing method name
|
84
91
|
# @param top_file_id [String] file id to start at (default = access key root file id)
|
85
92
|
# @param top_file_path [String] path of top folder (default = /)
|
86
|
-
|
93
|
+
# @param block [Proc] processing method, args: entry, path, state
|
94
|
+
def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
|
87
95
|
raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
|
88
|
-
raise
|
96
|
+
raise 'INTERNAL ERROR: Missing block' unless block
|
97
|
+
# start at top folder
|
89
98
|
folders_to_explore = [{id: top_file_id, path: top_file_path}]
|
90
99
|
Log.dump(:folders_to_explore, folders_to_explore)
|
91
100
|
until folders_to_explore.empty?
|
@@ -102,9 +111,9 @@ module Aspera
|
|
102
111
|
Log.dump(:folder_contents, folder_contents)
|
103
112
|
folder_contents.each do |entry|
|
104
113
|
relative_path = File.join(current_item[:path], entry['name'])
|
105
|
-
Log.log.debug{"
|
114
|
+
Log.log.debug{"process_folder_tree checking #{relative_path}"}
|
106
115
|
# continue only if method returns true
|
107
|
-
next unless
|
116
|
+
next unless yield(entry, relative_path, state)
|
108
117
|
# entry type is file, folder or link
|
109
118
|
case entry['type']
|
110
119
|
when 'folder'
|
@@ -112,85 +121,74 @@ module Aspera
|
|
112
121
|
when 'link'
|
113
122
|
node_id_to_node(entry['target_node_id'])&.process_folder_tree(
|
114
123
|
state: state,
|
115
|
-
method: method,
|
116
124
|
top_file_id: entry['target_id'],
|
117
|
-
top_file_path: relative_path
|
125
|
+
top_file_path: relative_path,
|
126
|
+
&block)
|
118
127
|
end
|
119
128
|
end
|
120
129
|
end
|
121
130
|
end # process_folder_tree
|
122
131
|
|
123
|
-
# processing method to resolve a file path to id
|
124
|
-
# @returns true if processing need to continue
|
125
|
-
def process_resolve_node_path(entry, _path, state)
|
126
|
-
# stop digging here if not in right path
|
127
|
-
return false unless entry['name'].eql?(state[:path].first)
|
128
|
-
# ok it matches, so we remove the match
|
129
|
-
state[:path].shift
|
130
|
-
case entry['type']
|
131
|
-
when 'file'
|
132
|
-
# file must be terminal
|
133
|
-
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
134
|
-
# it's terminal, we found it
|
135
|
-
state[:result] = {api: self, file_id: entry['id']}
|
136
|
-
return false
|
137
|
-
when 'folder'
|
138
|
-
if state[:path].empty?
|
139
|
-
# we found it
|
140
|
-
state[:result] = {api: self, file_id: entry['id']}
|
141
|
-
return false
|
142
|
-
end
|
143
|
-
when 'link'
|
144
|
-
if state[:path].empty?
|
145
|
-
# we found it
|
146
|
-
other_node = node_id_to_node(entry['target_node_id'])
|
147
|
-
raise 'cannot resolve link' if other_node.nil?
|
148
|
-
state[:result] = {api: other_node, file_id: entry['target_id']}
|
149
|
-
return false
|
150
|
-
end
|
151
|
-
else
|
152
|
-
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
153
|
-
end
|
154
|
-
# continue to dig folder
|
155
|
-
return true
|
156
|
-
end
|
157
|
-
|
158
132
|
# Navigate the path from given file id
|
159
133
|
# @param top_file_id [String] id initial file id
|
160
134
|
# @param path [String] file path
|
161
135
|
# @return [Hash] {.api,.file_id}
|
162
136
|
def resolve_api_fid(top_file_id, path)
|
163
137
|
raise 'file id shall be String' unless top_file_id.is_a?(String)
|
138
|
+
process_last_link = path.end_with?(PATH_SEPARATOR)
|
164
139
|
path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
|
165
140
|
return {api: self, file_id: top_file_id} if path_elements.empty?
|
166
141
|
resolve_state = {path: path_elements, result: nil}
|
167
|
-
process_folder_tree(state: resolve_state,
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
142
|
+
process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
|
143
|
+
# this block is called recursively for each entry in folder
|
144
|
+
# stop digging here if not in right path
|
145
|
+
next false unless entry['name'].eql?(state[:path].first)
|
146
|
+
# ok it matches, so we remove the match
|
147
|
+
state[:path].shift
|
148
|
+
case entry['type']
|
149
|
+
when 'file'
|
150
|
+
# file must be terminal
|
151
|
+
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
152
|
+
# it's terminal, we found it
|
153
|
+
state[:result] = {api: self, file_id: entry['id']}
|
154
|
+
next false
|
155
|
+
when 'folder'
|
156
|
+
if state[:path].empty?
|
157
|
+
# we found it
|
158
|
+
state[:result] = {api: self, file_id: entry['id']}
|
159
|
+
next false
|
160
|
+
end
|
161
|
+
when 'link'
|
162
|
+
if state[:path].empty?
|
163
|
+
if process_last_link
|
164
|
+
# we found it
|
165
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
166
|
+
raise 'cannot resolve link' if other_node.nil?
|
167
|
+
state[:result] = {api: other_node, file_id: entry['target_id']}
|
168
|
+
else
|
169
|
+
# we found it but we do not process the link
|
170
|
+
state[:result] = {api: self, file_id: entry['id']}
|
171
|
+
end
|
172
|
+
next false
|
173
|
+
end
|
174
|
+
else
|
175
|
+
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
182
176
|
end
|
183
|
-
|
184
|
-
|
177
|
+
# continue to dig folder
|
178
|
+
next true
|
185
179
|
end
|
186
|
-
#
|
187
|
-
return
|
180
|
+
raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
|
181
|
+
return resolve_state[:result]
|
188
182
|
end
|
189
183
|
|
190
184
|
def find_files(top_file_id, test_block)
|
191
185
|
Log.log.debug{"find_files: file id=#{top_file_id}"}
|
192
186
|
find_state = {found: [], test_block: test_block}
|
193
|
-
process_folder_tree(state: find_state,
|
187
|
+
process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
|
188
|
+
state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
|
189
|
+
# test all files deeply
|
190
|
+
true
|
191
|
+
end
|
194
192
|
return find_state[:found]
|
195
193
|
end
|
196
194
|
|
@@ -205,6 +203,8 @@ module Aspera
|
|
205
203
|
case params[:auth][:type]
|
206
204
|
when :basic
|
207
205
|
ak_name = params[:auth][:username]
|
206
|
+
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
207
|
+
ak_token = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
|
208
208
|
when :oauth2
|
209
209
|
ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
|
210
210
|
# TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
|
@@ -216,7 +216,7 @@ module Aspera
|
|
216
216
|
'direction' => direction,
|
217
217
|
'token' => ak_token,
|
218
218
|
'tags' => {
|
219
|
-
|
219
|
+
Fasp::TransferSpec::TAG_RESERVED => {
|
220
220
|
'node' => {
|
221
221
|
'access_key' => ak_name,
|
222
222
|
'file_id' => file_id
|
@@ -228,12 +228,7 @@ module Aspera
|
|
228
228
|
add_tspec_info(transfer_spec)
|
229
229
|
transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
|
230
230
|
# add application specific tags (AoC)
|
231
|
-
|
232
|
-
the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
|
233
|
-
# add basic token
|
234
|
-
if transfer_spec['token'].nil?
|
235
|
-
ts_basic_token(transfer_spec)
|
236
|
-
end
|
231
|
+
app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
|
237
232
|
# add remote host info
|
238
233
|
if self.class.use_standard_ports
|
239
234
|
# get default TCP/UDP ports and transfer user
|
@@ -244,23 +239,17 @@ module Aspera
|
|
244
239
|
transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
|
245
240
|
end
|
246
241
|
else
|
247
|
-
# retrieve values from API
|
248
|
-
|
242
|
+
# retrieve values from API (and keep a copy/cache)
|
243
|
+
@std_t_spec_cache ||= create(
|
249
244
|
'files/download_setup',
|
250
245
|
{transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
|
251
246
|
)[:data]['transfer_specs'].first['transfer_spec']
|
252
247
|
# copy some parts
|
253
|
-
|
248
|
+
TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
|
254
249
|
end
|
250
|
+
Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
|
251
|
+
unless transfer_spec['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
255
252
|
return transfer_spec
|
256
253
|
end
|
257
|
-
|
258
|
-
# set basic token in transfer spec
|
259
|
-
def ts_basic_token(ts)
|
260
|
-
Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
|
261
|
-
unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
262
|
-
raise 'ERROR: no secret in node object' unless params[:auth][:password]
|
263
|
-
ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
|
264
|
-
end
|
265
254
|
end
|
266
255
|
end
|
data/lib/aspera/oauth.rb
CHANGED
@@ -87,6 +87,7 @@ module Aspera
|
|
87
87
|
# @param lambda_create called to create token
|
88
88
|
# @param id_create called to generate unique id for token, for cache
|
89
89
|
def register_token_creator(id, lambda_create, id_create)
|
90
|
+
Log.log.debug{"registering token creator #{id}"}
|
90
91
|
raise 'ERROR: requites Symbol and 2 lambdas' unless id.is_a?(Symbol) && lambda_create.is_a?(Proc) && id_create.is_a?(Proc)
|
91
92
|
@create_handlers[id] = lambda_create
|
92
93
|
@id_handlers[id] = id_create
|
@@ -94,7 +95,7 @@ module Aspera
|
|
94
95
|
|
95
96
|
# @return one of the registered creators for the given create type
|
96
97
|
def token_creator(id)
|
97
|
-
raise "token grant method unknown: #{id}
|
98
|
+
raise "token grant method unknown: '#{id}' (#{id.class})" unless @create_handlers.key?(id)
|
98
99
|
@create_handlers[id]
|
99
100
|
end
|
100
101
|
|
@@ -191,7 +192,7 @@ module Aspera
|
|
191
192
|
# replace default values
|
192
193
|
@generic_parameters = DEFAULT_CREATE_PARAMS.deep_merge(a_params)
|
193
194
|
# legacy
|
194
|
-
@generic_parameters[:grant_method] ||= @generic_parameters.delete(:crtype) if @generic_parameters.key?(:crtype)
|
195
|
+
@generic_parameters[:grant_method] ||= @generic_parameters.delete(:crtype) if @generic_parameters.key?(:crtype) # cspell: disable-line
|
195
196
|
# check that type is known
|
196
197
|
self.class.token_creator(@generic_parameters[:grant_method])
|
197
198
|
# specific parameters for the creation type
|
@@ -272,7 +273,7 @@ module Aspera
|
|
272
273
|
|
273
274
|
# an API was already called, but failed, we need to regenerate or refresh
|
274
275
|
if use_refresh_token
|
275
|
-
if token_data.is_a?(Hash) && token_data.key?('refresh_token')
|
276
|
+
if token_data.is_a?(Hash) && token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
|
276
277
|
# save possible refresh token, before deleting the cache
|
277
278
|
refresh_token = token_data['refresh_token']
|
278
279
|
end
|
@@ -8,7 +8,7 @@ module Aspera
|
|
8
8
|
class PersistencyActionOnce
|
9
9
|
# @param :manager Mandatory Database
|
10
10
|
# @param :data Mandatory object to persist, must be same object from begin to end (assume array by default)
|
11
|
-
# @param :id
|
11
|
+
# @param :id Mandatory identifiers
|
12
12
|
# @param :delete Optional delete persistency condition
|
13
13
|
# @param :parse Optional parse method (default to JSON)
|
14
14
|
# @param :format Optional dump method (default to JSON)
|
@@ -12,6 +12,7 @@ module Aspera
|
|
12
12
|
CONVERSION_TYPES = %i[image office pdf plaintext video].freeze
|
13
13
|
|
14
14
|
# define how files are processed based on mime type
|
15
|
+
# spellchecker:disable
|
15
16
|
SUPPORTED_MIME_TYPES = {
|
16
17
|
'application/json' => :plaintext,
|
17
18
|
'application/mac-binhex40' => :office,
|
@@ -267,6 +268,7 @@ module Aspera
|
|
267
268
|
'ycbcra' => :image,
|
268
269
|
'yuv' => :image,
|
269
270
|
'zabw' => :office}.freeze
|
271
|
+
# spellchecker:enable
|
270
272
|
|
271
273
|
private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
|
272
274
|
|
@@ -301,12 +303,12 @@ module Aspera
|
|
301
303
|
def conversion_type(filepath, mimetype)
|
302
304
|
Log.log.debug{"conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})"}
|
303
305
|
# 1- get type from provided mime type, using local mapping
|
304
|
-
|
306
|
+
conversion_type = SUPPORTED_MIME_TYPES[mimetype] if !mimetype.nil?
|
305
307
|
# 2- else, from computed mime type (if available)
|
306
|
-
if
|
308
|
+
if conversion_type.nil? && @use_mimemagic
|
307
309
|
detected_mime = mime_from_file(filepath)
|
308
310
|
if !detected_mime.nil?
|
309
|
-
|
311
|
+
conversion_type = SUPPORTED_MIME_TYPES[detected_mime]
|
310
312
|
if !mimetype.nil?
|
311
313
|
if mimetype.eql?(detected_mime)
|
312
314
|
Log.log.debug('matching mime type per magic number')
|
@@ -319,9 +321,9 @@ module Aspera
|
|
319
321
|
end
|
320
322
|
# 3- else, from extensions, using local mapping
|
321
323
|
extension = File.extname(filepath.downcase)[1..-1]
|
322
|
-
|
323
|
-
Log.log.debug{"conversion_type(#{extension}): #{
|
324
|
-
return
|
324
|
+
conversion_type = SUPPORTED_EXTENSIONS[extension] if conversion_type.nil?
|
325
|
+
Log.log.debug{"conversion_type(#{extension}): #{conversion_type.class.name} [#{conversion_type}]"}
|
326
|
+
return conversion_type
|
325
327
|
end
|
326
328
|
end
|
327
329
|
end
|