aspera-cli 4.16.0 → 4.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +50 -19
- data/CONTRIBUTING.md +3 -1
- data/README.md +965 -793
- data/bin/asession +29 -21
- data/lib/aspera/{fasp/agent_alpha.rb → agent/alpha.rb} +26 -25
- data/lib/aspera/{fasp/agent_base.rb → agent/base.rb} +15 -12
- data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
- data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +49 -53
- data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +20 -19
- data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +20 -33
- data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb} +11 -11
- data/lib/aspera/api/aoc.rb +586 -0
- data/lib/aspera/api/ats.rb +46 -0
- data/lib/aspera/api/cos_node.rb +95 -0
- data/lib/aspera/api/node.rb +344 -0
- data/lib/aspera/ascmd.rb +46 -10
- data/lib/aspera/{fasp → ascp}/installation.rb +5 -5
- data/lib/aspera/{fasp → ascp}/management.rb +3 -8
- data/lib/aspera/{fasp → ascp}/products.rb +1 -1
- data/lib/aspera/assert.rb +30 -30
- data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
- data/lib/aspera/cli/extended_value.rb +1 -1
- data/lib/aspera/cli/formatter.rb +13 -13
- data/lib/aspera/cli/hints.rb +5 -5
- data/lib/aspera/cli/main.rb +35 -28
- data/lib/aspera/cli/manager.rb +25 -24
- data/lib/aspera/cli/plugin.rb +22 -15
- data/lib/aspera/cli/plugin_factory.rb +61 -0
- data/lib/aspera/cli/plugins/alee.rb +7 -7
- data/lib/aspera/cli/plugins/aoc.rb +83 -77
- data/lib/aspera/cli/plugins/ats.rb +32 -33
- data/lib/aspera/cli/plugins/bss.rb +3 -4
- data/lib/aspera/cli/plugins/config.rb +169 -186
- data/lib/aspera/cli/plugins/console.rb +8 -6
- data/lib/aspera/cli/plugins/cos.rb +19 -18
- data/lib/aspera/cli/plugins/faspex.rb +61 -54
- data/lib/aspera/cli/plugins/faspex5.rb +150 -103
- data/lib/aspera/cli/plugins/node.rb +68 -73
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -44
- data/lib/aspera/cli/plugins/preview.rb +31 -31
- data/lib/aspera/cli/plugins/server.rb +31 -33
- data/lib/aspera/cli/plugins/shares.rb +13 -11
- data/lib/aspera/cli/sync_actions.rb +8 -8
- data/lib/aspera/cli/transfer_agent.rb +32 -19
- data/lib/aspera/cli/transfer_progress.rb +1 -1
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -0
- data/lib/aspera/command_line_builder.rb +14 -14
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +2 -3
- data/lib/aspera/faspex_gw.rb +5 -6
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/id_generator.rb +2 -2
- data/lib/aspera/json_rpc.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +6 -6
- data/lib/aspera/keychain/macos_security.rb +27 -22
- data/lib/aspera/log.rb +2 -2
- data/lib/aspera/nagios.rb +3 -3
- data/lib/aspera/node_simulator.rb +5 -6
- data/lib/aspera/oauth/base.rb +143 -0
- data/lib/aspera/oauth/factory.rb +124 -0
- data/lib/aspera/oauth/generic.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +51 -0
- data/lib/aspera/oauth/url_json.rb +31 -0
- data/lib/aspera/oauth/web.rb +50 -0
- data/lib/aspera/oauth.rb +5 -331
- data/lib/aspera/open_application.rb +7 -7
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +5 -5
- data/lib/aspera/preview/terminal.rb +3 -2
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +4 -4
- data/lib/aspera/rest.rb +175 -144
- data/lib/aspera/rest_errors_aspera.rb +3 -3
- data/lib/aspera/resumer.rb +77 -0
- data/lib/aspera/ssh.rb +6 -1
- data/lib/aspera/{fasp → transfer}/error.rb +3 -3
- data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
- data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
- data/lib/aspera/{fasp → transfer}/parameters.rb +58 -89
- data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +18 -16
- data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
- data/lib/aspera/{fasp → transfer}/sync.rb +32 -32
- data/lib/aspera/{fasp → transfer}/uri.rb +9 -8
- data/lib/aspera/web_server_simple.rb +11 -3
- data.tar.gz.sig +0 -0
- metadata +36 -63
- metadata.gz.sig +0 -0
- data/lib/aspera/aoc.rb +0 -601
- data/lib/aspera/ats_api.rb +0 -47
- data/lib/aspera/cos_node.rb +0 -94
- data/lib/aspera/fasp/resume_policy.rb +0 -79
- data/lib/aspera/node.rb +0 -339
@@ -0,0 +1,95 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
5
|
+
require 'aspera/api/node'
|
6
|
+
require 'xmlsimple'
|
7
|
+
|
8
|
+
module Aspera
|
9
|
+
module Api
|
10
|
+
class CosNode < Node
|
11
|
+
IBM_CLOUD_TOKEN_URL = 'https://iam.cloud.ibm.com/identity'
|
12
|
+
TOKEN_FIELD = 'delegated_refresh_token'
|
13
|
+
class << self
|
14
|
+
def parameters_from_svc_credentials(service_credentials, bucket_region)
|
15
|
+
# check necessary contents
|
16
|
+
Aspera.assert_type(service_credentials, Hash){'service_credentials'}
|
17
|
+
Aspera::Log.dump('service_credentials', service_credentials)
|
18
|
+
%w[apikey resource_instance_id endpoints].each do |field|
|
19
|
+
Aspera.assert(service_credentials.key?(field)){"service_credentials must have a field: #{field}"}
|
20
|
+
end
|
21
|
+
# read endpoints from service provided in service credentials
|
22
|
+
endpoints = Aspera::Rest.new(base_url: service_credentials['endpoints']).read('')[:data]
|
23
|
+
Aspera::Log.dump('endpoints', endpoints)
|
24
|
+
endpoint = endpoints.dig('service-endpoints', 'regional', bucket_region, 'public', bucket_region)
|
25
|
+
raise "no such region: #{bucket_region}" if endpoint.nil?
|
26
|
+
return {
|
27
|
+
instance_id: service_credentials['resource_instance_id'],
|
28
|
+
api_key: service_credentials['apikey'],
|
29
|
+
endpoint: endpoint
|
30
|
+
}
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def initialize(instance_id:, api_key:, endpoint:, bucket:, auth_url: IBM_CLOUD_TOKEN_URL)
|
35
|
+
Aspera.assert_type(instance_id, String){'resource instance id (crn)'}
|
36
|
+
Aspera.assert_type(endpoint, String){'endpoint'}
|
37
|
+
endpoint = "https://#{endpoint}" unless endpoint.start_with?('http')
|
38
|
+
@auth_url = auth_url
|
39
|
+
@api_key = api_key
|
40
|
+
s3_api = Aspera::Rest.new(
|
41
|
+
base_url: endpoint,
|
42
|
+
not_auth_codes: %w[401 403], # error codes when not authorized
|
43
|
+
headers: {'ibm-service-instance-id' => instance_id},
|
44
|
+
auth: {
|
45
|
+
type: :oauth2,
|
46
|
+
grant_method: :generic,
|
47
|
+
base_url: @auth_url,
|
48
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
49
|
+
response_type: 'cloud_iam',
|
50
|
+
apikey: @api_key
|
51
|
+
})
|
52
|
+
# read FASP connection information for bucket
|
53
|
+
xml_result_text = s3_api.call(
|
54
|
+
operation: 'GET',
|
55
|
+
subpath: bucket,
|
56
|
+
headers: {'Accept' => 'application/xml'},
|
57
|
+
url_params: {'faspConnectionInfo' => nil}
|
58
|
+
)[:http].body
|
59
|
+
ats_info = XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
60
|
+
Aspera::Log.dump('ats_info', ats_info)
|
61
|
+
@storage_credentials = {
|
62
|
+
'type' => 'token',
|
63
|
+
'token' => {TOKEN_FIELD => nil}
|
64
|
+
}
|
65
|
+
super(
|
66
|
+
base_url: ats_info['ATSEndpoint'],
|
67
|
+
auth: {
|
68
|
+
type: :basic,
|
69
|
+
username: ats_info['AccessKey']['Id'],
|
70
|
+
password: ats_info['AccessKey']['Secret']},
|
71
|
+
add_tspec: {'tags'=>{Transfer::Spec::TAG_RESERVED=>{'node'=>{'storage_credentials'=>@storage_credentials}}}}
|
72
|
+
)
|
73
|
+
# update storage_credentials AND Rest params
|
74
|
+
generate_token
|
75
|
+
end
|
76
|
+
|
77
|
+
# potentially call this if delegated token is expired
|
78
|
+
def generate_token
|
79
|
+
# OAuth API to get delegated token
|
80
|
+
delegated_oauth = OAuth::Factory.instance.create(
|
81
|
+
base_url: @auth_url,
|
82
|
+
grant_method: :generic,
|
83
|
+
token_field: TOKEN_FIELD,
|
84
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
85
|
+
response_type: 'delegated_refresh_token',
|
86
|
+
apikey: @api_key,
|
87
|
+
receiver_client_ids: 'aspera_ats'
|
88
|
+
)
|
89
|
+
# get delegated token to be placed in rest call header and in transfer tags
|
90
|
+
@storage_credentials['token'][TOKEN_FIELD] = OAuth::Factory.bearer_extract(delegated_oauth.get_authorization)
|
91
|
+
@headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,344 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/cli/error'
|
4
|
+
require 'aspera/transfer/spec'
|
5
|
+
require 'aspera/rest'
|
6
|
+
require 'aspera/oauth'
|
7
|
+
require 'aspera/log'
|
8
|
+
require 'aspera/assert'
|
9
|
+
require 'aspera/environment'
|
10
|
+
require 'zlib'
|
11
|
+
require 'base64'
|
12
|
+
|
13
|
+
module Aspera
|
14
|
+
module Api
|
15
|
+
# Provides additional functions using node API with gen4 extensions (access keys)
|
16
|
+
class Node < Aspera::Rest
|
17
|
+
# permissions
|
18
|
+
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
19
|
+
# prefix for ruby code for filter (deprecated)
|
20
|
+
MATCH_EXEC_PREFIX = 'exec:'
|
21
|
+
MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
|
22
|
+
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
23
|
+
PATH_SEPARATOR = '/'
|
24
|
+
TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
|
25
|
+
SCOPE_USER = 'user:all'
|
26
|
+
SCOPE_ADMIN = 'admin:all'
|
27
|
+
SCOPE_PREFIX = 'node.'
|
28
|
+
SCOPE_SEPARATOR = ':'
|
29
|
+
SIGNATURE_DELIMITER = '==SIGNATURE=='
|
30
|
+
BEARER_TOKEN_VALIDITY_DEFAULT = 86400
|
31
|
+
BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
|
32
|
+
|
33
|
+
# register node special token decoder
|
34
|
+
OAuth::Factory.instance.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
|
35
|
+
|
36
|
+
# class instance variable, access with accessors on class
|
37
|
+
@use_standard_ports = true
|
38
|
+
|
39
|
+
class << self
|
40
|
+
attr_accessor :use_standard_ports
|
41
|
+
|
42
|
+
# For access keys: provide expression to match entry in folder
|
43
|
+
def file_matcher(match_expression)
|
44
|
+
case match_expression
|
45
|
+
when Proc then return match_expression
|
46
|
+
when Regexp then return ->(f){f['name'].match?(match_expression)}
|
47
|
+
when String
|
48
|
+
if match_expression.start_with?(MATCH_EXEC_PREFIX)
|
49
|
+
code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
|
50
|
+
Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
|
51
|
+
return Environment.secure_eval(code, __FILE__, __LINE__)
|
52
|
+
end
|
53
|
+
return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
|
54
|
+
when NilClass then return ->(_){true}
|
55
|
+
else Aspera.error_unexpected_value(match_expression.class.name, exception_class: Cli::BadArgument)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def file_matcher_from_argument(options)
|
60
|
+
return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
|
61
|
+
end
|
62
|
+
|
63
|
+
# node API scopes
|
64
|
+
def token_scope(access_key, scope)
|
65
|
+
return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
|
66
|
+
end
|
67
|
+
|
68
|
+
def decode_scope(scope)
|
69
|
+
items = scope.split(SCOPE_SEPARATOR, 2)
|
70
|
+
Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
|
71
|
+
Aspera.assert(items[0].start_with?(SCOPE_PREFIX)){"invalid scope: #{scope}"}
|
72
|
+
return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
|
73
|
+
end
|
74
|
+
|
75
|
+
# Create an Aspera Node bearer token
|
76
|
+
# @param payload [String] JSON payload to be included in the token
|
77
|
+
# @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
|
78
|
+
def bearer_token(access_key:, payload:, private_key:)
|
79
|
+
Aspera.assert_type(payload, Hash)
|
80
|
+
Aspera.assert(payload.key?('user_id'))
|
81
|
+
Aspera.assert_type(payload['user_id'], String)
|
82
|
+
Aspera.assert(!payload['user_id'].empty?)
|
83
|
+
Aspera.assert_type(private_key, OpenSSL::PKey::RSA)
|
84
|
+
# manage convenience parameters
|
85
|
+
expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
|
86
|
+
payload.delete('_validity')
|
87
|
+
scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
|
88
|
+
payload.delete('_scope')
|
89
|
+
payload['scope'] ||= token_scope(access_key, scope)
|
90
|
+
payload['auth_type'] ||= 'access_key'
|
91
|
+
payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
|
92
|
+
payload_json = JSON.generate(payload)
|
93
|
+
return Base64.strict_encode64(Zlib::Deflate.deflate([
|
94
|
+
payload_json,
|
95
|
+
SIGNATURE_DELIMITER,
|
96
|
+
Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
|
97
|
+
''
|
98
|
+
].join("\n")))
|
99
|
+
end
|
100
|
+
|
101
|
+
def decode_bearer_token(token)
|
102
|
+
return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
|
103
|
+
end
|
104
|
+
|
105
|
+
def bearer_headers(bearer_auth, access_key: nil)
|
106
|
+
# if username is not provided, use the access key from the token
|
107
|
+
if access_key.nil?
|
108
|
+
access_key = Node.decode_scope(Node.decode_bearer_token(OAuth::Factory.bearer_extract(bearer_auth))['scope'])[:access_key]
|
109
|
+
Aspera.assert(!access_key.nil?)
|
110
|
+
end
|
111
|
+
return {
|
112
|
+
Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
|
113
|
+
'Authorization' => bearer_auth
|
114
|
+
}
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
# fields in @app_info
|
119
|
+
REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
|
120
|
+
# methods of @app_info[:api]
|
121
|
+
REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
|
122
|
+
private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
|
123
|
+
|
124
|
+
attr_reader :app_info
|
125
|
+
|
126
|
+
# @param base_url [String] Rest parameters
|
127
|
+
# @param auth [String,NilClass] Rest parameters
|
128
|
+
# @param headers [String,NilClass] Rest parameters
|
129
|
+
# @param app_info [Hash,NilClass] Special processing for AoC
|
130
|
+
# @param add_tspec [Hash,NilClass] Additional transfer spec
|
131
|
+
def initialize(app_info: nil, add_tspec: nil, **rest_args)
|
132
|
+
# init Rest
|
133
|
+
super(**rest_args)
|
134
|
+
@app_info = app_info
|
135
|
+
# this is added to transfer spec, for instance to add tags (COS)
|
136
|
+
@add_tspec = add_tspec
|
137
|
+
if !@app_info.nil?
|
138
|
+
REQUIRED_APP_INFO_FIELDS.each do |field|
|
139
|
+
Aspera.assert(@app_info.key?(field)){"app_info lacks field #{field}"}
|
140
|
+
end
|
141
|
+
REQUIRED_APP_API_METHODS.each do |method|
|
142
|
+
Aspera.assert(@app_info[:api].respond_to?(method)){"#{@app_info[:api].class} lacks method #{method}"}
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
# update transfer spec with special additional tags
|
148
|
+
def add_tspec_info(tspec)
|
149
|
+
tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
|
150
|
+
return tspec
|
151
|
+
end
|
152
|
+
|
153
|
+
# @returns [Node] a Node or nil
|
154
|
+
def node_id_to_node(node_id)
|
155
|
+
if !@app_info.nil?
|
156
|
+
return self if node_id.eql?(@app_info[:node_info]['id'])
|
157
|
+
return @app_info[:api].node_api_from(
|
158
|
+
node_id: node_id,
|
159
|
+
workspace_id: @app_info[:workspace_id],
|
160
|
+
workspace_name: @app_info[:workspace_name])
|
161
|
+
end
|
162
|
+
Log.log.warn{"cannot resolve link with node id #{node_id}"}
|
163
|
+
return nil
|
164
|
+
end
|
165
|
+
|
166
|
+
# Recursively browse in a folder (with non-recursive method)
|
167
|
+
# sub folders are processed if the processing method returns true
|
168
|
+
# @param state [Object] state object sent to processing method
|
169
|
+
# @param top_file_id [String] file id to start at (default = access key root file id)
|
170
|
+
# @param top_file_path [String] path of top folder (default = /)
|
171
|
+
# @param block [Proc] processing method, arguments: entry, path, state
|
172
|
+
def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
|
173
|
+
Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
|
174
|
+
Aspera.assert(block){'Missing block'}
|
175
|
+
# start at top folder
|
176
|
+
folders_to_explore = [{id: top_file_id, path: top_file_path}]
|
177
|
+
Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
|
178
|
+
until folders_to_explore.empty?
|
179
|
+
current_item = folders_to_explore.shift
|
180
|
+
Log.log.debug{"searching #{current_item[:path]}".bg_green}
|
181
|
+
# get folder content
|
182
|
+
folder_contents =
|
183
|
+
begin
|
184
|
+
read("files/#{current_item[:id]}/files")[:data]
|
185
|
+
rescue StandardError => e
|
186
|
+
Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
|
187
|
+
[]
|
188
|
+
end
|
189
|
+
Log.log.debug{Log.dump(:folder_contents, folder_contents)}
|
190
|
+
folder_contents.each do |entry|
|
191
|
+
relative_path = File.join(current_item[:path], entry['name'])
|
192
|
+
Log.log.debug{"process_folder_tree checking #{relative_path}"}
|
193
|
+
# continue only if method returns true
|
194
|
+
next unless yield(entry, relative_path, state)
|
195
|
+
# entry type is file, folder or link
|
196
|
+
case entry['type']
|
197
|
+
when 'folder'
|
198
|
+
folders_to_explore.push({id: entry['id'], path: relative_path})
|
199
|
+
when 'link'
|
200
|
+
node_id_to_node(entry['target_node_id'])&.process_folder_tree(
|
201
|
+
state: state,
|
202
|
+
top_file_id: entry['target_id'],
|
203
|
+
top_file_path: relative_path,
|
204
|
+
&block)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end # process_folder_tree
|
209
|
+
|
210
|
+
# Navigate the path from given file id
|
211
|
+
# @param top_file_id [String] id initial file id
|
212
|
+
# @param path [String] file path
|
213
|
+
# @return [Hash] {.api,.file_id}
|
214
|
+
def resolve_api_fid(top_file_id, path)
|
215
|
+
Aspera.assert_type(top_file_id, String)
|
216
|
+
process_last_link = path.end_with?(PATH_SEPARATOR)
|
217
|
+
path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
|
218
|
+
return {api: self, file_id: top_file_id} if path_elements.empty?
|
219
|
+
resolve_state = {path: path_elements, result: nil}
|
220
|
+
process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
|
221
|
+
# this block is called recursively for each entry in folder
|
222
|
+
# stop digging here if not in right path
|
223
|
+
next false unless entry['name'].eql?(state[:path].first)
|
224
|
+
# ok it matches, so we remove the match
|
225
|
+
state[:path].shift
|
226
|
+
case entry['type']
|
227
|
+
when 'file'
|
228
|
+
# file must be terminal
|
229
|
+
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
230
|
+
# it's terminal, we found it
|
231
|
+
state[:result] = {api: self, file_id: entry['id']}
|
232
|
+
next false
|
233
|
+
when 'folder'
|
234
|
+
if state[:path].empty?
|
235
|
+
# we found it
|
236
|
+
state[:result] = {api: self, file_id: entry['id']}
|
237
|
+
next false
|
238
|
+
end
|
239
|
+
when 'link'
|
240
|
+
if state[:path].empty?
|
241
|
+
if process_last_link
|
242
|
+
# we found it
|
243
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
244
|
+
raise 'cannot resolve link' if other_node.nil?
|
245
|
+
state[:result] = {api: other_node, file_id: entry['target_id']}
|
246
|
+
else
|
247
|
+
# we found it but we do not process the link
|
248
|
+
state[:result] = {api: self, file_id: entry['id']}
|
249
|
+
end
|
250
|
+
next false
|
251
|
+
end
|
252
|
+
else
|
253
|
+
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
254
|
+
end
|
255
|
+
# continue to dig folder
|
256
|
+
next true
|
257
|
+
end
|
258
|
+
raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
|
259
|
+
return resolve_state[:result]
|
260
|
+
end
|
261
|
+
|
262
|
+
def find_files(top_file_id, test_block)
|
263
|
+
Log.log.debug{"find_files: file id=#{top_file_id}"}
|
264
|
+
find_state = {found: [], test_block: test_block}
|
265
|
+
process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
|
266
|
+
state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
|
267
|
+
# test all files deeply
|
268
|
+
true
|
269
|
+
end
|
270
|
+
return find_state[:found]
|
271
|
+
end
|
272
|
+
|
273
|
+
def refreshed_transfer_token
|
274
|
+
return oauth_token(force_refresh: true)
|
275
|
+
end
|
276
|
+
|
277
|
+
# Create transfer spec for gen4
|
278
|
+
def transfer_spec_gen4(file_id, direction, ts_merge=nil)
|
279
|
+
ak_name = nil
|
280
|
+
ak_token = nil
|
281
|
+
case auth_params[:type]
|
282
|
+
when :basic
|
283
|
+
ak_name = auth_params[:username]
|
284
|
+
Aspera.assert(auth_params[:password]){'no secret in node object'}
|
285
|
+
ak_token = Rest.basic_token(auth_params[:username], auth_params[:password])
|
286
|
+
when :oauth2
|
287
|
+
ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
|
288
|
+
# TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
|
289
|
+
# get bearer token, possibly use cache
|
290
|
+
ak_token = oauth_token(force_refresh: false)
|
291
|
+
else Aspera.error_unexpected_value(auth_params[:type])
|
292
|
+
end
|
293
|
+
transfer_spec = {
|
294
|
+
'direction' => direction,
|
295
|
+
'token' => ak_token,
|
296
|
+
'tags' => {
|
297
|
+
Transfer::Spec::TAG_RESERVED => {
|
298
|
+
'node' => {
|
299
|
+
'access_key' => ak_name,
|
300
|
+
'file_id' => file_id
|
301
|
+
} # node
|
302
|
+
} # aspera
|
303
|
+
} # tags
|
304
|
+
}
|
305
|
+
# add specials tags (cos)
|
306
|
+
add_tspec_info(transfer_spec)
|
307
|
+
transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
|
308
|
+
# add application specific tags (AoC)
|
309
|
+
app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
|
310
|
+
# add remote host info
|
311
|
+
if self.class.use_standard_ports
|
312
|
+
# get default TCP/UDP ports and transfer user
|
313
|
+
transfer_spec.merge!(Transfer::Spec::AK_TSPEC_BASE)
|
314
|
+
# by default: same address as node API
|
315
|
+
transfer_spec['remote_host'] = URI.parse(base_url).host
|
316
|
+
# AoC allows specification of other url
|
317
|
+
if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
|
318
|
+
transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
|
319
|
+
end
|
320
|
+
info = read('info')[:data]
|
321
|
+
# get the transfer user from info on access key
|
322
|
+
transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
|
323
|
+
# get settings from name.value array to hash key.value
|
324
|
+
settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
|
325
|
+
# check WSS ports
|
326
|
+
%w[wss_enabled wss_port].each do |i|
|
327
|
+
transfer_spec[i] = settings[i] if settings.key?(i)
|
328
|
+
end if settings.is_a?(Hash)
|
329
|
+
else
|
330
|
+
# retrieve values from API (and keep a copy/cache)
|
331
|
+
@std_t_spec_cache ||= create(
|
332
|
+
'files/download_setup',
|
333
|
+
{transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
|
334
|
+
)[:data]['transfer_specs'].first['transfer_spec']
|
335
|
+
# copy some parts
|
336
|
+
TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
|
337
|
+
end
|
338
|
+
Log.log.warn{"Expected transfer user: #{Transfer::Spec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
|
339
|
+
unless transfer_spec['remote_user'].eql?(Transfer::Spec::ACCESS_KEY_TRANSFER_USER)
|
340
|
+
return transfer_spec
|
341
|
+
end
|
342
|
+
end
|
343
|
+
end
|
344
|
+
end
|
data/lib/aspera/ascmd.rb
CHANGED
@@ -10,8 +10,20 @@ module Aspera
|
|
10
10
|
# execute: "ascmd -h" to get syntax
|
11
11
|
# Note: "ls" can take filters: as_ls -f *.txt -f *.bin /
|
12
12
|
class AsCmd
|
13
|
+
# number of arguments for each operation
|
14
|
+
OPS_ARGS = {
|
15
|
+
cp: 2,
|
16
|
+
df: 0,
|
17
|
+
du: 1,
|
18
|
+
info: nil,
|
19
|
+
ls: 1,
|
20
|
+
md5sum: 1,
|
21
|
+
mkdir: 1,
|
22
|
+
mv: 2,
|
23
|
+
rm: 1
|
24
|
+
}.freeze
|
13
25
|
# list of supported actions
|
14
|
-
OPERATIONS =
|
26
|
+
OPERATIONS = OPS_ARGS.keys.freeze
|
15
27
|
|
16
28
|
# @param command_executor [Object] provides the "execute" method, taking a command to execute, and stdin to feed to it, typically: ssh or local
|
17
29
|
def initialize(command_executor)
|
@@ -22,16 +34,36 @@ module Aspera
|
|
22
34
|
# @param [Symbol] one of OPERATIONS
|
23
35
|
# @param [Array] parameters for "as" command
|
24
36
|
# @return result of command, type depends on command (bool, array, hash)
|
25
|
-
def execute_single(action_sym, arguments
|
37
|
+
def execute_single(action_sym, arguments)
|
38
|
+
arguments = [] if arguments.nil?
|
39
|
+
Log.log.debug{"execute_single:#{action_sym}:#{arguments}"}
|
40
|
+
Aspera.assert_type(action_sym, Symbol)
|
41
|
+
Aspera.assert_type(arguments, Array)
|
42
|
+
Aspera.assert(arguments.all?(String), 'arguments must be strings')
|
43
|
+
# lines of commands (String's)
|
44
|
+
command_lines = []
|
26
45
|
# add "as_" command
|
27
|
-
main_command =
|
28
|
-
|
46
|
+
main_command = "as_#{action_sym}"
|
47
|
+
arg_batches =
|
48
|
+
if OPS_ARGS[action_sym].nil? || OPS_ARGS[action_sym].zero?
|
49
|
+
[arguments]
|
50
|
+
else
|
51
|
+
# split arguments into batches
|
52
|
+
arguments.each_slice(OPS_ARGS[action_sym]).to_a
|
53
|
+
end
|
54
|
+
arg_batches.each do |args|
|
55
|
+
command = [main_command]
|
29
56
|
# enclose arguments in double quotes, protect backslash and double quotes
|
30
|
-
|
57
|
+
args.each do |v|
|
58
|
+
command.push(%Q{"#{v.gsub(/["\\]/){|s|"\\#{s}"}}"})
|
59
|
+
end
|
60
|
+
command_lines.push(command.join(' '))
|
31
61
|
end
|
62
|
+
command_lines.push('as_exit')
|
63
|
+
command_lines.push('')
|
32
64
|
# execute the main command and then exit
|
33
|
-
stdin_input =
|
34
|
-
Log.log.
|
65
|
+
stdin_input = command_lines.join("\n")
|
66
|
+
Log.log.trace1{"execute_single:#{stdin_input}"}
|
35
67
|
# execute, get binary output
|
36
68
|
byte_buffer = @command_executor.execute('ascmd', stdin_input).unpack('C*')
|
37
69
|
raise 'ERROR: empty answer from server' if byte_buffer.empty?
|
@@ -53,7 +85,11 @@ module Aspera
|
|
53
85
|
end
|
54
86
|
end
|
55
87
|
# for info, second overrides first, so restore it
|
56
|
-
case result.keys.length
|
88
|
+
case result.keys.length
|
89
|
+
when 0 then result = system_info
|
90
|
+
when 1 then result = result[result.keys.first]
|
91
|
+
else Aspera.error_unexpected_value(result.keys.length)
|
92
|
+
end
|
57
93
|
# raise error as exception
|
58
94
|
raise Error.new(result[:errno], result[:errstr], action_sym, arguments) if
|
59
95
|
result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{|i|i[:name]}.sort)
|
@@ -66,7 +102,7 @@ module Aspera
|
|
66
102
|
super(); @errno = errno; @errstr = errstr; @command = cmd; @arguments = arguments; end # rubocop:disable Style/Semicolon
|
67
103
|
|
68
104
|
def message; "ascmd: #{@errstr} (#{@errno})"; end
|
69
|
-
def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments
|
105
|
+
def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments&.join(',')}"; end
|
70
106
|
end # Error
|
71
107
|
|
72
108
|
# description of result structures (see ascmdtypes.h). Base types are big endian
|
@@ -174,7 +210,7 @@ module Aspera
|
|
174
210
|
end
|
175
211
|
end
|
176
212
|
end
|
177
|
-
else error_unexpected_value(type_descr[:decode])
|
213
|
+
else Aspera.error_unexpected_value(type_descr[:decode])
|
178
214
|
end # is_a
|
179
215
|
return result
|
180
216
|
end
|
@@ -3,7 +3,7 @@
|
|
3
3
|
# cspell:ignore protobuf ckpt
|
4
4
|
require 'aspera/environment'
|
5
5
|
require 'aspera/data_repository'
|
6
|
-
require 'aspera/
|
6
|
+
require 'aspera/ascp/products'
|
7
7
|
require 'aspera/log'
|
8
8
|
require 'aspera/assert'
|
9
9
|
require 'aspera/web_server_simple'
|
@@ -16,7 +16,7 @@ require 'fileutils'
|
|
16
16
|
require 'openssl'
|
17
17
|
|
18
18
|
module Aspera
|
19
|
-
module
|
19
|
+
module Ascp
|
20
20
|
# Singleton that tells where to find ascp and other local resources (keys..) , using the "path(:name)" method.
|
21
21
|
# It is used by object : AgentDirect to find necessary resources
|
22
22
|
# By default it takes the first Aspera product found
|
@@ -139,10 +139,10 @@ module Aspera
|
|
139
139
|
check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
|
140
140
|
end
|
141
141
|
file = k.eql?(:fallback_certificate) ? file_cert : file_key
|
142
|
-
else error_unexpected_value(k)
|
142
|
+
else Aspera.error_unexpected_value(k)
|
143
143
|
end
|
144
144
|
return nil if file_is_optional && !File.exist?(file)
|
145
|
-
assert(File.exist?(file)){"no such file: #{file}"}
|
145
|
+
Aspera.assert(File.exist?(file)){"no such file: #{file}"}
|
146
146
|
return file
|
147
147
|
end
|
148
148
|
|
@@ -283,6 +283,6 @@ module Aspera
|
|
283
283
|
def transferd_filepath
|
284
284
|
return File.join(sdk_folder, 'asperatransferd' + Environment.exe_extension) # cspell:disable-line
|
285
285
|
end
|
286
|
-
end
|
286
|
+
end
|
287
287
|
end
|
288
288
|
end
|
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Aspera
|
4
|
-
module
|
5
|
-
#
|
4
|
+
module Ascp
|
5
|
+
# processing of ascp management port events
|
6
6
|
class Management
|
7
7
|
# cspell: disable
|
8
8
|
OPERATIONS = %w[
|
@@ -202,12 +202,7 @@ module Aspera
|
|
202
202
|
# translates mgt port event into (enhanced) typed event
|
203
203
|
def enhanced_event_format(event)
|
204
204
|
return event.keys.each_with_object({}) do |e, h|
|
205
|
-
|
206
|
-
new_name = e
|
207
|
-
.gsub(/([a-z\d])([A-Z])/, '\1_\2')
|
208
|
-
.gsub(/([a-z\d])(usec)$/, '\1_\2')
|
209
|
-
.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
|
210
|
-
.downcase
|
205
|
+
new_name = e.capital_to_snake.gsub(/(usec)$/, '_\1').downcase
|
211
206
|
value = event[e]
|
212
207
|
value = value.to_i if INTEGER_FIELDS.include?(e)
|
213
208
|
value = value.eql?(BOOLEAN_TRUE) if BOOLEAN_FIELDS.include?(e)
|