aspera-cli 4.16.0 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +50 -19
- data/CONTRIBUTING.md +3 -1
- data/README.md +965 -793
- data/bin/asession +29 -21
- data/lib/aspera/{fasp/agent_alpha.rb → agent/alpha.rb} +26 -25
- data/lib/aspera/{fasp/agent_base.rb → agent/base.rb} +15 -12
- data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
- data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +49 -53
- data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +20 -19
- data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +20 -33
- data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb} +11 -11
- data/lib/aspera/api/aoc.rb +586 -0
- data/lib/aspera/api/ats.rb +46 -0
- data/lib/aspera/api/cos_node.rb +95 -0
- data/lib/aspera/api/node.rb +344 -0
- data/lib/aspera/ascmd.rb +46 -10
- data/lib/aspera/{fasp → ascp}/installation.rb +5 -5
- data/lib/aspera/{fasp → ascp}/management.rb +3 -8
- data/lib/aspera/{fasp → ascp}/products.rb +1 -1
- data/lib/aspera/assert.rb +30 -30
- data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
- data/lib/aspera/cli/extended_value.rb +1 -1
- data/lib/aspera/cli/formatter.rb +13 -13
- data/lib/aspera/cli/hints.rb +5 -5
- data/lib/aspera/cli/main.rb +35 -28
- data/lib/aspera/cli/manager.rb +25 -24
- data/lib/aspera/cli/plugin.rb +22 -15
- data/lib/aspera/cli/plugin_factory.rb +61 -0
- data/lib/aspera/cli/plugins/alee.rb +7 -7
- data/lib/aspera/cli/plugins/aoc.rb +83 -77
- data/lib/aspera/cli/plugins/ats.rb +32 -33
- data/lib/aspera/cli/plugins/bss.rb +3 -4
- data/lib/aspera/cli/plugins/config.rb +169 -186
- data/lib/aspera/cli/plugins/console.rb +8 -6
- data/lib/aspera/cli/plugins/cos.rb +19 -18
- data/lib/aspera/cli/plugins/faspex.rb +61 -54
- data/lib/aspera/cli/plugins/faspex5.rb +150 -103
- data/lib/aspera/cli/plugins/node.rb +68 -73
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -44
- data/lib/aspera/cli/plugins/preview.rb +31 -31
- data/lib/aspera/cli/plugins/server.rb +31 -33
- data/lib/aspera/cli/plugins/shares.rb +13 -11
- data/lib/aspera/cli/sync_actions.rb +8 -8
- data/lib/aspera/cli/transfer_agent.rb +32 -19
- data/lib/aspera/cli/transfer_progress.rb +1 -1
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -0
- data/lib/aspera/command_line_builder.rb +14 -14
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +2 -3
- data/lib/aspera/faspex_gw.rb +5 -6
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/id_generator.rb +2 -2
- data/lib/aspera/json_rpc.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +6 -6
- data/lib/aspera/keychain/macos_security.rb +27 -22
- data/lib/aspera/log.rb +2 -2
- data/lib/aspera/nagios.rb +3 -3
- data/lib/aspera/node_simulator.rb +5 -6
- data/lib/aspera/oauth/base.rb +143 -0
- data/lib/aspera/oauth/factory.rb +124 -0
- data/lib/aspera/oauth/generic.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +51 -0
- data/lib/aspera/oauth/url_json.rb +31 -0
- data/lib/aspera/oauth/web.rb +50 -0
- data/lib/aspera/oauth.rb +5 -331
- data/lib/aspera/open_application.rb +7 -7
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +5 -5
- data/lib/aspera/preview/terminal.rb +3 -2
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +4 -4
- data/lib/aspera/rest.rb +175 -144
- data/lib/aspera/rest_errors_aspera.rb +3 -3
- data/lib/aspera/resumer.rb +77 -0
- data/lib/aspera/ssh.rb +6 -1
- data/lib/aspera/{fasp → transfer}/error.rb +3 -3
- data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
- data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
- data/lib/aspera/{fasp → transfer}/parameters.rb +58 -89
- data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +18 -16
- data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
- data/lib/aspera/{fasp → transfer}/sync.rb +32 -32
- data/lib/aspera/{fasp → transfer}/uri.rb +9 -8
- data/lib/aspera/web_server_simple.rb +11 -3
- data.tar.gz.sig +0 -0
- metadata +36 -63
- metadata.gz.sig +0 -0
- data/lib/aspera/aoc.rb +0 -601
- data/lib/aspera/ats_api.rb +0 -47
- data/lib/aspera/cos_node.rb +0 -94
- data/lib/aspera/fasp/resume_policy.rb +0 -79
- data/lib/aspera/node.rb +0 -339
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aspera/log'
|
|
4
|
+
require 'aspera/assert'
|
|
5
|
+
require 'aspera/api/node'
|
|
6
|
+
require 'xmlsimple'
|
|
7
|
+
|
|
8
|
+
module Aspera
|
|
9
|
+
module Api
|
|
10
|
+
class CosNode < Node
|
|
11
|
+
IBM_CLOUD_TOKEN_URL = 'https://iam.cloud.ibm.com/identity'
|
|
12
|
+
TOKEN_FIELD = 'delegated_refresh_token'
|
|
13
|
+
class << self
|
|
14
|
+
def parameters_from_svc_credentials(service_credentials, bucket_region)
|
|
15
|
+
# check necessary contents
|
|
16
|
+
Aspera.assert_type(service_credentials, Hash){'service_credentials'}
|
|
17
|
+
Aspera::Log.dump('service_credentials', service_credentials)
|
|
18
|
+
%w[apikey resource_instance_id endpoints].each do |field|
|
|
19
|
+
Aspera.assert(service_credentials.key?(field)){"service_credentials must have a field: #{field}"}
|
|
20
|
+
end
|
|
21
|
+
# read endpoints from service provided in service credentials
|
|
22
|
+
endpoints = Aspera::Rest.new(base_url: service_credentials['endpoints']).read('')[:data]
|
|
23
|
+
Aspera::Log.dump('endpoints', endpoints)
|
|
24
|
+
endpoint = endpoints.dig('service-endpoints', 'regional', bucket_region, 'public', bucket_region)
|
|
25
|
+
raise "no such region: #{bucket_region}" if endpoint.nil?
|
|
26
|
+
return {
|
|
27
|
+
instance_id: service_credentials['resource_instance_id'],
|
|
28
|
+
api_key: service_credentials['apikey'],
|
|
29
|
+
endpoint: endpoint
|
|
30
|
+
}
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def initialize(instance_id:, api_key:, endpoint:, bucket:, auth_url: IBM_CLOUD_TOKEN_URL)
|
|
35
|
+
Aspera.assert_type(instance_id, String){'resource instance id (crn)'}
|
|
36
|
+
Aspera.assert_type(endpoint, String){'endpoint'}
|
|
37
|
+
endpoint = "https://#{endpoint}" unless endpoint.start_with?('http')
|
|
38
|
+
@auth_url = auth_url
|
|
39
|
+
@api_key = api_key
|
|
40
|
+
s3_api = Aspera::Rest.new(
|
|
41
|
+
base_url: endpoint,
|
|
42
|
+
not_auth_codes: %w[401 403], # error codes when not authorized
|
|
43
|
+
headers: {'ibm-service-instance-id' => instance_id},
|
|
44
|
+
auth: {
|
|
45
|
+
type: :oauth2,
|
|
46
|
+
grant_method: :generic,
|
|
47
|
+
base_url: @auth_url,
|
|
48
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
|
49
|
+
response_type: 'cloud_iam',
|
|
50
|
+
apikey: @api_key
|
|
51
|
+
})
|
|
52
|
+
# read FASP connection information for bucket
|
|
53
|
+
xml_result_text = s3_api.call(
|
|
54
|
+
operation: 'GET',
|
|
55
|
+
subpath: bucket,
|
|
56
|
+
headers: {'Accept' => 'application/xml'},
|
|
57
|
+
url_params: {'faspConnectionInfo' => nil}
|
|
58
|
+
)[:http].body
|
|
59
|
+
ats_info = XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
|
60
|
+
Aspera::Log.dump('ats_info', ats_info)
|
|
61
|
+
@storage_credentials = {
|
|
62
|
+
'type' => 'token',
|
|
63
|
+
'token' => {TOKEN_FIELD => nil}
|
|
64
|
+
}
|
|
65
|
+
super(
|
|
66
|
+
base_url: ats_info['ATSEndpoint'],
|
|
67
|
+
auth: {
|
|
68
|
+
type: :basic,
|
|
69
|
+
username: ats_info['AccessKey']['Id'],
|
|
70
|
+
password: ats_info['AccessKey']['Secret']},
|
|
71
|
+
add_tspec: {'tags'=>{Transfer::Spec::TAG_RESERVED=>{'node'=>{'storage_credentials'=>@storage_credentials}}}}
|
|
72
|
+
)
|
|
73
|
+
# update storage_credentials AND Rest params
|
|
74
|
+
generate_token
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# potentially call this if delegated token is expired
|
|
78
|
+
def generate_token
|
|
79
|
+
# OAuth API to get delegated token
|
|
80
|
+
delegated_oauth = OAuth::Factory.instance.create(
|
|
81
|
+
base_url: @auth_url,
|
|
82
|
+
grant_method: :generic,
|
|
83
|
+
token_field: TOKEN_FIELD,
|
|
84
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
|
85
|
+
response_type: 'delegated_refresh_token',
|
|
86
|
+
apikey: @api_key,
|
|
87
|
+
receiver_client_ids: 'aspera_ats'
|
|
88
|
+
)
|
|
89
|
+
# get delegated token to be placed in rest call header and in transfer tags
|
|
90
|
+
@storage_credentials['token'][TOKEN_FIELD] = OAuth::Factory.bearer_extract(delegated_oauth.get_authorization)
|
|
91
|
+
@headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aspera/cli/error'
|
|
4
|
+
require 'aspera/transfer/spec'
|
|
5
|
+
require 'aspera/rest'
|
|
6
|
+
require 'aspera/oauth'
|
|
7
|
+
require 'aspera/log'
|
|
8
|
+
require 'aspera/assert'
|
|
9
|
+
require 'aspera/environment'
|
|
10
|
+
require 'zlib'
|
|
11
|
+
require 'base64'
|
|
12
|
+
|
|
13
|
+
module Aspera
|
|
14
|
+
module Api
|
|
15
|
+
# Provides additional functions using node API with gen4 extensions (access keys)
|
|
16
|
+
class Node < Aspera::Rest
|
|
17
|
+
# permissions
|
|
18
|
+
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
|
19
|
+
# prefix for ruby code for filter (deprecated)
|
|
20
|
+
MATCH_EXEC_PREFIX = 'exec:'
|
|
21
|
+
MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
|
|
22
|
+
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
|
23
|
+
PATH_SEPARATOR = '/'
|
|
24
|
+
TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
|
|
25
|
+
SCOPE_USER = 'user:all'
|
|
26
|
+
SCOPE_ADMIN = 'admin:all'
|
|
27
|
+
SCOPE_PREFIX = 'node.'
|
|
28
|
+
SCOPE_SEPARATOR = ':'
|
|
29
|
+
SIGNATURE_DELIMITER = '==SIGNATURE=='
|
|
30
|
+
BEARER_TOKEN_VALIDITY_DEFAULT = 86400
|
|
31
|
+
BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
|
|
32
|
+
|
|
33
|
+
# register node special token decoder
|
|
34
|
+
OAuth::Factory.instance.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
|
|
35
|
+
|
|
36
|
+
# class instance variable, access with accessors on class
|
|
37
|
+
@use_standard_ports = true
|
|
38
|
+
|
|
39
|
+
class << self
|
|
40
|
+
attr_accessor :use_standard_ports
|
|
41
|
+
|
|
42
|
+
# For access keys: provide expression to match entry in folder
|
|
43
|
+
def file_matcher(match_expression)
|
|
44
|
+
case match_expression
|
|
45
|
+
when Proc then return match_expression
|
|
46
|
+
when Regexp then return ->(f){f['name'].match?(match_expression)}
|
|
47
|
+
when String
|
|
48
|
+
if match_expression.start_with?(MATCH_EXEC_PREFIX)
|
|
49
|
+
code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
|
|
50
|
+
Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
|
|
51
|
+
return Environment.secure_eval(code, __FILE__, __LINE__)
|
|
52
|
+
end
|
|
53
|
+
return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
|
|
54
|
+
when NilClass then return ->(_){true}
|
|
55
|
+
else Aspera.error_unexpected_value(match_expression.class.name, exception_class: Cli::BadArgument)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def file_matcher_from_argument(options)
|
|
60
|
+
return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# node API scopes
|
|
64
|
+
def token_scope(access_key, scope)
|
|
65
|
+
return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def decode_scope(scope)
|
|
69
|
+
items = scope.split(SCOPE_SEPARATOR, 2)
|
|
70
|
+
Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
|
|
71
|
+
Aspera.assert(items[0].start_with?(SCOPE_PREFIX)){"invalid scope: #{scope}"}
|
|
72
|
+
return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Create an Aspera Node bearer token
|
|
76
|
+
# @param payload [String] JSON payload to be included in the token
|
|
77
|
+
# @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
|
|
78
|
+
def bearer_token(access_key:, payload:, private_key:)
|
|
79
|
+
Aspera.assert_type(payload, Hash)
|
|
80
|
+
Aspera.assert(payload.key?('user_id'))
|
|
81
|
+
Aspera.assert_type(payload['user_id'], String)
|
|
82
|
+
Aspera.assert(!payload['user_id'].empty?)
|
|
83
|
+
Aspera.assert_type(private_key, OpenSSL::PKey::RSA)
|
|
84
|
+
# manage convenience parameters
|
|
85
|
+
expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
|
|
86
|
+
payload.delete('_validity')
|
|
87
|
+
scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
|
|
88
|
+
payload.delete('_scope')
|
|
89
|
+
payload['scope'] ||= token_scope(access_key, scope)
|
|
90
|
+
payload['auth_type'] ||= 'access_key'
|
|
91
|
+
payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
|
|
92
|
+
payload_json = JSON.generate(payload)
|
|
93
|
+
return Base64.strict_encode64(Zlib::Deflate.deflate([
|
|
94
|
+
payload_json,
|
|
95
|
+
SIGNATURE_DELIMITER,
|
|
96
|
+
Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
|
|
97
|
+
''
|
|
98
|
+
].join("\n")))
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def decode_bearer_token(token)
|
|
102
|
+
return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def bearer_headers(bearer_auth, access_key: nil)
|
|
106
|
+
# if username is not provided, use the access key from the token
|
|
107
|
+
if access_key.nil?
|
|
108
|
+
access_key = Node.decode_scope(Node.decode_bearer_token(OAuth::Factory.bearer_extract(bearer_auth))['scope'])[:access_key]
|
|
109
|
+
Aspera.assert(!access_key.nil?)
|
|
110
|
+
end
|
|
111
|
+
return {
|
|
112
|
+
Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
|
|
113
|
+
'Authorization' => bearer_auth
|
|
114
|
+
}
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# fields in @app_info
|
|
119
|
+
REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
|
|
120
|
+
# methods of @app_info[:api]
|
|
121
|
+
REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
|
|
122
|
+
private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
|
|
123
|
+
|
|
124
|
+
attr_reader :app_info
|
|
125
|
+
|
|
126
|
+
# @param base_url [String] Rest parameters
|
|
127
|
+
# @param auth [String,NilClass] Rest parameters
|
|
128
|
+
# @param headers [String,NilClass] Rest parameters
|
|
129
|
+
# @param app_info [Hash,NilClass] Special processing for AoC
|
|
130
|
+
# @param add_tspec [Hash,NilClass] Additional transfer spec
|
|
131
|
+
def initialize(app_info: nil, add_tspec: nil, **rest_args)
|
|
132
|
+
# init Rest
|
|
133
|
+
super(**rest_args)
|
|
134
|
+
@app_info = app_info
|
|
135
|
+
# this is added to transfer spec, for instance to add tags (COS)
|
|
136
|
+
@add_tspec = add_tspec
|
|
137
|
+
if !@app_info.nil?
|
|
138
|
+
REQUIRED_APP_INFO_FIELDS.each do |field|
|
|
139
|
+
Aspera.assert(@app_info.key?(field)){"app_info lacks field #{field}"}
|
|
140
|
+
end
|
|
141
|
+
REQUIRED_APP_API_METHODS.each do |method|
|
|
142
|
+
Aspera.assert(@app_info[:api].respond_to?(method)){"#{@app_info[:api].class} lacks method #{method}"}
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# update transfer spec with special additional tags
|
|
148
|
+
def add_tspec_info(tspec)
|
|
149
|
+
tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
|
|
150
|
+
return tspec
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# @returns [Node] a Node or nil
|
|
154
|
+
def node_id_to_node(node_id)
|
|
155
|
+
if !@app_info.nil?
|
|
156
|
+
return self if node_id.eql?(@app_info[:node_info]['id'])
|
|
157
|
+
return @app_info[:api].node_api_from(
|
|
158
|
+
node_id: node_id,
|
|
159
|
+
workspace_id: @app_info[:workspace_id],
|
|
160
|
+
workspace_name: @app_info[:workspace_name])
|
|
161
|
+
end
|
|
162
|
+
Log.log.warn{"cannot resolve link with node id #{node_id}"}
|
|
163
|
+
return nil
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Recursively browse in a folder (with non-recursive method)
|
|
167
|
+
# sub folders are processed if the processing method returns true
|
|
168
|
+
# @param state [Object] state object sent to processing method
|
|
169
|
+
# @param top_file_id [String] file id to start at (default = access key root file id)
|
|
170
|
+
# @param top_file_path [String] path of top folder (default = /)
|
|
171
|
+
# @param block [Proc] processing method, arguments: entry, path, state
|
|
172
|
+
def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
|
|
173
|
+
Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
|
|
174
|
+
Aspera.assert(block){'Missing block'}
|
|
175
|
+
# start at top folder
|
|
176
|
+
folders_to_explore = [{id: top_file_id, path: top_file_path}]
|
|
177
|
+
Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
|
|
178
|
+
until folders_to_explore.empty?
|
|
179
|
+
current_item = folders_to_explore.shift
|
|
180
|
+
Log.log.debug{"searching #{current_item[:path]}".bg_green}
|
|
181
|
+
# get folder content
|
|
182
|
+
folder_contents =
|
|
183
|
+
begin
|
|
184
|
+
read("files/#{current_item[:id]}/files")[:data]
|
|
185
|
+
rescue StandardError => e
|
|
186
|
+
Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
|
|
187
|
+
[]
|
|
188
|
+
end
|
|
189
|
+
Log.log.debug{Log.dump(:folder_contents, folder_contents)}
|
|
190
|
+
folder_contents.each do |entry|
|
|
191
|
+
relative_path = File.join(current_item[:path], entry['name'])
|
|
192
|
+
Log.log.debug{"process_folder_tree checking #{relative_path}"}
|
|
193
|
+
# continue only if method returns true
|
|
194
|
+
next unless yield(entry, relative_path, state)
|
|
195
|
+
# entry type is file, folder or link
|
|
196
|
+
case entry['type']
|
|
197
|
+
when 'folder'
|
|
198
|
+
folders_to_explore.push({id: entry['id'], path: relative_path})
|
|
199
|
+
when 'link'
|
|
200
|
+
node_id_to_node(entry['target_node_id'])&.process_folder_tree(
|
|
201
|
+
state: state,
|
|
202
|
+
top_file_id: entry['target_id'],
|
|
203
|
+
top_file_path: relative_path,
|
|
204
|
+
&block)
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
end # process_folder_tree
|
|
209
|
+
|
|
210
|
+
# Navigate the path from given file id
|
|
211
|
+
# @param top_file_id [String] id initial file id
|
|
212
|
+
# @param path [String] file path
|
|
213
|
+
# @return [Hash] {.api,.file_id}
|
|
214
|
+
def resolve_api_fid(top_file_id, path)
|
|
215
|
+
Aspera.assert_type(top_file_id, String)
|
|
216
|
+
process_last_link = path.end_with?(PATH_SEPARATOR)
|
|
217
|
+
path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
|
|
218
|
+
return {api: self, file_id: top_file_id} if path_elements.empty?
|
|
219
|
+
resolve_state = {path: path_elements, result: nil}
|
|
220
|
+
process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
|
|
221
|
+
# this block is called recursively for each entry in folder
|
|
222
|
+
# stop digging here if not in right path
|
|
223
|
+
next false unless entry['name'].eql?(state[:path].first)
|
|
224
|
+
# ok it matches, so we remove the match
|
|
225
|
+
state[:path].shift
|
|
226
|
+
case entry['type']
|
|
227
|
+
when 'file'
|
|
228
|
+
# file must be terminal
|
|
229
|
+
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
|
|
230
|
+
# it's terminal, we found it
|
|
231
|
+
state[:result] = {api: self, file_id: entry['id']}
|
|
232
|
+
next false
|
|
233
|
+
when 'folder'
|
|
234
|
+
if state[:path].empty?
|
|
235
|
+
# we found it
|
|
236
|
+
state[:result] = {api: self, file_id: entry['id']}
|
|
237
|
+
next false
|
|
238
|
+
end
|
|
239
|
+
when 'link'
|
|
240
|
+
if state[:path].empty?
|
|
241
|
+
if process_last_link
|
|
242
|
+
# we found it
|
|
243
|
+
other_node = node_id_to_node(entry['target_node_id'])
|
|
244
|
+
raise 'cannot resolve link' if other_node.nil?
|
|
245
|
+
state[:result] = {api: other_node, file_id: entry['target_id']}
|
|
246
|
+
else
|
|
247
|
+
# we found it but we do not process the link
|
|
248
|
+
state[:result] = {api: self, file_id: entry['id']}
|
|
249
|
+
end
|
|
250
|
+
next false
|
|
251
|
+
end
|
|
252
|
+
else
|
|
253
|
+
Log.log.warn{"Unknown element type: #{entry['type']}"}
|
|
254
|
+
end
|
|
255
|
+
# continue to dig folder
|
|
256
|
+
next true
|
|
257
|
+
end
|
|
258
|
+
raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
|
|
259
|
+
return resolve_state[:result]
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def find_files(top_file_id, test_block)
|
|
263
|
+
Log.log.debug{"find_files: file id=#{top_file_id}"}
|
|
264
|
+
find_state = {found: [], test_block: test_block}
|
|
265
|
+
process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
|
|
266
|
+
state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
|
|
267
|
+
# test all files deeply
|
|
268
|
+
true
|
|
269
|
+
end
|
|
270
|
+
return find_state[:found]
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def refreshed_transfer_token
|
|
274
|
+
return oauth_token(force_refresh: true)
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# Create transfer spec for gen4
|
|
278
|
+
def transfer_spec_gen4(file_id, direction, ts_merge=nil)
|
|
279
|
+
ak_name = nil
|
|
280
|
+
ak_token = nil
|
|
281
|
+
case auth_params[:type]
|
|
282
|
+
when :basic
|
|
283
|
+
ak_name = auth_params[:username]
|
|
284
|
+
Aspera.assert(auth_params[:password]){'no secret in node object'}
|
|
285
|
+
ak_token = Rest.basic_token(auth_params[:username], auth_params[:password])
|
|
286
|
+
when :oauth2
|
|
287
|
+
ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
|
|
288
|
+
# TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
|
|
289
|
+
# get bearer token, possibly use cache
|
|
290
|
+
ak_token = oauth_token(force_refresh: false)
|
|
291
|
+
else Aspera.error_unexpected_value(auth_params[:type])
|
|
292
|
+
end
|
|
293
|
+
transfer_spec = {
|
|
294
|
+
'direction' => direction,
|
|
295
|
+
'token' => ak_token,
|
|
296
|
+
'tags' => {
|
|
297
|
+
Transfer::Spec::TAG_RESERVED => {
|
|
298
|
+
'node' => {
|
|
299
|
+
'access_key' => ak_name,
|
|
300
|
+
'file_id' => file_id
|
|
301
|
+
} # node
|
|
302
|
+
} # aspera
|
|
303
|
+
} # tags
|
|
304
|
+
}
|
|
305
|
+
# add specials tags (cos)
|
|
306
|
+
add_tspec_info(transfer_spec)
|
|
307
|
+
transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
|
|
308
|
+
# add application specific tags (AoC)
|
|
309
|
+
app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
|
|
310
|
+
# add remote host info
|
|
311
|
+
if self.class.use_standard_ports
|
|
312
|
+
# get default TCP/UDP ports and transfer user
|
|
313
|
+
transfer_spec.merge!(Transfer::Spec::AK_TSPEC_BASE)
|
|
314
|
+
# by default: same address as node API
|
|
315
|
+
transfer_spec['remote_host'] = URI.parse(base_url).host
|
|
316
|
+
# AoC allows specification of other url
|
|
317
|
+
if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
|
|
318
|
+
transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
|
|
319
|
+
end
|
|
320
|
+
info = read('info')[:data]
|
|
321
|
+
# get the transfer user from info on access key
|
|
322
|
+
transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
|
|
323
|
+
# get settings from name.value array to hash key.value
|
|
324
|
+
settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
|
|
325
|
+
# check WSS ports
|
|
326
|
+
%w[wss_enabled wss_port].each do |i|
|
|
327
|
+
transfer_spec[i] = settings[i] if settings.key?(i)
|
|
328
|
+
end if settings.is_a?(Hash)
|
|
329
|
+
else
|
|
330
|
+
# retrieve values from API (and keep a copy/cache)
|
|
331
|
+
@std_t_spec_cache ||= create(
|
|
332
|
+
'files/download_setup',
|
|
333
|
+
{transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
|
|
334
|
+
)[:data]['transfer_specs'].first['transfer_spec']
|
|
335
|
+
# copy some parts
|
|
336
|
+
TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
|
|
337
|
+
end
|
|
338
|
+
Log.log.warn{"Expected transfer user: #{Transfer::Spec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
|
|
339
|
+
unless transfer_spec['remote_user'].eql?(Transfer::Spec::ACCESS_KEY_TRANSFER_USER)
|
|
340
|
+
return transfer_spec
|
|
341
|
+
end
|
|
342
|
+
end
|
|
343
|
+
end
|
|
344
|
+
end
|
data/lib/aspera/ascmd.rb
CHANGED
|
@@ -10,8 +10,20 @@ module Aspera
|
|
|
10
10
|
# execute: "ascmd -h" to get syntax
|
|
11
11
|
# Note: "ls" can take filters: as_ls -f *.txt -f *.bin /
|
|
12
12
|
class AsCmd
|
|
13
|
+
# number of arguments for each operation
|
|
14
|
+
OPS_ARGS = {
|
|
15
|
+
cp: 2,
|
|
16
|
+
df: 0,
|
|
17
|
+
du: 1,
|
|
18
|
+
info: nil,
|
|
19
|
+
ls: 1,
|
|
20
|
+
md5sum: 1,
|
|
21
|
+
mkdir: 1,
|
|
22
|
+
mv: 2,
|
|
23
|
+
rm: 1
|
|
24
|
+
}.freeze
|
|
13
25
|
# list of supported actions
|
|
14
|
-
OPERATIONS =
|
|
26
|
+
OPERATIONS = OPS_ARGS.keys.freeze
|
|
15
27
|
|
|
16
28
|
# @param command_executor [Object] provides the "execute" method, taking a command to execute, and stdin to feed to it, typically: ssh or local
|
|
17
29
|
def initialize(command_executor)
|
|
@@ -22,16 +34,36 @@ module Aspera
|
|
|
22
34
|
# @param [Symbol] one of OPERATIONS
|
|
23
35
|
# @param [Array] parameters for "as" command
|
|
24
36
|
# @return result of command, type depends on command (bool, array, hash)
|
|
25
|
-
def execute_single(action_sym, arguments
|
|
37
|
+
def execute_single(action_sym, arguments)
|
|
38
|
+
arguments = [] if arguments.nil?
|
|
39
|
+
Log.log.debug{"execute_single:#{action_sym}:#{arguments}"}
|
|
40
|
+
Aspera.assert_type(action_sym, Symbol)
|
|
41
|
+
Aspera.assert_type(arguments, Array)
|
|
42
|
+
Aspera.assert(arguments.all?(String), 'arguments must be strings')
|
|
43
|
+
# lines of commands (String's)
|
|
44
|
+
command_lines = []
|
|
26
45
|
# add "as_" command
|
|
27
|
-
main_command =
|
|
28
|
-
|
|
46
|
+
main_command = "as_#{action_sym}"
|
|
47
|
+
arg_batches =
|
|
48
|
+
if OPS_ARGS[action_sym].nil? || OPS_ARGS[action_sym].zero?
|
|
49
|
+
[arguments]
|
|
50
|
+
else
|
|
51
|
+
# split arguments into batches
|
|
52
|
+
arguments.each_slice(OPS_ARGS[action_sym]).to_a
|
|
53
|
+
end
|
|
54
|
+
arg_batches.each do |args|
|
|
55
|
+
command = [main_command]
|
|
29
56
|
# enclose arguments in double quotes, protect backslash and double quotes
|
|
30
|
-
|
|
57
|
+
args.each do |v|
|
|
58
|
+
command.push(%Q{"#{v.gsub(/["\\]/){|s|"\\#{s}"}}"})
|
|
59
|
+
end
|
|
60
|
+
command_lines.push(command.join(' '))
|
|
31
61
|
end
|
|
62
|
+
command_lines.push('as_exit')
|
|
63
|
+
command_lines.push('')
|
|
32
64
|
# execute the main command and then exit
|
|
33
|
-
stdin_input =
|
|
34
|
-
Log.log.
|
|
65
|
+
stdin_input = command_lines.join("\n")
|
|
66
|
+
Log.log.trace1{"execute_single:#{stdin_input}"}
|
|
35
67
|
# execute, get binary output
|
|
36
68
|
byte_buffer = @command_executor.execute('ascmd', stdin_input).unpack('C*')
|
|
37
69
|
raise 'ERROR: empty answer from server' if byte_buffer.empty?
|
|
@@ -53,7 +85,11 @@ module Aspera
|
|
|
53
85
|
end
|
|
54
86
|
end
|
|
55
87
|
# for info, second overrides first, so restore it
|
|
56
|
-
case result.keys.length
|
|
88
|
+
case result.keys.length
|
|
89
|
+
when 0 then result = system_info
|
|
90
|
+
when 1 then result = result[result.keys.first]
|
|
91
|
+
else Aspera.error_unexpected_value(result.keys.length)
|
|
92
|
+
end
|
|
57
93
|
# raise error as exception
|
|
58
94
|
raise Error.new(result[:errno], result[:errstr], action_sym, arguments) if
|
|
59
95
|
result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{|i|i[:name]}.sort)
|
|
@@ -66,7 +102,7 @@ module Aspera
|
|
|
66
102
|
super(); @errno = errno; @errstr = errstr; @command = cmd; @arguments = arguments; end # rubocop:disable Style/Semicolon
|
|
67
103
|
|
|
68
104
|
def message; "ascmd: #{@errstr} (#{@errno})"; end
|
|
69
|
-
def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments
|
|
105
|
+
def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments&.join(',')}"; end
|
|
70
106
|
end # Error
|
|
71
107
|
|
|
72
108
|
# description of result structures (see ascmdtypes.h). Base types are big endian
|
|
@@ -174,7 +210,7 @@ module Aspera
|
|
|
174
210
|
end
|
|
175
211
|
end
|
|
176
212
|
end
|
|
177
|
-
else error_unexpected_value(type_descr[:decode])
|
|
213
|
+
else Aspera.error_unexpected_value(type_descr[:decode])
|
|
178
214
|
end # is_a
|
|
179
215
|
return result
|
|
180
216
|
end
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# cspell:ignore protobuf ckpt
|
|
4
4
|
require 'aspera/environment'
|
|
5
5
|
require 'aspera/data_repository'
|
|
6
|
-
require 'aspera/
|
|
6
|
+
require 'aspera/ascp/products'
|
|
7
7
|
require 'aspera/log'
|
|
8
8
|
require 'aspera/assert'
|
|
9
9
|
require 'aspera/web_server_simple'
|
|
@@ -16,7 +16,7 @@ require 'fileutils'
|
|
|
16
16
|
require 'openssl'
|
|
17
17
|
|
|
18
18
|
module Aspera
|
|
19
|
-
module
|
|
19
|
+
module Ascp
|
|
20
20
|
# Singleton that tells where to find ascp and other local resources (keys..) , using the "path(:name)" method.
|
|
21
21
|
# It is used by object : AgentDirect to find necessary resources
|
|
22
22
|
# By default it takes the first Aspera product found
|
|
@@ -139,10 +139,10 @@ module Aspera
|
|
|
139
139
|
check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
|
|
140
140
|
end
|
|
141
141
|
file = k.eql?(:fallback_certificate) ? file_cert : file_key
|
|
142
|
-
else error_unexpected_value(k)
|
|
142
|
+
else Aspera.error_unexpected_value(k)
|
|
143
143
|
end
|
|
144
144
|
return nil if file_is_optional && !File.exist?(file)
|
|
145
|
-
assert(File.exist?(file)){"no such file: #{file}"}
|
|
145
|
+
Aspera.assert(File.exist?(file)){"no such file: #{file}"}
|
|
146
146
|
return file
|
|
147
147
|
end
|
|
148
148
|
|
|
@@ -283,6 +283,6 @@ module Aspera
|
|
|
283
283
|
def transferd_filepath
|
|
284
284
|
return File.join(sdk_folder, 'asperatransferd' + Environment.exe_extension) # cspell:disable-line
|
|
285
285
|
end
|
|
286
|
-
end
|
|
286
|
+
end
|
|
287
287
|
end
|
|
288
288
|
end
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Aspera
|
|
4
|
-
module
|
|
5
|
-
#
|
|
4
|
+
module Ascp
|
|
5
|
+
# processing of ascp management port events
|
|
6
6
|
class Management
|
|
7
7
|
# cspell: disable
|
|
8
8
|
OPERATIONS = %w[
|
|
@@ -202,12 +202,7 @@ module Aspera
|
|
|
202
202
|
# translates mgt port event into (enhanced) typed event
|
|
203
203
|
def enhanced_event_format(event)
|
|
204
204
|
return event.keys.each_with_object({}) do |e, h|
|
|
205
|
-
|
|
206
|
-
new_name = e
|
|
207
|
-
.gsub(/([a-z\d])([A-Z])/, '\1_\2')
|
|
208
|
-
.gsub(/([a-z\d])(usec)$/, '\1_\2')
|
|
209
|
-
.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
|
|
210
|
-
.downcase
|
|
205
|
+
new_name = e.capital_to_snake.gsub(/(usec)$/, '_\1').downcase
|
|
211
206
|
value = event[e]
|
|
212
207
|
value = value.to_i if INTEGER_FIELDS.include?(e)
|
|
213
208
|
value = value.eql?(BOOLEAN_TRUE) if BOOLEAN_FIELDS.include?(e)
|