aspera-cli 4.26.2 → 4.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +310 -21
- data/CONTRIBUTING.md +99 -14
- data/TODO.md +50 -0
- data/bin/ascli +3 -3
- data/docs/README.md +10977 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +29 -7
- data/lib/aspera/agent/connect.rb +58 -21
- data/lib/aspera/agent/desktop.rb +53 -18
- data/lib/aspera/agent/direct.rb +71 -40
- data/lib/aspera/agent/factory.rb +3 -3
- data/lib/aspera/agent/httpgw.rb +55 -14
- data/lib/aspera/agent/node.rb +34 -2
- data/lib/aspera/agent/transferd.rb +48 -12
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +42 -44
- data/lib/aspera/api/cos_node.rb +10 -9
- data/lib/aspera/api/faspex.rb +16 -15
- data/lib/aspera/api/httpgw.rb +23 -23
- data/lib/aspera/api/node.rb +51 -70
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +32 -30
- data/lib/aspera/ascp/installation.rb +75 -69
- data/lib/aspera/ascp/management.rb +7 -8
- data/lib/aspera/assert.rb +12 -13
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +125 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +141 -0
- data/lib/aspera/cli/context.rb +19 -6
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +27 -26
- data/lib/aspera/cli/formatter.rb +95 -59
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +3 -3
- data/lib/aspera/cli/http.rb +74 -30
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +225 -0
- data/lib/aspera/cli/option_declarator.rb +77 -0
- data/lib/aspera/cli/options.schema.yaml +634 -7
- data/lib/aspera/cli/parser.rb +1217 -0
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1178 -881
- data/lib/aspera/cli/plugins/ats.rb +205 -162
- data/lib/aspera/cli/plugins/base.rb +703 -175
- data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
- data/lib/aspera/cli/plugins/config.rb +434 -757
- data/lib/aspera/cli/plugins/console.rb +107 -65
- data/lib/aspera/cli/plugins/cos.rb +46 -33
- data/lib/aspera/cli/plugins/factory.rb +6 -6
- data/lib/aspera/cli/plugins/faspex5.rb +627 -388
- data/lib/aspera/cli/plugins/faspio.rb +54 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +988 -840
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
- data/lib/aspera/cli/plugins/preview.rb +195 -157
- data/lib/aspera/cli/plugins/server.rb +141 -91
- data/lib/aspera/cli/plugins/shares.rb +343 -110
- data/lib/aspera/cli/preset_actions.rb +159 -0
- data/lib/aspera/cli/preset_manager.rb +82 -39
- data/lib/aspera/cli/result.rb +78 -26
- data/lib/aspera/cli/runner.rb +323 -153
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +89 -64
- data/lib/aspera/cli/terminal_formatter.rb +3 -3
- data/lib/aspera/cli/transfer_actions.rb +90 -0
- data/lib/aspera/cli/transfer_agent.rb +129 -64
- data/lib/aspera/cli/transfer_progress.rb +9 -9
- data/lib/aspera/cli/vault_manager.rb +79 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +28 -30
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +34 -34
- data/lib/aspera/command_line_converter.rb +1 -1
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +2 -1
- data/lib/aspera/dot_container.rb +12 -12
- data/lib/aspera/environment.rb +30 -23
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +5 -5
- data/lib/aspera/faspex_postproc.rb +16 -10
- data/lib/aspera/graphql.rb +37 -0
- data/lib/aspera/hash_ext.rb +8 -2
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/base.rb +10 -3
- data/lib/aspera/keychain/encrypted_hash.rb +92 -17
- data/lib/aspera/keychain/factory.rb +20 -8
- data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
- data/lib/aspera/keychain/macos_security.rb +23 -25
- data/lib/aspera/keychain/one_password_api.rb +86 -0
- data/lib/aspera/keychain/one_password_base.rb +34 -0
- data/lib/aspera/keychain/one_password_cli.rb +98 -0
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +29 -15
- data/lib/aspera/markdown.rb +90 -7
- data/lib/aspera/nagios.rb +8 -8
- data/lib/aspera/node_simulator.rb +35 -27
- data/lib/aspera/oauth/base.rb +9 -10
- data/lib/aspera/oauth/boot.rb +5 -5
- data/lib/aspera/oauth/factory.rb +12 -10
- data/lib/aspera/oauth/jwt.rb +9 -11
- data/lib/aspera/oauth/web.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +12 -12
- data/lib/aspera/persistency_folder.rb +18 -11
- data/lib/aspera/preview/file_types.rb +5 -5
- data/lib/aspera/preview/generator.rb +52 -49
- data/lib/aspera/preview/options.rb +3 -2
- data/lib/aspera/preview/terminal.rb +5 -5
- data/lib/aspera/preview/utils.rb +66 -19
- data/lib/aspera/products/connect.rb +2 -2
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/other.rb +3 -3
- data/lib/aspera/products/transferd.rb +4 -4
- data/lib/aspera/proxy_auto_config.rb +15 -14
- data/lib/aspera/rest.rb +159 -151
- data/lib/aspera/rest_error_analyzer.rb +6 -6
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +11 -10
- data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
- data/lib/aspera/schema/documentation.rb +65 -28
- data/lib/aspera/schema/reader.rb +136 -13
- data/lib/aspera/schema/registry.rb +60 -8
- data/lib/aspera/secret_hider.rb +12 -7
- data/lib/aspera/ssh.rb +64 -31
- data/lib/aspera/ssl.rb +5 -5
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/database.rb +2 -2
- data/lib/aspera/sync/operations.rb +22 -25
- data/lib/aspera/temp_file_manager.rb +25 -5
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +36 -34
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +10 -10
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +12 -3
- data/lib/aspera/uri_reader.rb +52 -15
- data/lib/aspera/web_auth.rb +8 -7
- data/lib/aspera/web_server_simple.rb +15 -12
- data/lib/aspera/yaml.rb +5 -4
- data.tar.gz.sig +0 -0
- metadata +34 -6
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/manager.rb +0 -766
- data/lib/aspera/cli/plugins/faspex.rb +0 -539
- data/lib/aspera/json_rpc.rb +0 -52
- data/lib/aspera/transfer/uri.rb +0 -56
data/lib/aspera/api/node.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'aspera/link_header'
|
|
3
4
|
require 'aspera/transfer/spec'
|
|
4
5
|
require 'aspera/rest'
|
|
5
6
|
require 'aspera/oauth'
|
|
@@ -42,14 +43,11 @@ module Aspera
|
|
|
42
43
|
SEPARATOR = ':'
|
|
43
44
|
NODE_PREFIX = 'node.'
|
|
44
45
|
end
|
|
45
|
-
# Accepted types in `file_matcher`
|
|
46
|
-
MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
|
|
47
46
|
# Delimiter in decoded node token
|
|
48
47
|
SIGNATURE_DELIMITER = '==SIGNATURE=='
|
|
49
48
|
# Default validity when generating a bearer token "manually"
|
|
50
49
|
BEARER_TOKEN_VALIDITY_DEFAULT = 86400
|
|
51
|
-
private_constant :
|
|
52
|
-
:SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT
|
|
50
|
+
private_constant :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT
|
|
53
51
|
# Node API permissions: delete list mkdir preview read rename write
|
|
54
52
|
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
|
55
53
|
# Special HTTP Headers
|
|
@@ -62,7 +60,7 @@ module Aspera
|
|
|
62
60
|
HEADER_X_TOTAL_COUNT = 'X-Total-Count'
|
|
63
61
|
|
|
64
62
|
# Register node special token decoder
|
|
65
|
-
OAuth::Factory.instance.register_decoder(lambda{ |token| Node.decode_bearer_token(token)})
|
|
63
|
+
OAuth::Factory.instance.register_decoder(lambda { |token| Node.decode_bearer_token(token) })
|
|
66
64
|
|
|
67
65
|
# Class instance variable, access with accessors on class
|
|
68
66
|
@api_options = {
|
|
@@ -78,12 +76,12 @@ module Aspera
|
|
|
78
76
|
attr_reader :api_options
|
|
79
77
|
attr_reader :use_dynamic_key
|
|
80
78
|
|
|
81
|
-
def api_options=(
|
|
82
|
-
Aspera.assert_type(
|
|
83
|
-
|
|
84
|
-
Aspera.assert(@api_options.key?(
|
|
85
|
-
Aspera.assert_type(
|
|
86
|
-
@api_options[
|
|
79
|
+
def api_options=(options_hash)
|
|
80
|
+
Aspera.assert_type(options_hash, Hash)
|
|
81
|
+
options_hash.each do |key, val|
|
|
82
|
+
Aspera.assert(@api_options.key?(key.to_sym)) { "unknown api option: #{key} (#{OPTIONS.join(', ')})" }
|
|
83
|
+
Aspera.assert_type(val, TrueClass, FalseClass) { "api options value for #{key} should be boolean" }
|
|
84
|
+
@api_options[key.to_sym] = val
|
|
87
85
|
end
|
|
88
86
|
end
|
|
89
87
|
|
|
@@ -105,48 +103,30 @@ module Aspera
|
|
|
105
103
|
end
|
|
106
104
|
|
|
107
105
|
# Adds fields `public_keys` in provided Hash, if dynamic key is set.
|
|
108
|
-
# @param
|
|
106
|
+
# @param parameters [Hash] Hash to add public key to
|
|
109
107
|
# @return [Hash] Hash with public key added
|
|
110
|
-
def add_public_key(
|
|
108
|
+
def add_public_key(parameters)
|
|
111
109
|
if @dynamic_key
|
|
112
110
|
ssh_key = Net::SSH::Buffer.from(:key, @dynamic_key)
|
|
113
111
|
# Get pub key in OpenSSH public key format (authorized_keys)
|
|
114
|
-
|
|
112
|
+
parameters['public_keys'] = [
|
|
115
113
|
ssh_key.read_string,
|
|
116
114
|
Base64.strict_encode64(ssh_key.to_s)
|
|
117
115
|
].join(' ')
|
|
118
116
|
end
|
|
119
|
-
return
|
|
117
|
+
return parameters
|
|
120
118
|
end
|
|
121
119
|
|
|
122
120
|
# Adds fields `ssh_private_key` in provided Hash, if dynamic key is set.
|
|
123
|
-
# @param
|
|
124
|
-
def add_private_key(
|
|
125
|
-
|
|
126
|
-
return
|
|
127
|
-
end
|
|
128
|
-
|
|
129
|
-
# For access keys: provide expression to match entry in folder
|
|
130
|
-
# @param match_expression one of supported types
|
|
131
|
-
# @return lambda function
|
|
132
|
-
def file_matcher(match_expression)
|
|
133
|
-
case match_expression
|
|
134
|
-
when Proc then return match_expression
|
|
135
|
-
when Regexp then return ->(f){f['name'].match?(match_expression)}
|
|
136
|
-
when String
|
|
137
|
-
return ->(f){File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
|
|
138
|
-
when NilClass then return ->(_){true}
|
|
139
|
-
else Aspera.error_unexpected_value(match_expression.class.name, type: ParameterError)
|
|
140
|
-
end
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
# @return [Proc] lambda from provided CLI options
|
|
144
|
-
def file_matcher_from_argument(options)
|
|
145
|
-
return file_matcher(options.get_next_argument('filter', validation: MATCH_TYPES, mandatory: false))
|
|
121
|
+
# @param parameters [Hash] Hash to add private key to
|
|
122
|
+
def add_private_key(parameters)
|
|
123
|
+
parameters['ssh_private_key'] = @dynamic_key.to_pem if @dynamic_key
|
|
124
|
+
return parameters
|
|
146
125
|
end
|
|
147
126
|
|
|
148
127
|
# Split path into folder + filename
|
|
149
|
-
# @
|
|
128
|
+
# @param path [String] file path to split
|
|
129
|
+
# @return [Array(String, String)] containing folder + inside folder/file
|
|
150
130
|
def split_folder(path)
|
|
151
131
|
folder = path.split(PATH_SEPARATOR)
|
|
152
132
|
inside = folder.pop
|
|
@@ -160,11 +140,12 @@ module Aspera
|
|
|
160
140
|
end
|
|
161
141
|
|
|
162
142
|
# Decode node scope into access key and scope
|
|
163
|
-
# @
|
|
143
|
+
# @param scope [String] scope string to decode
|
|
144
|
+
# @return [Hash] decoded scope with access key and scope parts
|
|
164
145
|
def decode_scope(scope)
|
|
165
146
|
items = scope.split(Scope::SEPARATOR, 2)
|
|
166
|
-
Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
|
|
167
|
-
Aspera.assert(items[0].start_with?(Scope::NODE_PREFIX)){"invalid scope: #{scope}"}
|
|
147
|
+
Aspera.assert(items.length.eql?(2)) { "invalid scope: #{scope}" }
|
|
148
|
+
Aspera.assert(items[0].start_with?(Scope::NODE_PREFIX)) { "invalid scope: #{scope}" }
|
|
168
149
|
return {access_key: items[0].delete_prefix(Scope::NODE_PREFIX), scope: items[1]}
|
|
169
150
|
end
|
|
170
151
|
|
|
@@ -240,7 +221,7 @@ module Aspera
|
|
|
240
221
|
return tspec
|
|
241
222
|
end
|
|
242
223
|
|
|
243
|
-
# @
|
|
224
|
+
# @return [Node, nil] a Node Api object or nil if no App defined
|
|
244
225
|
def node_id_to_node(node_id)
|
|
245
226
|
if !@app_info.nil?
|
|
246
227
|
return self if node_id.eql?(@app_info.node_info['id'])
|
|
@@ -250,10 +231,10 @@ module Aspera
|
|
|
250
231
|
workspace_name: @app_info.workspace_name
|
|
251
232
|
)
|
|
252
233
|
end
|
|
253
|
-
Log.log.warn{"Cannot resolve link with node id #{node_id}, no resolver"}
|
|
234
|
+
Log.log.warn { "Cannot resolve link with node id #{node_id}, no resolver" }
|
|
254
235
|
return
|
|
255
236
|
rescue RestCallError => e
|
|
256
|
-
Log.log.warn{"Cannot resolve link with node id #{node_id}: #{e.message}"}
|
|
237
|
+
Log.log.warn { "Cannot resolve link with node id #{node_id}: #{e.message}" }
|
|
257
238
|
return
|
|
258
239
|
end
|
|
259
240
|
|
|
@@ -268,7 +249,7 @@ module Aspera
|
|
|
268
249
|
entry['target_id'] = link_entry['target_id']
|
|
269
250
|
end
|
|
270
251
|
return true unless entry['target_node_id'].nil? || entry['target_id'].nil?
|
|
271
|
-
Log.log.warn{"Missing target information for link: #{entry['name']}"}
|
|
252
|
+
Log.log.warn { "Missing target information for link: #{entry['name']}" }
|
|
272
253
|
return false
|
|
273
254
|
end
|
|
274
255
|
|
|
@@ -320,8 +301,8 @@ module Aspera
|
|
|
320
301
|
end
|
|
321
302
|
rescue StandardError => e
|
|
322
303
|
raise e if exception
|
|
323
|
-
Log.log.warn{"#{path || file_id}: #{e.class} #{e.message}"}
|
|
324
|
-
Log.log.debug{(['Backtrace:'] + e.backtrace).join("\n")}
|
|
304
|
+
Log.log.warn { "#{path || file_id}: #{e.class} #{e.message}" }
|
|
305
|
+
Log.log.debug { (['Backtrace:'] + e.backtrace).join("\n") }
|
|
325
306
|
ensure
|
|
326
307
|
RestParameters.instance.spinner_cb.call(folder_items.count, action: :success)
|
|
327
308
|
end
|
|
@@ -338,14 +319,14 @@ module Aspera
|
|
|
338
319
|
# @para query [Hash, nil] optional query for `read`
|
|
339
320
|
def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil)
|
|
340
321
|
Aspera.assert(!top_file_path.nil?, 'top_file_path not set')
|
|
341
|
-
Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info.node_info['id'] : 'nil'}, file id=#{top_file_id}, path=#{top_file_path}"}
|
|
322
|
+
Log.log.debug { "process_folder_tree: node=#{@app_info ? @app_info.node_info['id'] : 'nil'}, file id=#{top_file_id}, path=#{top_file_path}" }
|
|
342
323
|
# Start at top folder
|
|
343
324
|
folders_to_explore = [{id: top_file_id, path: top_file_path}]
|
|
344
325
|
Log.dump(:folders_to_explore, folders_to_explore)
|
|
345
326
|
until folders_to_explore.empty?
|
|
346
327
|
# Consume first in job list
|
|
347
328
|
current_item = folders_to_explore.shift
|
|
348
|
-
Log.log.debug{"Exploring #{current_item[:path]}".bg_green}
|
|
329
|
+
Log.log.debug { "Exploring #{current_item[:path]}".bg_green }
|
|
349
330
|
# Get folder content
|
|
350
331
|
folder_contents = read_folder_content(current_item[:id], query, exception: false, path: current_item[:path])
|
|
351
332
|
Log.dump(:folder_contents, folder_contents)
|
|
@@ -355,7 +336,7 @@ module Aspera
|
|
|
355
336
|
next
|
|
356
337
|
end
|
|
357
338
|
current_path = File.join(current_item[:path], entry['name'])
|
|
358
|
-
Log.log.debug{"process_folder_tree: checking #{current_path}"}
|
|
339
|
+
Log.log.debug { "process_folder_tree: checking #{current_path}" }
|
|
359
340
|
# Call block, continue only if method returns true
|
|
360
341
|
next unless send(method_sym, entry, current_path, state)
|
|
361
342
|
# Entry type is file, folder or link
|
|
@@ -390,8 +371,8 @@ module Aspera
|
|
|
390
371
|
return NodeFileId.new(self, top_file_id) if path_elements.empty?
|
|
391
372
|
resolve_state = {path: path_elements, consumed: [], result: nil, process_last_link: process_last_link}
|
|
392
373
|
process_folder_tree(method_sym: :process_api_fid, state: resolve_state, top_file_id: top_file_id)
|
|
393
|
-
|
|
394
|
-
Log.log.debug{"resolve_api_fid: #{path} -> #{resolve_state[:result].node_api.base_url} #{resolve_state[:result].file_id}"}
|
|
374
|
+
Aspera.assert(!resolve_state[:result].nil?, type: ParameterError) { "Entry not found: #{resolve_state[:path].first} in /#{resolve_state[:consumed].join(PATH_SEPARATOR)}" }
|
|
375
|
+
Log.log.debug { "resolve_api_fid: #{path} -> #{resolve_state[:result].node_api.base_url} #{resolve_state[:result].file_id}" }
|
|
395
376
|
return resolve_state[:result]
|
|
396
377
|
end
|
|
397
378
|
|
|
@@ -406,10 +387,10 @@ module Aspera
|
|
|
406
387
|
def resolve_api_fid_paths(top_file_id, paths)
|
|
407
388
|
Aspera.assert_type(paths, Array)
|
|
408
389
|
Aspera.assert(paths.size.positive?, 'paths must not be empty')
|
|
409
|
-
split_sources = paths.map{ |p| Pathname(p['source']).each_filename.to_a}
|
|
390
|
+
split_sources = paths.map { |p| Pathname(p['source']).each_filename.to_a }
|
|
410
391
|
root = []
|
|
411
392
|
split_sources.map(&:size).min.times do |i|
|
|
412
|
-
parts = split_sources.map{ |s| s[i]}
|
|
393
|
+
parts = split_sources.map { |s| s[i] }
|
|
413
394
|
break unless parts.uniq.size == 1
|
|
414
395
|
root << parts.first
|
|
415
396
|
end
|
|
@@ -437,7 +418,7 @@ module Aspera
|
|
|
437
418
|
# Single source is 'folder' or 'link'
|
|
438
419
|
# TODO: add this ? , 'destination'=>file_info['name']
|
|
439
420
|
[{'source' => '.'}]
|
|
440
|
-
else Aspera.error_unexpected_value(file_info['type']){'source type'}
|
|
421
|
+
else Aspera.error_unexpected_value(file_info['type']) { 'source type' }
|
|
441
422
|
end
|
|
442
423
|
end
|
|
443
424
|
[apifid, source_paths]
|
|
@@ -447,7 +428,7 @@ module Aspera
|
|
|
447
428
|
# @param top_file_id [String] Search root
|
|
448
429
|
# @param test_lambda [Proc] Test function
|
|
449
430
|
def find_files(top_file_id, test_lambda)
|
|
450
|
-
Log.log.debug{"find_files: file id=#{top_file_id}"}
|
|
431
|
+
Log.log.debug { "find_files: file id=#{top_file_id}" }
|
|
451
432
|
find_state = {found: [], test_lambda: test_lambda}
|
|
452
433
|
process_folder_tree(method_sym: :process_find_files, state: find_state, top_file_id: top_file_id)
|
|
453
434
|
return find_state[:found]
|
|
@@ -532,7 +513,7 @@ module Aspera
|
|
|
532
513
|
# Get the transfer user from info on access key
|
|
533
514
|
transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
|
|
534
515
|
# Get settings from name.value array to hash key.value
|
|
535
|
-
settings = info['settings']&.to_h{ |i| [i['name'], i['value']]}
|
|
516
|
+
settings = info['settings']&.to_h { |i| [i['name'], i['value']] }
|
|
536
517
|
# Check WSS ports
|
|
537
518
|
Transfer::Spec::WSS_FIELDS.each do |i|
|
|
538
519
|
transfer_spec[i] = settings[i] if settings.key?(i)
|
|
@@ -540,7 +521,7 @@ module Aspera
|
|
|
540
521
|
else
|
|
541
522
|
transfer_spec.merge!(transport_params)
|
|
542
523
|
end
|
|
543
|
-
Aspera.assert_values(transfer_spec['remote_user'], [Transfer::Spec::ACCESS_KEY_TRANSFER_USER], type: :warn){'transfer user'}
|
|
524
|
+
Aspera.assert_values(transfer_spec['remote_user'], [Transfer::Spec::ACCESS_KEY_TRANSFER_USER], type: :warn) { 'transfer user' }
|
|
544
525
|
return transfer_spec
|
|
545
526
|
end
|
|
546
527
|
|
|
@@ -549,8 +530,8 @@ module Aspera
|
|
|
549
530
|
# @param call_args [Hash] additional arguments to pass to `Rest.call`
|
|
550
531
|
# @return [Array] list of items returned by the API call
|
|
551
532
|
def read_with_paging(subpath, query = nil, iteration: nil, **call_args)
|
|
552
|
-
Aspera.assert_type(iteration, Array, NilClass){'iteration'}
|
|
553
|
-
Aspera.assert_type(query, Hash, NilClass){'query'}
|
|
533
|
+
Aspera.assert_type(iteration, Array, NilClass) { 'iteration' }
|
|
534
|
+
Aspera.assert_type(query, Hash, NilClass) { 'query' }
|
|
554
535
|
Aspera.assert(!call_args.key?(:query), ':query must not be in call_args (use query parameter)')
|
|
555
536
|
query = {} if query.nil?
|
|
556
537
|
query[:iteration_token] = iteration[0] unless iteration.nil? || iteration[0].nil?
|
|
@@ -560,7 +541,7 @@ module Aspera
|
|
|
560
541
|
item_list = []
|
|
561
542
|
loop do
|
|
562
543
|
data, http = read(subpath, query, **call_args, ret: :both)
|
|
563
|
-
Aspera.assert_type(data, Array){"Expected data to be an Array, got: #{data.class}"}
|
|
544
|
+
Aspera.assert_type(data, Array) { "Expected data to be an Array, got: #{data.class}" }
|
|
564
545
|
# no data
|
|
565
546
|
break if data.empty?
|
|
566
547
|
item_list.concat(data)
|
|
@@ -572,7 +553,7 @@ module Aspera
|
|
|
572
553
|
# Update progress spinner
|
|
573
554
|
RestParameters.instance.spinner_cb.call(item_list.length)
|
|
574
555
|
# Parse Link header according to RFC 8288 to extract next iteration token
|
|
575
|
-
next_url =
|
|
556
|
+
next_url = LinkHeader.parse(http['Link']).find_href(rel: 'next')
|
|
576
557
|
next_iteration_token = nil
|
|
577
558
|
if next_url
|
|
578
559
|
begin
|
|
@@ -580,7 +561,7 @@ module Aspera
|
|
|
580
561
|
query_params = Rest.query_to_h(parsed_uri.query) if parsed_uri.query
|
|
581
562
|
next_iteration_token = query_params['iteration_token'] if query_params
|
|
582
563
|
rescue URI::InvalidURIError => e
|
|
583
|
-
Log.log.warn{"Invalid URI in Link header: #{next_url} - #{e.message}"}
|
|
564
|
+
Log.log.warn { "Invalid URI in Link header: #{next_url} - #{e.message}" }
|
|
584
565
|
end
|
|
585
566
|
end
|
|
586
567
|
# Stop if no next token
|
|
@@ -619,8 +600,8 @@ module Aspera
|
|
|
619
600
|
query['page'] += 1
|
|
620
601
|
end
|
|
621
602
|
rescue StandardError => e
|
|
622
|
-
Log.log.warn{"#{e.class} #{e.message}"}
|
|
623
|
-
Log.log.debug{(['Backtrace:'] + e.backtrace).join("\n")}
|
|
603
|
+
Log.log.warn { "#{e.class} #{e.message}" }
|
|
604
|
+
Log.log.debug { (['Backtrace:'] + e.backtrace).join("\n") }
|
|
624
605
|
ensure
|
|
625
606
|
RestParameters.instance.spinner_cb.call(items.count, action: :success)
|
|
626
607
|
return items # rubocop:disable Lint/EnsureReturn
|
|
@@ -629,7 +610,7 @@ module Aspera
|
|
|
629
610
|
private
|
|
630
611
|
|
|
631
612
|
# Method called in loop for each entry for `resolve_api_fid`
|
|
632
|
-
# @return `true` to continue digging, `false` to stop processing: set state[:result] if found
|
|
613
|
+
# @return [Boolean] `true` to continue digging, `false` to stop processing: set state[:result] if found
|
|
633
614
|
def process_api_fid(entry, path, state)
|
|
634
615
|
# Stop digging here if not in right path
|
|
635
616
|
return false unless entry['name'].eql?(state[:path].first)
|
|
@@ -641,7 +622,7 @@ module Aspera
|
|
|
641
622
|
# File must be terminal
|
|
642
623
|
raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless path_fully_consumed
|
|
643
624
|
# It's terminal, we found it
|
|
644
|
-
Log.log.debug{"process_api_fid: found #{path} -> #{entry['id']}"}
|
|
625
|
+
Log.log.debug { "process_api_fid: found #{path} -> #{entry['id']}" }
|
|
645
626
|
state[:result] = NodeFileId.new(self, entry['id'])
|
|
646
627
|
return false
|
|
647
628
|
when 'folder'
|
|
@@ -656,7 +637,7 @@ module Aspera
|
|
|
656
637
|
# We found it
|
|
657
638
|
other_node = nil
|
|
658
639
|
other_node = node_id_to_node(entry['target_node_id']) if entry_has_link_information(entry)
|
|
659
|
-
|
|
640
|
+
Aspera.assert(!other_node.nil?, type: Error) { 'Cannot resolve link' }
|
|
660
641
|
state[:result] = NodeFileId.new(other_node, entry['target_id'])
|
|
661
642
|
else
|
|
662
643
|
# We found it but we do not process the link
|
|
@@ -664,7 +645,7 @@ module Aspera
|
|
|
664
645
|
end
|
|
665
646
|
return false
|
|
666
647
|
end
|
|
667
|
-
else Aspera.error_unexpected_value(entry['type'], type: :warn){'folder entry type'}
|
|
648
|
+
else Aspera.error_unexpected_value(entry['type'], type: :warn) { 'folder entry type' }
|
|
668
649
|
end
|
|
669
650
|
# Continue to dig folder
|
|
670
651
|
return true
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
query ($organization_id: ID!) {
|
|
2
|
+
aoc (organization_id: $organization_id) {
|
|
3
|
+
bssSubscription {
|
|
4
|
+
aocVersion endDate startDate termMonths plan trial termType
|
|
5
|
+
aocOrganizations { id }
|
|
6
|
+
additionalStorageVolumeGb additionalEgressVolumeGb
|
|
7
|
+
term { startDate endDate transferVolumeGb egressVolumeGb storageVolumeGb transferVolumeOffsetGb }
|
|
8
|
+
paygoRate { transferRate storageRate currency }
|
|
9
|
+
aocPlanData {
|
|
10
|
+
tier trial
|
|
11
|
+
workspaces { max }
|
|
12
|
+
users { planAmount max }
|
|
13
|
+
samlIntegration activity sharedInboxes uniqueUrls support watermarking byok
|
|
14
|
+
automation { planAmount, max }
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
query ($organization_id: ID!, $startDate: Date!, $endDate: Date!, $aggregate: TransferUsageAggregateOption!) {
|
|
2
|
+
aoc (organization_id: $organization_id) {
|
|
3
|
+
bssSubscription {
|
|
4
|
+
aocOrganizations { id }
|
|
5
|
+
additionalStorageVolumeGb additionalEgressVolumeGb
|
|
6
|
+
aocPlanData { tier trial }
|
|
7
|
+
term { transferVolumeGb egressVolumeGb storageVolumeGb startDate endDate transferVolumeOffsetGb }
|
|
8
|
+
termMonths
|
|
9
|
+
transferUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { mbTotal }
|
|
10
|
+
egressUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { usageMb }
|
|
11
|
+
}
|
|
12
|
+
subscriptionEntitlements {
|
|
13
|
+
id
|
|
14
|
+
transferUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { mbTotal }
|
|
15
|
+
egressUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { usageMb }
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
data/lib/aspera/ascmd.rb
CHANGED
|
@@ -28,18 +28,18 @@ module Aspera
|
|
|
28
28
|
ENUM_START = 1
|
|
29
29
|
|
|
30
30
|
# Description of result structures (see ascmdtypes.h).
|
|
31
|
-
# Base types are big endian
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
# :
|
|
42
|
-
# rubocop:disable Layout/FirstHashElementLineBreak
|
|
31
|
+
# Base types are big endian.
|
|
32
|
+
# Keys are type names; the index in each `fields` array equals (TLV numerical type - `ENUM_START`).
|
|
33
|
+
# Decoding always starts at `result`.
|
|
34
|
+
# Some fields have special handling indicated by `special`:
|
|
35
|
+
# - `:field_list` : a Hash or Array of child TLVs
|
|
36
|
+
# - `:list_tlv_list` : a list of TLV sub-lists (Array of Hash)
|
|
37
|
+
# - `:list_tlv_restart` : same but restarted per entry
|
|
38
|
+
# `decode` values:
|
|
39
|
+
# - `:base` : a single scalar value (unpack with `unpack`)
|
|
40
|
+
# - `:buffer_list` : an Array of `{btype, buffer}` entries
|
|
41
|
+
# - `:field_list` : a Hash or Array built from child TLVs
|
|
42
|
+
# rubocop:disable-next Layout/FirstHashElementLineBreak
|
|
43
43
|
TYPES_DESCR = {
|
|
44
44
|
result: {decode: :field_list,
|
|
45
45
|
fields: [{name: :file, is_a: :stat}, {name: :dir, is_a: :stat, special: :list_tlv_list}, {name: :size, is_a: :size}, {name: :error, is_a: :error},
|
|
@@ -73,7 +73,6 @@ module Aspera
|
|
|
73
73
|
zstr: {decode: :base, unpack: 'Z*'},
|
|
74
74
|
blist: {decode: :buffer_list}
|
|
75
75
|
}.freeze
|
|
76
|
-
# rubocop:enable Layout/FirstHashElementLineBreak
|
|
77
76
|
|
|
78
77
|
private_constant :TYPES_DESCR, :ENUM_START, :OPS_ARGS
|
|
79
78
|
|
|
@@ -87,14 +86,14 @@ module Aspera
|
|
|
87
86
|
|
|
88
87
|
# execute an "as" command on a remote server
|
|
89
88
|
# Version 2 allows use of reverse proxy with multiple addresses.
|
|
90
|
-
# @param [Symbol] one of OPERATIONS
|
|
91
|
-
# @param [Array]
|
|
89
|
+
# @param action_sym [Symbol] one of OPERATIONS
|
|
90
|
+
# @param arguments [Array] parameters for "as" command
|
|
92
91
|
# @return [Boolean,Array,Hash] result of command, type depends on command
|
|
93
|
-
def execute_single(action_sym, arguments, version:
|
|
92
|
+
def execute_single(action_sym, arguments, version: 2, host: nil)
|
|
94
93
|
arguments = [] if arguments.nil?
|
|
95
|
-
Log.log.debug{"execute_single:#{action_sym}:#{arguments}"}
|
|
94
|
+
Log.log.debug { "execute_single:#{action_sym}:#{arguments}" }
|
|
96
95
|
Aspera.assert_type(action_sym, Symbol)
|
|
97
|
-
Aspera.assert_array_all(arguments, String){'arguments'}
|
|
96
|
+
Aspera.assert_array_all(arguments, String) { 'arguments' }
|
|
98
97
|
remote_cmd = 'ascmd'
|
|
99
98
|
# lines of commands (String's)
|
|
100
99
|
command_lines = []
|
|
@@ -118,7 +117,7 @@ module Aspera
|
|
|
118
117
|
# enclose arguments in double quotes, protect backslash and double quotes
|
|
119
118
|
# ascmd uses space as token separator, and optional quotes ('") or \ to escape
|
|
120
119
|
args.each do |v|
|
|
121
|
-
command.push(%Q{"#{v.gsub(/["\\]/){ |s| "\\#{s}"}}"})
|
|
120
|
+
command.push(%Q{"#{v.gsub(/["\\]/) { |s| "\\#{s}" }}"})
|
|
122
121
|
end
|
|
123
122
|
command_lines.push(command.join(' '))
|
|
124
123
|
end
|
|
@@ -126,7 +125,7 @@ module Aspera
|
|
|
126
125
|
command_lines.push('')
|
|
127
126
|
# execute the main command and then exit
|
|
128
127
|
stdin_input = command_lines.join("\n")
|
|
129
|
-
Log.log.trace1{"execute_single:#{stdin_input}"}
|
|
128
|
+
Log.log.trace1 { "execute_single:#{stdin_input}" }
|
|
130
129
|
# execute, get binary output
|
|
131
130
|
byte_buffer = @command_executor.execute(remote_cmd, input: stdin_input).unpack('C*')
|
|
132
131
|
Aspera.assert(!byte_buffer.empty?, 'empty answer from server')
|
|
@@ -155,7 +154,7 @@ module Aspera
|
|
|
155
154
|
end
|
|
156
155
|
# raise error as exception
|
|
157
156
|
raise Error.new(result[:errno], result[:errstr], action_sym, arguments) if
|
|
158
|
-
result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{ |i| i[:name]}.sort)
|
|
157
|
+
result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map { |i| i[:name] }.sort)
|
|
159
158
|
return result
|
|
160
159
|
end
|
|
161
160
|
|
|
@@ -169,20 +168,23 @@ module Aspera
|
|
|
169
168
|
end
|
|
170
169
|
|
|
171
170
|
class << self
|
|
172
|
-
# Get description of structure's field
|
|
171
|
+
# Get description of structure's field
|
|
172
|
+
# @param struct_name [Symbol] name of the structure type
|
|
173
|
+
# @param typed_buffer [Hash] typed buffer providing the field name via `:btype`
|
|
174
|
+
# @return [Hash] field description
|
|
173
175
|
def field_description(struct_name, typed_buffer)
|
|
174
176
|
result = TYPES_DESCR[struct_name][:fields][typed_buffer[:btype] - ENUM_START]
|
|
175
|
-
|
|
177
|
+
Aspera.assert(!result.nil?) { "Unrecognized field for #{struct_name}: #{typed_buffer[:btype]}\n#{typed_buffer[:buffer]}" }
|
|
176
178
|
return result
|
|
177
179
|
end
|
|
178
180
|
|
|
179
181
|
# Decodes the provided buffer as provided type name
|
|
180
|
-
# @return a decoded type: single, Array, or Hash
|
|
182
|
+
# @return [Boolean, Array, Hash] a decoded type: single, Array, or Hash
|
|
181
183
|
def parse(buffer, type_name, indent_level = nil)
|
|
182
184
|
indent_level = (indent_level || -1) + 1
|
|
183
185
|
type_descr = TYPES_DESCR[type_name]
|
|
184
|
-
|
|
185
|
-
Log.log.trace1{"#{' .' * indent_level}parse:#{type_name}:#{type_descr[:decode]}:#{buffer[0, 16]}...".red}
|
|
186
|
+
Aspera.assert(!type_descr.nil?) { "Unexpected type #{type_name}" }
|
|
187
|
+
Log.log.trace1 { "#{' .' * indent_level}parse:#{type_name}:#{type_descr[:decode]}:#{buffer[0, 16]}...".red }
|
|
186
188
|
result = nil
|
|
187
189
|
case type_descr[:decode]
|
|
188
190
|
when :base
|
|
@@ -192,7 +194,7 @@ module Aspera
|
|
|
192
194
|
byte_array = [byte_array] unless byte_array.is_a?(Array)
|
|
193
195
|
result = byte_array.pack('C*').unpack1(type_descr[:unpack])
|
|
194
196
|
result.force_encoding('UTF-8') if type_name.eql?(:zstr)
|
|
195
|
-
Log.log.trace1{"#{' .' * indent_level}-> base:#{byte_array} -> #{result}"}
|
|
197
|
+
Log.log.trace1 { "#{' .' * indent_level}-> base:#{byte_array} -> #{result}" }
|
|
196
198
|
result = Time.at(result) if type_name.eql?(:epoch)
|
|
197
199
|
when :buffer_list
|
|
198
200
|
# return a list of type_buffer
|
|
@@ -203,7 +205,7 @@ module Aspera
|
|
|
203
205
|
Aspera.assert(buffer.length >= length, 'not enough bytes')
|
|
204
206
|
value = buffer.shift(length)
|
|
205
207
|
result.push({btype: btype, buffer: value})
|
|
206
|
-
Log.log.trace1{"#{' .' * indent_level}:buffer_list[#{result.length - 1}] #{result.last}"}
|
|
208
|
+
Log.log.trace1 { "#{' .' * indent_level}:buffer_list[#{result.length - 1}] #{result.last}" }
|
|
207
209
|
end
|
|
208
210
|
when :field_list
|
|
209
211
|
# by default the result is one struct
|
|
@@ -212,7 +214,7 @@ module Aspera
|
|
|
212
214
|
parse(buffer, :blist, indent_level).each do |typed_buffer|
|
|
213
215
|
# what type of field is it ?
|
|
214
216
|
field_info = field_description(type_name, typed_buffer)
|
|
215
|
-
Log.log.trace1{"#{' .' * indent_level}+ field(special=#{field_info[:special]})=#{field_info[:name]}".green}
|
|
217
|
+
Log.log.trace1 { "#{' .' * indent_level}+ field(special=#{field_info[:special]})=#{field_info[:name]}".green }
|
|
216
218
|
case field_info[:special]
|
|
217
219
|
when nil # normal case
|
|
218
220
|
result[field_info[:name]] = parse(typed_buffer[:buffer], field_info[:is_a], indent_level)
|
|
@@ -222,7 +224,7 @@ module Aspera
|
|
|
222
224
|
result[field_info[:name]] ||= []
|
|
223
225
|
result[field_info[:name]].push(parse(typed_buffer[:buffer], field_info[:is_a], indent_level))
|
|
224
226
|
when :list_tlv_list # field is an array of values in a list of buffers
|
|
225
|
-
result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{ |r| parse(r[:buffer], field_info[:is_a], indent_level)}
|
|
227
|
+
result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map { |r| parse(r[:buffer], field_info[:is_a], indent_level) }
|
|
226
228
|
when :list_tlv_restart # field is an array of values, but a new value is started on index 1
|
|
227
229
|
fl = result[field_info[:name]] = []
|
|
228
230
|
parse(typed_buffer[:buffer], :blist, indent_level).map do |tb|
|