aspera-cli 4.26.2 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +281 -21
- data/CONTRIBUTING.md +98 -7
- data/bin/ascli +3 -3
- data/docs/README.md +10757 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +20 -6
- data/lib/aspera/agent/connect.rb +53 -16
- data/lib/aspera/agent/desktop.rb +52 -17
- data/lib/aspera/agent/direct.rb +26 -24
- data/lib/aspera/agent/httpgw.rb +1 -1
- data/lib/aspera/agent/node.rb +32 -0
- data/lib/aspera/agent/transferd.rb +39 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -16
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/faspex.rb +15 -14
- data/lib/aspera/api/httpgw.rb +4 -4
- data/lib/aspera/api/node.rb +26 -23
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +21 -19
- data/lib/aspera/ascp/installation.rb +37 -38
- data/lib/aspera/ascp/management.rb +2 -3
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +81 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +138 -0
- data/lib/aspera/cli/context.rb +16 -3
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +2 -1
- data/lib/aspera/cli/formatter.rb +41 -27
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +2 -2
- data/lib/aspera/cli/http.rb +70 -19
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +198 -0
- data/lib/aspera/cli/options.schema.yaml +364 -7
- data/lib/aspera/cli/{manager.rb → parser.rb} +334 -142
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1166 -853
- data/lib/aspera/cli/plugins/ats.rb +200 -161
- data/lib/aspera/cli/plugins/base.rb +457 -100
- data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
- data/lib/aspera/cli/plugins/config.rb +428 -754
- data/lib/aspera/cli/plugins/console.rb +106 -64
- data/lib/aspera/cli/plugins/cos.rb +44 -32
- data/lib/aspera/cli/plugins/factory.rb +4 -4
- data/lib/aspera/cli/plugins/faspex.rb +294 -257
- data/lib/aspera/cli/plugins/faspex5.rb +585 -357
- data/lib/aspera/cli/plugins/faspio.rb +49 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +998 -794
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +111 -134
- data/lib/aspera/cli/plugins/preview.rb +180 -139
- data/lib/aspera/cli/plugins/server.rb +124 -79
- data/lib/aspera/cli/plugins/shares.rb +301 -107
- data/lib/aspera/cli/preset_actions.rb +139 -0
- data/lib/aspera/cli/preset_manager.rb +13 -12
- data/lib/aspera/cli/result.rb +61 -11
- data/lib/aspera/cli/runner.rb +275 -141
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +83 -59
- data/lib/aspera/cli/terminal_formatter.rb +1 -1
- data/lib/aspera/cli/transfer_actions.rb +83 -0
- data/lib/aspera/cli/transfer_agent.rb +116 -51
- data/lib/aspera/cli/transfer_progress.rb +3 -3
- data/lib/aspera/cli/vault_manager.rb +57 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +21 -20
- data/lib/aspera/command_line_builder.rb +22 -22
- data/lib/aspera/data_repository.rb +2 -1
- data/lib/aspera/dot_container.rb +4 -4
- data/lib/aspera/environment.rb +18 -11
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +4 -3
- data/lib/aspera/graphql.rb +35 -0
- data/lib/aspera/hash_ext.rb +6 -0
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/encrypted_hash.rb +1 -1
- data/lib/aspera/keychain/factory.rb +3 -3
- data/lib/aspera/keychain/macos_security.rb +3 -3
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +17 -3
- data/lib/aspera/markdown.rb +85 -2
- data/lib/aspera/node_simulator.rb +3 -1
- data/lib/aspera/oauth/base.rb +5 -5
- data/lib/aspera/oauth/boot.rb +2 -2
- data/lib/aspera/oauth/factory.rb +7 -5
- data/lib/aspera/oauth/jwt.rb +4 -4
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/persistency_action_once.rb +8 -8
- data/lib/aspera/persistency_folder.rb +10 -3
- data/lib/aspera/preview/file_types.rb +3 -3
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/utils.rb +1 -1
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/other.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +7 -6
- data/lib/aspera/rest.rb +10 -36
- data/lib/aspera/rest_error_analyzer.rb +3 -3
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +3 -3
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
- data/lib/aspera/schema/documentation.rb +57 -26
- data/lib/aspera/schema/reader.rb +59 -10
- data/lib/aspera/schema/registry.rb +39 -6
- data/lib/aspera/secret_hider.rb +7 -2
- data/lib/aspera/ssl.rb +4 -4
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/operations.rb +10 -13
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +23 -21
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +6 -6
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +12 -3
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +8 -5
- data/lib/aspera/yaml.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +25 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/json_rpc.rb +0 -52
data/lib/aspera/agent/node.rb
CHANGED
|
@@ -13,6 +13,38 @@ module Aspera
|
|
|
13
13
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
|
14
14
|
# before using it, the use must set the `node_api` member.
|
|
15
15
|
class Node < Base
|
|
16
|
+
class << self
|
|
17
|
+
# Re-query a previously started transfer via the Node REST API.
|
|
18
|
+
# agent_params must contain 'url', 'username', 'password' (and optionally 'root_id').
|
|
19
|
+
# @return [Hash] normalized status hash
|
|
20
|
+
def transfer_status(transfer_id, agent_params)
|
|
21
|
+
rest_params = {base_url: agent_params['url']}
|
|
22
|
+
if OAuth::Factory.bearer_auth?(agent_params['password'])
|
|
23
|
+
rest_params[:headers] = Api::Node.bearer_headers(agent_params['password'], access_key: agent_params['username'])
|
|
24
|
+
else
|
|
25
|
+
rest_params[:auth] = {type: :basic, username: agent_params['username'], password: agent_params['password']}
|
|
26
|
+
end
|
|
27
|
+
node_api = Rest.new(**rest_params)
|
|
28
|
+
data = node_api.read("ops/transfers/#{transfer_id}") || {'status' => 'unknown'}
|
|
29
|
+
normalize_status(data)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Normalize a node API transfer hash into the standard async-transfer fields.
|
|
33
|
+
def normalize_status(data)
|
|
34
|
+
result = {'bytes_transferred' => data['bytes_transferred'].to_i}
|
|
35
|
+
case data['status']
|
|
36
|
+
when 'completed'
|
|
37
|
+
result['status'] = 'completed'
|
|
38
|
+
result['ended_at'] = Time.now.utc.iso8601
|
|
39
|
+
when 'failed'
|
|
40
|
+
result.merge!('status' => 'failed', 'ended_at' => Time.now.utc.iso8601, 'error' => data['error_desc'].to_s)
|
|
41
|
+
else
|
|
42
|
+
result['status'] = 'running'
|
|
43
|
+
end
|
|
44
|
+
result
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
16
48
|
# @param url [String] the base url of the node api
|
|
17
49
|
# @param username [String] the username to use for the node api
|
|
18
50
|
# @param password [String] the password to use for the node api
|
|
@@ -21,6 +21,37 @@ module Aspera
|
|
|
21
21
|
|
|
22
22
|
private_constant :LOCAL_SOCKET_ADDR, :PORT_SEP, :AUTO_LOCAL_TCP_PORT
|
|
23
23
|
|
|
24
|
+
class << self
|
|
25
|
+
# Re-query a previously started transfer from a running transferd daemon.
|
|
26
|
+
# agent_params must contain 'url' (the daemon endpoint, e.g. "127.0.0.1:12345").
|
|
27
|
+
# @return [Hash] normalized status hash
|
|
28
|
+
def transfer_status(transfer_id, agent_params)
|
|
29
|
+
client = ::Transferd::Api::TransferService::Stub.new(agent_params['url'], :this_channel_is_insecure)
|
|
30
|
+
result = {'bytes_transferred' => 0}
|
|
31
|
+
client.monitor_transfers(::Transferd::Api::RegistrationRequest.new(transferId: [transfer_id])) do |response|
|
|
32
|
+
result = case response.status
|
|
33
|
+
when :COMPLETED
|
|
34
|
+
{
|
|
35
|
+
'status' => 'completed', 'ended_at' => Time.now.utc.iso8601,
|
|
36
|
+
'bytes_transferred' => response.transferInfo.bytesTransferred
|
|
37
|
+
}
|
|
38
|
+
when :FAILED, :CANCELED
|
|
39
|
+
{
|
|
40
|
+
'status' => response.status == :FAILED ? 'failed' : 'cancelled',
|
|
41
|
+
'ended_at' => Time.now.utc.iso8601, 'bytes_transferred' => 0,
|
|
42
|
+
'error' => JSON.parse(response.message.to_s)['Description']
|
|
43
|
+
}
|
|
44
|
+
else
|
|
45
|
+
{'status' => 'running', 'bytes_transferred' => response.transferInfo.bytesTransferred}
|
|
46
|
+
end
|
|
47
|
+
break
|
|
48
|
+
end
|
|
49
|
+
result
|
|
50
|
+
rescue StandardError => e
|
|
51
|
+
{'status' => 'running', 'bytes_transferred' => 0, 'error' => e.message}
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
24
55
|
# @param url [String] URL of the transfer manager daemon
|
|
25
56
|
# @param start [Boolean] If `false`, expect that an external daemon is already running
|
|
26
57
|
# @param stop [Boolean] If `false`, do not shutdown daemon on exit
|
|
@@ -35,9 +66,10 @@ module Aspera
|
|
|
35
66
|
@transfer_id = nil
|
|
36
67
|
@stop = stop
|
|
37
68
|
is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
|
|
38
|
-
|
|
69
|
+
Aspera.assert(!(is_local_auto_port && (!@stop || !start)), type: Error){'Cannot set options `stop` or `start` to false with port zero'}
|
|
39
70
|
# keep PID for optional shutdown
|
|
40
71
|
@daemon_pid = nil
|
|
72
|
+
@daemon_endpoint = nil
|
|
41
73
|
daemon_endpoint = url
|
|
42
74
|
Log.dump(:daemon_endpoint, daemon_endpoint)
|
|
43
75
|
# retry loop
|
|
@@ -48,6 +80,7 @@ module Aspera
|
|
|
48
80
|
@transfer_client = ::Transferd::Api::TransferService::Stub.new(daemon_endpoint, :this_channel_is_insecure)
|
|
49
81
|
# Initiate actual connection
|
|
50
82
|
get_info_response = @transfer_client.get_info(::Transferd::Api::InstanceInfoRequest.new)
|
|
83
|
+
@daemon_endpoint = daemon_endpoint
|
|
51
84
|
Log.log.debug{"Daemon info: #{get_info_response}"}
|
|
52
85
|
Log.log.warn('Attached to existing daemon') unless @daemon_pid || !start || !@stop
|
|
53
86
|
at_exit{shutdown}
|
|
@@ -55,7 +88,7 @@ module Aspera
|
|
|
55
88
|
# if transferd is external: do not start it, or other error
|
|
56
89
|
raise if !start || !e.message.include?('failed to connect')
|
|
57
90
|
# we already tried to start a daemon, but it failed
|
|
58
|
-
Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}
|
|
91
|
+
Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}"}
|
|
59
92
|
Log.log.warn('no daemon present, starting daemon...') if !start
|
|
60
93
|
# transferd only supports local ip and port
|
|
61
94
|
daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
|
|
@@ -105,6 +138,9 @@ module Aspera
|
|
|
105
138
|
end
|
|
106
139
|
end
|
|
107
140
|
|
|
141
|
+
# Actual endpoint the daemon is listening on (resolved after connect, e.g. when port 0 was used)
|
|
142
|
+
attr_reader :daemon_endpoint
|
|
143
|
+
|
|
108
144
|
# :reek:UnusedParameters token_regenerator
|
|
109
145
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
|
110
146
|
Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
|
|
@@ -116,7 +152,7 @@ module Aspera
|
|
|
116
152
|
) # transfer definition
|
|
117
153
|
# send start transfer request to the transfer manager daemon
|
|
118
154
|
start_response = @transfer_client.start_transfer(transfer_request)
|
|
119
|
-
|
|
155
|
+
Aspera.assert(!start_response.status.eql?(:FAILED), type: Transfer::Error){start_response.error.description}
|
|
120
156
|
Log.log.debug{"start transfer response #{start_response}"}
|
|
121
157
|
@transfer_id = start_response.transferId
|
|
122
158
|
Log.log.debug{"transfer started with id #{@transfer_id}"}
|
data/lib/aspera/api/alee.rb
CHANGED
data/lib/aspera/api/aoc.rb
CHANGED
|
@@ -25,23 +25,23 @@ module Aspera
|
|
|
25
25
|
|
|
26
26
|
class AppInfo
|
|
27
27
|
# @return [Aspera::Api::AoC] Rest Api object
|
|
28
|
-
attr_reader
|
|
28
|
+
attr_reader :api
|
|
29
29
|
# @return [Hash, nil] Package information, `"files"` or `"packages"`
|
|
30
|
-
attr_reader
|
|
30
|
+
attr_reader :package
|
|
31
31
|
# @return [Hash] Result of GET /nodes/:id
|
|
32
|
-
attr_reader
|
|
32
|
+
attr_reader :node_info
|
|
33
33
|
# @return [String] Workspace id
|
|
34
|
-
attr_reader
|
|
34
|
+
attr_reader :workspace_id
|
|
35
35
|
# @return [String] Workspace name
|
|
36
|
-
attr_reader
|
|
36
|
+
attr_reader :workspace_name
|
|
37
37
|
# @return [String, nil] "Share as" link name
|
|
38
38
|
attr_accessor :opt_link_name
|
|
39
39
|
|
|
40
40
|
# @param api [Api::Aoc] Rest Api object
|
|
41
41
|
# @param package [Hash,nil] package information or nil
|
|
42
42
|
# @param node_info [Hash] Result of GET /nodes/:id
|
|
43
|
-
# @param workspace_id
|
|
44
|
-
# @param workspace_name [String]
|
|
43
|
+
# @param workspace_id [String] Workspace identifier
|
|
44
|
+
# @param workspace_name [String] Workspace name
|
|
45
45
|
def initialize(api, package, node_info, workspace_id, workspace_name)
|
|
46
46
|
@api = api
|
|
47
47
|
@package = package
|
|
@@ -68,8 +68,8 @@ module Aspera
|
|
|
68
68
|
GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{ |i| i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
|
|
69
69
|
# cookie prefix so that console can decode identity
|
|
70
70
|
COOKIE_PREFIX_CONSOLE_AOC = 'aspera.aoc'
|
|
71
|
-
# path in URL of public links
|
|
72
|
-
PUBLIC_LINK_PATHS = %w[/packages/public/receive /packages/public/send /files/public /public/files /public/package /public/send].freeze
|
|
71
|
+
# path in URL of public links (warning if not in list)
|
|
72
|
+
PUBLIC_LINK_PATHS = %w[/packages/public/receive /packages/public/send /files/public /public/files /public/files/ /public/package /public/send /public/send/package].freeze
|
|
73
73
|
JWT_AUDIENCE = 'https://api.asperafiles.com/api/v1/oauth2/token'
|
|
74
74
|
OAUTH_API_SUBPATH = 'api/v1/oauth2'
|
|
75
75
|
# minimum fields for user info if retrieval fails
|
|
@@ -269,7 +269,6 @@ module Aspera
|
|
|
269
269
|
attr_accessor :ws_ids
|
|
270
270
|
|
|
271
271
|
# By default: no workspace
|
|
272
|
-
# @param
|
|
273
272
|
def initialize(
|
|
274
273
|
scope: nil,
|
|
275
274
|
subpath: API_V1,
|
|
@@ -298,8 +297,7 @@ module Aspera
|
|
|
298
297
|
# default values for client id
|
|
299
298
|
client_id, client_secret = self.class.get_client_info if client_id.nil?
|
|
300
299
|
# access key secrets are provided out of band to get node api access
|
|
301
|
-
#
|
|
302
|
-
# value: associated secret
|
|
300
|
+
# maps access key (String) to its associated secret (String)
|
|
303
301
|
@secret_finder = secret_finder
|
|
304
302
|
@cache_user_info = nil
|
|
305
303
|
@cache_url_token_info = nil
|
|
@@ -480,14 +478,14 @@ module Aspera
|
|
|
480
478
|
# @param workspace_name [String,nil] workspace name
|
|
481
479
|
# @param scope [String,nil] e.g. Node::Scope::USER, or Node::Scope::ADMIN, or nil (requires secret)
|
|
482
480
|
# @param package_info [Hash,nil] created package information
|
|
483
|
-
# @
|
|
481
|
+
# @return [Node] a node API for access key
|
|
484
482
|
def node_api_from(node_id:, workspace_id: nil, workspace_name: nil, scope: Node::Scope::USER, package_info: nil)
|
|
485
483
|
Aspera.assert_type(node_id, String)
|
|
486
484
|
node_info = read("nodes/#{node_id}")
|
|
487
485
|
workspace_name = read("workspaces/#{workspace_id}")['name'] if workspace_name.nil? && !workspace_id.nil?
|
|
488
486
|
app_info = AppInfo.new(self, package_info, node_info, workspace_id, workspace_name)
|
|
489
487
|
node_params = {base_url: node_info['url']}
|
|
490
|
-
ak_secret = @secret_finder&.
|
|
488
|
+
ak_secret = @secret_finder&.lookup(url: node_info['url'], username: node_info['access_key'])
|
|
491
489
|
# If secret is available, or no scope, use basic auth
|
|
492
490
|
if scope.nil? || ak_secret
|
|
493
491
|
Aspera.assert(ak_secret, type: Error){"Secret not found for access key #{node_info['access_key']}@#{node_info['url']}"}
|
|
@@ -545,7 +543,7 @@ module Aspera
|
|
|
545
543
|
# @param package_data [Hash] The whole package creation payload
|
|
546
544
|
# @param rcpt_lst_field [String] The field in structure, i.e. recipients or bcc_recipients
|
|
547
545
|
# @param new_user_option [Hash] Additional fields for contact creation
|
|
548
|
-
# @return nil
|
|
546
|
+
# @return [nil] `package_data` is modified in place
|
|
549
547
|
def resolve_package_recipients(package_data, rcpt_lst_field, new_user_option)
|
|
550
548
|
return unless package_data.key?(rcpt_lst_field)
|
|
551
549
|
Aspera.assert_type(package_data[rcpt_lst_field], Array){rcpt_lst_field}
|
|
@@ -609,7 +607,7 @@ module Aspera
|
|
|
609
607
|
# @param package_data [Hash] package creation (with extensions...)
|
|
610
608
|
# @param validate_meta [TrueClass,FalseClass] true to validate parameters locally
|
|
611
609
|
# @param new_user_option [Hash,NilClass] options if an unknown user is specified
|
|
612
|
-
# @return transfer spec, node api and package information
|
|
610
|
+
# @return [Array] transfer spec, node api and package information
|
|
613
611
|
def create_package_simple(package_data, validate_meta, new_user_option)
|
|
614
612
|
update_package_metadata_for_api(package_data)
|
|
615
613
|
# list of files to include in package, optional
|
data/lib/aspera/api/cos_node.rb
CHANGED
|
@@ -24,7 +24,7 @@ module Aspera
|
|
|
24
24
|
endpoints = Aspera::Rest.new(base_url: service_credentials['endpoints']).read('')
|
|
25
25
|
Log.dump(:endpoints, endpoints)
|
|
26
26
|
endpoint = endpoints.dig('service-endpoints', 'regional', bucket_region, 'public', bucket_region)
|
|
27
|
-
|
|
27
|
+
Aspera.assert(!endpoint.nil?){"no such region: #{bucket_region}"}
|
|
28
28
|
return {
|
|
29
29
|
instance_id: service_credentials['resource_instance_id'],
|
|
30
30
|
api_key: service_credentials['apikey'],
|
data/lib/aspera/api/faspex.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'aspera/assert'
|
|
3
4
|
require 'aspera/rest'
|
|
4
5
|
require 'aspera/oauth/base'
|
|
5
6
|
require 'aspera/rest_list'
|
|
@@ -11,9 +12,9 @@ module Aspera
|
|
|
11
12
|
class << self
|
|
12
13
|
attr_accessor :additional_info
|
|
13
14
|
end
|
|
14
|
-
# @param context
|
|
15
|
-
# @param redirect_uri
|
|
16
|
-
# @param path_authorize
|
|
15
|
+
# @param context [String] The `context` query parameter in public link
|
|
16
|
+
# @param redirect_uri [String] URI of web UI login
|
|
17
|
+
# @param path_authorize [String] Path to provide passcode
|
|
17
18
|
def initialize(
|
|
18
19
|
context:,
|
|
19
20
|
redirect_uri:,
|
|
@@ -125,7 +126,7 @@ module Aspera
|
|
|
125
126
|
mfa_code
|
|
126
127
|
]
|
|
127
128
|
class << self
|
|
128
|
-
# @return true if the URL is a public link
|
|
129
|
+
# @return [Boolean] true if the URL is a public link
|
|
129
130
|
def public_link?(url)
|
|
130
131
|
url.include?('?context=')
|
|
131
132
|
end
|
|
@@ -138,15 +139,15 @@ module Aspera
|
|
|
138
139
|
end
|
|
139
140
|
attr_reader :pub_link_context
|
|
140
141
|
|
|
141
|
-
# @param url
|
|
142
|
-
# @param auth
|
|
143
|
-
# @param password
|
|
144
|
-
# @param client_id
|
|
145
|
-
# @param client_secret
|
|
146
|
-
# @param redirect_uri
|
|
147
|
-
# @param username
|
|
148
|
-
# @param private_key
|
|
149
|
-
# @param passphrase
|
|
142
|
+
# @param url [String] Faspex URL, can be a public link
|
|
143
|
+
# @param auth [Symbol] Authentication method: :boot (token in header), :web (open browser), :jwt (client_id + private key), :public_link (context in URL)
|
|
144
|
+
# @param password [String, nil] For :boot auth, the token copied directly from browser in developer mode
|
|
145
|
+
# @param client_id [String, nil] For :web and :jwt auth, the client_id of web UI application
|
|
146
|
+
# @param client_secret [String, nil] For :web auth, the client_secret of web UI application (not needed for :jwt)
|
|
147
|
+
# @param redirect_uri [String, nil] For :web auth, the redirect_uri of web UI application (must be the same as in application configuration)
|
|
148
|
+
# @param username [String, nil] For :jwt auth, the username of the user to impersonate
|
|
149
|
+
# @param private_key [String, nil] For :jwt auth, the private key to sign JWT token
|
|
150
|
+
# @param passphrase [String, nil] For :jwt auth, the passphrase of the private key
|
|
150
151
|
def initialize(
|
|
151
152
|
url:,
|
|
152
153
|
auth:,
|
|
@@ -169,7 +170,7 @@ module Aspera
|
|
|
169
170
|
Log.dump(:final_url, final_url, level: :trace1)
|
|
170
171
|
# Get context from query
|
|
171
172
|
encoded_context = Rest.query_to_h(URI.parse(final_url).query)['context']
|
|
172
|
-
|
|
173
|
+
Aspera.assert(!encoded_context.nil?, type: ParameterError){'Bad faspex5 public link, missing context in query'}
|
|
173
174
|
# public link information (contains passcode and allowed usage)
|
|
174
175
|
@pub_link_context = JSON.parse(Base64.decode64(encoded_context))
|
|
175
176
|
Log.dump(:pub_link_context, @pub_link_context, level: :trace1)
|
data/lib/aspera/api/httpgw.rb
CHANGED
|
@@ -81,7 +81,7 @@ module Aspera
|
|
|
81
81
|
end
|
|
82
82
|
|
|
83
83
|
# Check header ourself and give precise error message, as websocket will only throw error without details
|
|
84
|
-
# @param [String] Response Header
|
|
84
|
+
# @param header [String] Response Header
|
|
85
85
|
# @return [String] Response Header
|
|
86
86
|
def validated_ws_response_header(header)
|
|
87
87
|
first_line = header.split("\r\n").first
|
|
@@ -210,7 +210,7 @@ module Aspera
|
|
|
210
210
|
fileIndex: file_index
|
|
211
211
|
}
|
|
212
212
|
file = file_to_send[:file]
|
|
213
|
-
file = File.open(file) unless file.is_a?(Transfer::FauxFile)
|
|
213
|
+
file = File.open(file) unless file.is_a?(Transfer::FauxFile) # rubocop:disable Style/FileOpen
|
|
214
214
|
begin
|
|
215
215
|
until file.eof?
|
|
216
216
|
slice_bin_data = file.read(@upload_chunk_size)
|
|
@@ -291,7 +291,7 @@ module Aspera
|
|
|
291
291
|
return @api_info
|
|
292
292
|
end
|
|
293
293
|
|
|
294
|
-
# @return the base url of the gateway
|
|
294
|
+
# @return [String] the base url of the gateway
|
|
295
295
|
def base_url
|
|
296
296
|
return @gw_root_url
|
|
297
297
|
end
|
|
@@ -308,7 +308,7 @@ module Aspera
|
|
|
308
308
|
Log.dump(:gw_url, url)
|
|
309
309
|
# add scheme if missing
|
|
310
310
|
url = "https://#{url}" unless url.match?(%r{^[a-z]{1,6}://})
|
|
311
|
-
|
|
311
|
+
Aspera.assert(url.start_with?('https://'), type: Error){'GW URL shall be with scheme https'}
|
|
312
312
|
# remove trailing slash and version (o=only once) if present
|
|
313
313
|
# TODO: issue warning ?
|
|
314
314
|
url = url.chomp('/').gsub(%r{/#{API_V1}$}o, '')
|
data/lib/aspera/api/node.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'aspera/link_header'
|
|
3
4
|
require 'aspera/transfer/spec'
|
|
4
5
|
require 'aspera/rest'
|
|
5
6
|
require 'aspera/oauth'
|
|
@@ -78,12 +79,12 @@ module Aspera
|
|
|
78
79
|
attr_reader :api_options
|
|
79
80
|
attr_reader :use_dynamic_key
|
|
80
81
|
|
|
81
|
-
def api_options=(
|
|
82
|
-
Aspera.assert_type(
|
|
83
|
-
|
|
84
|
-
Aspera.assert(@api_options.key?(
|
|
85
|
-
Aspera.assert_type(
|
|
86
|
-
@api_options[
|
|
82
|
+
def api_options=(options_hash)
|
|
83
|
+
Aspera.assert_type(options_hash, Hash)
|
|
84
|
+
options_hash.each do |key, val|
|
|
85
|
+
Aspera.assert(@api_options.key?(key.to_sym)){"unknown api option: #{key} (#{OPTIONS.join(', ')})"}
|
|
86
|
+
Aspera.assert_type(val, TrueClass, FalseClass){"api options value for #{key} should be boolean"}
|
|
87
|
+
@api_options[key.to_sym] = val
|
|
87
88
|
end
|
|
88
89
|
end
|
|
89
90
|
|
|
@@ -105,30 +106,30 @@ module Aspera
|
|
|
105
106
|
end
|
|
106
107
|
|
|
107
108
|
# Adds fields `public_keys` in provided Hash, if dynamic key is set.
|
|
108
|
-
# @param
|
|
109
|
+
# @param parameters [Hash] Hash to add public key to
|
|
109
110
|
# @return [Hash] Hash with public key added
|
|
110
|
-
def add_public_key(
|
|
111
|
+
def add_public_key(parameters)
|
|
111
112
|
if @dynamic_key
|
|
112
113
|
ssh_key = Net::SSH::Buffer.from(:key, @dynamic_key)
|
|
113
114
|
# Get pub key in OpenSSH public key format (authorized_keys)
|
|
114
|
-
|
|
115
|
+
parameters['public_keys'] = [
|
|
115
116
|
ssh_key.read_string,
|
|
116
117
|
Base64.strict_encode64(ssh_key.to_s)
|
|
117
118
|
].join(' ')
|
|
118
119
|
end
|
|
119
|
-
return
|
|
120
|
+
return parameters
|
|
120
121
|
end
|
|
121
122
|
|
|
122
123
|
# Adds fields `ssh_private_key` in provided Hash, if dynamic key is set.
|
|
123
|
-
# @param
|
|
124
|
-
def add_private_key(
|
|
125
|
-
|
|
126
|
-
return
|
|
124
|
+
# @param parameters [Hash] Hash to add private key to
|
|
125
|
+
def add_private_key(parameters)
|
|
126
|
+
parameters['ssh_private_key'] = @dynamic_key.to_pem if @dynamic_key
|
|
127
|
+
return parameters
|
|
127
128
|
end
|
|
128
129
|
|
|
129
130
|
# For access keys: provide expression to match entry in folder
|
|
130
|
-
# @param match_expression one of supported types
|
|
131
|
-
# @return lambda function
|
|
131
|
+
# @param match_expression [Proc, Regexp, String] one of supported types
|
|
132
|
+
# @return [Proc] lambda function
|
|
132
133
|
def file_matcher(match_expression)
|
|
133
134
|
case match_expression
|
|
134
135
|
when Proc then return match_expression
|
|
@@ -146,7 +147,8 @@ module Aspera
|
|
|
146
147
|
end
|
|
147
148
|
|
|
148
149
|
# Split path into folder + filename
|
|
149
|
-
# @
|
|
150
|
+
# @param path [String] file path to split
|
|
151
|
+
# @return [Array(String, String)] containing folder + inside folder/file
|
|
150
152
|
def split_folder(path)
|
|
151
153
|
folder = path.split(PATH_SEPARATOR)
|
|
152
154
|
inside = folder.pop
|
|
@@ -160,7 +162,8 @@ module Aspera
|
|
|
160
162
|
end
|
|
161
163
|
|
|
162
164
|
# Decode node scope into access key and scope
|
|
163
|
-
# @
|
|
165
|
+
# @param scope [String] scope string to decode
|
|
166
|
+
# @return [Hash] decoded scope with access key and scope parts
|
|
164
167
|
def decode_scope(scope)
|
|
165
168
|
items = scope.split(Scope::SEPARATOR, 2)
|
|
166
169
|
Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
|
|
@@ -240,7 +243,7 @@ module Aspera
|
|
|
240
243
|
return tspec
|
|
241
244
|
end
|
|
242
245
|
|
|
243
|
-
# @
|
|
246
|
+
# @return [Node, nil] a Node Api object or nil if no App defined
|
|
244
247
|
def node_id_to_node(node_id)
|
|
245
248
|
if !@app_info.nil?
|
|
246
249
|
return self if node_id.eql?(@app_info.node_info['id'])
|
|
@@ -390,7 +393,7 @@ module Aspera
|
|
|
390
393
|
return NodeFileId.new(self, top_file_id) if path_elements.empty?
|
|
391
394
|
resolve_state = {path: path_elements, consumed: [], result: nil, process_last_link: process_last_link}
|
|
392
395
|
process_folder_tree(method_sym: :process_api_fid, state: resolve_state, top_file_id: top_file_id)
|
|
393
|
-
|
|
396
|
+
Aspera.assert(!resolve_state[:result].nil?, type: ParameterError){"Entry not found: #{resolve_state[:path].first} in /#{resolve_state[:consumed].join(PATH_SEPARATOR)}"}
|
|
394
397
|
Log.log.debug{"resolve_api_fid: #{path} -> #{resolve_state[:result].node_api.base_url} #{resolve_state[:result].file_id}"}
|
|
395
398
|
return resolve_state[:result]
|
|
396
399
|
end
|
|
@@ -572,7 +575,7 @@ module Aspera
|
|
|
572
575
|
# Update progress spinner
|
|
573
576
|
RestParameters.instance.spinner_cb.call(item_list.length)
|
|
574
577
|
# Parse Link header according to RFC 8288 to extract next iteration token
|
|
575
|
-
next_url =
|
|
578
|
+
next_url = LinkHeader.parse(http['Link']).find_href(rel: 'next')
|
|
576
579
|
next_iteration_token = nil
|
|
577
580
|
if next_url
|
|
578
581
|
begin
|
|
@@ -629,7 +632,7 @@ module Aspera
|
|
|
629
632
|
private
|
|
630
633
|
|
|
631
634
|
# Method called in loop for each entry for `resolve_api_fid`
|
|
632
|
-
# @return `true` to continue digging, `false` to stop processing: set state[:result] if found
|
|
635
|
+
# @return [Boolean] `true` to continue digging, `false` to stop processing: set state[:result] if found
|
|
633
636
|
def process_api_fid(entry, path, state)
|
|
634
637
|
# Stop digging here if not in right path
|
|
635
638
|
return false unless entry['name'].eql?(state[:path].first)
|
|
@@ -656,7 +659,7 @@ module Aspera
|
|
|
656
659
|
# We found it
|
|
657
660
|
other_node = nil
|
|
658
661
|
other_node = node_id_to_node(entry['target_node_id']) if entry_has_link_information(entry)
|
|
659
|
-
|
|
662
|
+
Aspera.assert(!other_node.nil?, type: Error){'Cannot resolve link'}
|
|
660
663
|
state[:result] = NodeFileId.new(other_node, entry['target_id'])
|
|
661
664
|
else
|
|
662
665
|
# We found it but we do not process the link
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
query ($organization_id: ID!) {
|
|
2
|
+
aoc (organization_id: $organization_id) {
|
|
3
|
+
bssSubscription {
|
|
4
|
+
aocVersion endDate startDate termMonths plan trial termType
|
|
5
|
+
aocOrganizations { id }
|
|
6
|
+
additionalStorageVolumeGb additionalEgressVolumeGb
|
|
7
|
+
term { startDate endDate transferVolumeGb egressVolumeGb storageVolumeGb transferVolumeOffsetGb }
|
|
8
|
+
paygoRate { transferRate storageRate currency }
|
|
9
|
+
aocPlanData {
|
|
10
|
+
tier trial
|
|
11
|
+
workspaces { max }
|
|
12
|
+
users { planAmount max }
|
|
13
|
+
samlIntegration activity sharedInboxes uniqueUrls support watermarking byok
|
|
14
|
+
automation { planAmount, max }
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
query ($organization_id: ID!, $startDate: Date!, $endDate: Date!, $aggregate: TransferUsageAggregateOption!) {
|
|
2
|
+
aoc (organization_id: $organization_id) {
|
|
3
|
+
bssSubscription {
|
|
4
|
+
aocOrganizations { id }
|
|
5
|
+
additionalStorageVolumeGb additionalEgressVolumeGb
|
|
6
|
+
aocPlanData { tier trial }
|
|
7
|
+
term { transferVolumeGb egressVolumeGb storageVolumeGb startDate endDate transferVolumeOffsetGb }
|
|
8
|
+
termMonths
|
|
9
|
+
transferUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { mbTotal }
|
|
10
|
+
egressUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { usageMb }
|
|
11
|
+
}
|
|
12
|
+
subscriptionEntitlements {
|
|
13
|
+
id
|
|
14
|
+
transferUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { mbTotal }
|
|
15
|
+
egressUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { usageMb }
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
data/lib/aspera/ascmd.rb
CHANGED
|
@@ -28,18 +28,18 @@ module Aspera
|
|
|
28
28
|
ENUM_START = 1
|
|
29
29
|
|
|
30
30
|
# Description of result structures (see ascmdtypes.h).
|
|
31
|
-
# Base types are big endian
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
# :
|
|
42
|
-
# rubocop:disable Layout/FirstHashElementLineBreak
|
|
31
|
+
# Base types are big endian.
|
|
32
|
+
# Keys are type names; the index in each `fields` array equals (TLV numerical type - `ENUM_START`).
|
|
33
|
+
# Decoding always starts at `result`.
|
|
34
|
+
# Some fields have special handling indicated by `special`:
|
|
35
|
+
# - `:field_list` : a Hash or Array of child TLVs
|
|
36
|
+
# - `:list_tlv_list` : a list of TLV sub-lists (Array of Hash)
|
|
37
|
+
# - `:list_tlv_restart` : same but restarted per entry
|
|
38
|
+
# `decode` values:
|
|
39
|
+
# - `:base` : a single scalar value (unpack with `unpack`)
|
|
40
|
+
# - `:buffer_list` : an Array of `{btype, buffer}` entries
|
|
41
|
+
# - `:field_list` : a Hash or Array built from child TLVs
|
|
42
|
+
# rubocop:disable-next Layout/FirstHashElementLineBreak
|
|
43
43
|
TYPES_DESCR = {
|
|
44
44
|
result: {decode: :field_list,
|
|
45
45
|
fields: [{name: :file, is_a: :stat}, {name: :dir, is_a: :stat, special: :list_tlv_list}, {name: :size, is_a: :size}, {name: :error, is_a: :error},
|
|
@@ -73,7 +73,6 @@ module Aspera
|
|
|
73
73
|
zstr: {decode: :base, unpack: 'Z*'},
|
|
74
74
|
blist: {decode: :buffer_list}
|
|
75
75
|
}.freeze
|
|
76
|
-
# rubocop:enable Layout/FirstHashElementLineBreak
|
|
77
76
|
|
|
78
77
|
private_constant :TYPES_DESCR, :ENUM_START, :OPS_ARGS
|
|
79
78
|
|
|
@@ -87,8 +86,8 @@ module Aspera
|
|
|
87
86
|
|
|
88
87
|
# execute an "as" command on a remote server
|
|
89
88
|
# Version 2 allows use of reverse proxy with multiple addresses.
|
|
90
|
-
# @param [Symbol] one of OPERATIONS
|
|
91
|
-
# @param [Array]
|
|
89
|
+
# @param action_sym [Symbol] one of OPERATIONS
|
|
90
|
+
# @param arguments [Array] parameters for "as" command
|
|
92
91
|
# @return [Boolean,Array,Hash] result of command, type depends on command
|
|
93
92
|
def execute_single(action_sym, arguments, version: 1, host: nil)
|
|
94
93
|
arguments = [] if arguments.nil?
|
|
@@ -169,19 +168,22 @@ module Aspera
|
|
|
169
168
|
end
|
|
170
169
|
|
|
171
170
|
class << self
|
|
172
|
-
# Get description of structure's field
|
|
171
|
+
# Get description of structure's field
|
|
172
|
+
# @param struct_name [Symbol] name of the structure type
|
|
173
|
+
# @param typed_buffer [Hash] typed buffer providing the field name via `:btype`
|
|
174
|
+
# @return [Hash] field description
|
|
173
175
|
def field_description(struct_name, typed_buffer)
|
|
174
176
|
result = TYPES_DESCR[struct_name][:fields][typed_buffer[:btype] - ENUM_START]
|
|
175
|
-
|
|
177
|
+
Aspera.assert(!result.nil?){"Unrecognized field for #{struct_name}: #{typed_buffer[:btype]}\n#{typed_buffer[:buffer]}"}
|
|
176
178
|
return result
|
|
177
179
|
end
|
|
178
180
|
|
|
179
181
|
# Decodes the provided buffer as provided type name
|
|
180
|
-
# @return a decoded type: single, Array, or Hash
|
|
182
|
+
# @return [Boolean, Array, Hash] a decoded type: single, Array, or Hash
|
|
181
183
|
def parse(buffer, type_name, indent_level = nil)
|
|
182
184
|
indent_level = (indent_level || -1) + 1
|
|
183
185
|
type_descr = TYPES_DESCR[type_name]
|
|
184
|
-
|
|
186
|
+
Aspera.assert(!type_descr.nil?){"Unexpected type #{type_name}"}
|
|
185
187
|
Log.log.trace1{"#{' .' * indent_level}parse:#{type_name}:#{type_descr[:decode]}:#{buffer[0, 16]}...".red}
|
|
186
188
|
result = nil
|
|
187
189
|
case type_descr[:decode]
|