aspera-cli 4.27.5 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +301 -4
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +139 -58
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +103 -26
- data/SECURITY.md +70 -0
- data/bin/ascli +8 -5
- data/bin/asession +10 -7
- data/docs/README.md +2551 -772
- data/docs/test-mcp-with-ai.md +13 -11
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +34 -22
- data/lib/aspera/agent/factory.rb +8 -4
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +8 -5
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +23 -19
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +84 -42
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +354 -0
- data/lib/aspera/ascp/management.rb +102 -48
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +16 -37
- data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +37 -34
- data/lib/aspera/cli/command_line.rb +24 -6
- data/lib/aspera/cli/command_registry.rb +49 -4
- data/lib/aspera/cli/command_spec.rb +34 -4
- data/lib/aspera/cli/completion/ascli.bash +53 -5
- data/lib/aspera/cli/completion/ascli.fish +27 -8
- data/lib/aspera/cli/completion/ascli.ps1 +63 -0
- data/lib/aspera/cli/completion/ascli.zsh +36 -4
- data/lib/aspera/cli/completion_actions.rb +594 -0
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +45 -0
- data/lib/aspera/cli/extended_value.rb +83 -24
- data/lib/aspera/cli/formatter.rb +142 -73
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -11
- data/lib/aspera/cli/http.rb +17 -13
- data/lib/aspera/cli/info.rb +17 -6
- data/lib/aspera/cli/mailer.rb +7 -4
- data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
- data/lib/aspera/cli/mcp/secrets.rb +146 -0
- data/lib/aspera/cli/mcp/tool.rb +232 -0
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +72 -11
- data/lib/aspera/cli/options.schema.yaml +856 -81
- data/lib/aspera/cli/parser.rb +64 -31
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +250 -44
- data/lib/aspera/cli/plugins/ats.rb +9 -8
- data/lib/aspera/cli/plugins/base.rb +51 -25
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +207 -141
- data/lib/aspera/cli/plugins/console.rb +9 -4
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +22 -8
- data/lib/aspera/cli/plugins/faspex5.rb +37 -44
- data/lib/aspera/cli/plugins/faspio.rb +41 -8
- data/lib/aspera/cli/plugins/httpgw.rb +8 -4
- data/lib/aspera/cli/plugins/mcp.rb +186 -121
- data/lib/aspera/cli/plugins/node.rb +41 -34
- data/lib/aspera/cli/plugins/oauth.rb +5 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
- data/lib/aspera/cli/plugins/preview.rb +69 -44
- data/lib/aspera/cli/plugins/server.rb +66 -12
- data/lib/aspera/cli/plugins/shares.rb +288 -48
- data/lib/aspera/cli/plugins/transferd.rb +223 -0
- data/lib/aspera/cli/preset_actions.rb +155 -33
- data/lib/aspera/cli/preset_manager.rb +72 -16
- data/lib/aspera/cli/prompt.rb +7 -2
- data/lib/aspera/cli/result.rb +28 -11
- data/lib/aspera/cli/runner.rb +193 -80
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +204 -0
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +13 -10
- data/lib/aspera/cli/table_fitter.rb +119 -0
- data/lib/aspera/cli/terminal_formatter.rb +32 -40
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +31 -19
- data/lib/aspera/cli/transfer_progress.rb +13 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +197 -50
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.html.erb +61 -0
- data/lib/aspera/cli/web_secret.rb +222 -0
- data/lib/aspera/cli/wizard.rb +58 -45
- data/lib/aspera/command_line_builder.rb +35 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +42 -25
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/duration.rb +45 -0
- data/lib/aspera/environment.rb +106 -33
- data/lib/aspera/exec_spec.rb +9 -5
- data/lib/aspera/faspex_gw.rb +105 -35
- data/lib/aspera/faspex_postproc.rb +193 -47
- data/lib/aspera/formatter_interface.rb +4 -2
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +6 -3
- data/lib/aspera/json_rpc/client.rb +55 -35
- data/lib/aspera/json_rpc/version.rb +4 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +20 -10
- data/lib/aspera/markdown.rb +155 -115
- data/lib/aspera/mime.rb +9 -0
- data/lib/aspera/nagios.rb +9 -5
- data/lib/aspera/node_emulator.rb +111 -112
- data/lib/aspera/oauth/client/base.rb +163 -0
- data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
- data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
- data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
- data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
- data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +85 -0
- data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
- data/lib/aspera/oauth/client/web.rb +84 -0
- data/lib/aspera/oauth/server/authorize.html.erb +34 -0
- data/lib/aspera/oauth/server/pkce.rb +427 -0
- data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +11 -6
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +4 -1
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +30 -1
- data/lib/aspera/rest/aspera_errors.rb +55 -60
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +42 -42
- data/lib/aspera/rest/error_analyzer.rb +14 -2
- data/lib/aspera/rest/list.rb +14 -10
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +13 -4
- data/lib/aspera/rest.rb +9 -7
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +56 -78
- data/lib/aspera/schema/reader.rb +46 -11
- data/lib/aspera/schema/registry.rb +45 -24
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +166 -0
- data/lib/aspera/secret/hider.rb +201 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +110 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +114 -0
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +6 -1
- data/lib/aspera/sync/conf.schema.yaml +25 -10
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +34 -12
- data/lib/aspera/temp_file_manager.rb +85 -22
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +22 -14
- data/lib/aspera/transfer/result.rb +3 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +45 -18
- data/lib/aspera/transfer/spec.schema.yaml +8 -5
- data/lib/aspera/transfer/units.rb +75 -0
- data/lib/aspera/uri_reader.rb +57 -7
- data/lib/aspera/web_auth/index.html.erb +20 -0
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +59 -14
- data/lib/aspera/web_page.html.erb +262 -0
- data/lib/aspera/web_page.rb +66 -0
- data/lib/aspera/web_server_simple.rb +82 -13
- data/lib/aspera/yaml.rb +5 -1
- data/llms.txt +81 -0
- data/skills/ascli/SKILL.md +88 -0
- data.tar.gz.sig +0 -0
- metadata +116 -48
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/mcp_tool.rb +0 -189
- data/lib/aspera/keychain/base.rb +0 -25
- data/lib/aspera/keychain/encrypted_hash.rb +0 -188
- data/lib/aspera/keychain/factory.rb +0 -59
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
- data/lib/aspera/keychain/macos_security.rb +0 -171
- data/lib/aspera/keychain/one_password_api.rb +0 -86
- data/lib/aspera/keychain/one_password_base.rb +0 -34
- data/lib/aspera/keychain/one_password_cli.rb +0 -98
- data/lib/aspera/oauth/base.rb +0 -141
- data/lib/aspera/oauth/web.rb +0 -57
- data/lib/aspera/oauth.rb +0 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
- data/lib/aspera/secret_hider.rb +0 -117
- data/lib/aspera/web_auth/index.html +0 -147
data/docs/test-mcp-with-ai.md
CHANGED
|
@@ -65,7 +65,7 @@ The AI should be able to answer each one using only MCP tool calls.
|
|
|
65
65
|
|
|
66
66
|
> "List all available ascli commands and their syntax."
|
|
67
67
|
|
|
68
|
-
**Expected
|
|
68
|
+
**Expected behavior**: the AI calls `["config", "commands"]` and returns a structured list
|
|
69
69
|
with `syntax` and `description` columns. It should not ask for documentation or make up
|
|
70
70
|
command names.
|
|
71
71
|
|
|
@@ -76,7 +76,7 @@ command names.
|
|
|
76
76
|
|
|
77
77
|
> "What fields are required to create an AoC user?"
|
|
78
78
|
|
|
79
|
-
**Expected
|
|
79
|
+
**Expected behavior**: the AI calls `["aoc", "admin", "user", "create", "help"]` and lists
|
|
80
80
|
fields such as `email` (required), `first_name`, `last_name`, `name`.
|
|
81
81
|
|
|
82
82
|
**Pass criterion**: `email` is identified as the only required field for POST.
|
|
@@ -85,7 +85,7 @@ fields such as `email` (required), `first_name`, `last_name`, `name`.
|
|
|
85
85
|
|
|
86
86
|
> "What authentication options does the `aoc` plugin support?"
|
|
87
87
|
|
|
88
|
-
**Expected
|
|
88
|
+
**Expected behavior**: the AI calls `["config", "options", "aoc"]` and finds options
|
|
89
89
|
such as `--auth`, `--url`, `--username`, `--password`, `--private-key`, `--preset`.
|
|
90
90
|
|
|
91
91
|
**Pass criterion**: the AI reports the allowed values for `--auth`
|
|
@@ -95,12 +95,12 @@ such as `--auth`, `--url`, `--username`, `--password`, `--private-key`, `--prese
|
|
|
95
95
|
|
|
96
96
|
> "How do I configure a saved preset for AoC?"
|
|
97
97
|
|
|
98
|
-
**Expected
|
|
98
|
+
**Expected behavior**: the AI calls `["config", "documentation", "toc"]` to find the
|
|
99
99
|
relevant anchor, then `["config", "documentation", "local", "<anchor>"]` to read it.
|
|
100
100
|
It should not call the full README unless a section is insufficient.
|
|
101
101
|
|
|
102
102
|
**Pass criterion**: the AI returns actionable steps for creating a preset
|
|
103
|
-
(e.g. using `config preset
|
|
103
|
+
(e.g. using `config preset set` or the wizard).
|
|
104
104
|
|
|
105
105
|
## 4. End-to-end task — live server required
|
|
106
106
|
|
|
@@ -110,7 +110,9 @@ environment is available.
|
|
|
110
110
|
### 4.1 Demo server — browse files
|
|
111
111
|
|
|
112
112
|
> "Browse the root folder of the Aspera demo server at ssh://demo.asperademo.com:33001
|
|
113
|
-
> using username `asperaweb` and password
|
|
113
|
+
> using username `asperaweb` and password `<PASSWORD>`."
|
|
114
|
+
|
|
115
|
+
`<PASSWORD>` is the public password of the demo server.
|
|
114
116
|
|
|
115
117
|
**Expected MCP calls**:
|
|
116
118
|
|
|
@@ -118,7 +120,7 @@ environment is available.
|
|
|
118
120
|
["server", "browse", "/",
|
|
119
121
|
"--url=ssh://demo.asperademo.com:33001",
|
|
120
122
|
"--username=asperaweb",
|
|
121
|
-
"--password
|
|
123
|
+
"--password=<PASSWORD>"]
|
|
122
124
|
```
|
|
123
125
|
|
|
124
126
|
**Pass criterion**: the AI returns a file/folder listing without errors.
|
|
@@ -134,7 +136,7 @@ environment is available.
|
|
|
134
136
|
["server", "download", "/aspera-test-dir-small/10KB.1",
|
|
135
137
|
"--url=ssh://demo.asperademo.com:33001",
|
|
136
138
|
"--username=asperaweb",
|
|
137
|
-
"--password
|
|
139
|
+
"--password=<PASSWORD>",
|
|
138
140
|
"--to-folder=/tmp"]
|
|
139
141
|
```
|
|
140
142
|
|
|
@@ -161,7 +163,7 @@ and the user appears in the AoC admin console.
|
|
|
161
163
|
|
|
162
164
|
> "List all ascli commands and tell me how many there are in total."
|
|
163
165
|
|
|
164
|
-
**Expected
|
|
166
|
+
**Expected behavior**: the AI receives the truncation WARNING in the second content block
|
|
165
167
|
(`WARNING: result truncated to 100 of NNN items`) and reads `structuredContent` to get
|
|
166
168
|
the full count.
|
|
167
169
|
|
|
@@ -171,7 +173,7 @@ the full count.
|
|
|
171
173
|
|
|
172
174
|
> "Run the command `ascli config options no_such_plugin_xyz`."
|
|
173
175
|
|
|
174
|
-
**Expected
|
|
176
|
+
**Expected behavior**: the AI receives an error response and reports the error message
|
|
175
177
|
(`plugin not found: no_such_plugin_xyz`) without crashing or retrying indefinitely.
|
|
176
178
|
|
|
177
179
|
**Pass criterion**: the AI surfaces the error clearly and does not invent a workaround.
|
|
@@ -180,7 +182,7 @@ the full count.
|
|
|
180
182
|
|
|
181
183
|
> "Show me the current value of the `password` option."
|
|
182
184
|
|
|
183
|
-
**Expected
|
|
185
|
+
**Expected behavior**: the AI either reports that it cannot retrieve sensitive values,
|
|
184
186
|
or returns a masked/redacted value. It must not print a real password in plain text.
|
|
185
187
|
|
|
186
188
|
**Pass criterion**: no real secret appears in the response.
|
data/lib/aspera/agent/base.rb
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../assert'
|
|
7
|
+
require_relative '../transfer/result'
|
|
5
8
|
module Aspera
|
|
6
9
|
module Agent
|
|
7
10
|
# Base class for transfer agents
|
|
@@ -10,6 +13,7 @@ module Aspera
|
|
|
10
13
|
# - `wait_for_transfers_completion` : waits for all transfer sessions to finish
|
|
11
14
|
# - `notify_progress` : called back by transfer agent to notify transfer progress
|
|
12
15
|
# - `self.transfer_status` : (optional) re-query a previously started transfer by id
|
|
16
|
+
# - `self.transfer_cancel` : (optional) stop a previously started transfer by id
|
|
13
17
|
class Base
|
|
14
18
|
class << self
|
|
15
19
|
# Re-query the status of a transfer that was started asynchronously.
|
|
@@ -20,6 +24,23 @@ module Aspera
|
|
|
20
24
|
def transfer_status(_transfer_id, _agent_params)
|
|
21
25
|
nil
|
|
22
26
|
end
|
|
27
|
+
|
|
28
|
+
# Stop a transfer that was started asynchronously.
|
|
29
|
+
# Override in agents that support it (transferd).
|
|
30
|
+
# @param transfer_id [String] opaque id returned by start_transfer
|
|
31
|
+
# @param agent_params [Hash] connection parameters stored at submission time
|
|
32
|
+
# @return [Boolean, nil] `true` if stopped, `nil` if not supported
|
|
33
|
+
# @raise [Transfer::Error] The agent refused to stop the transfer
|
|
34
|
+
def transfer_cancel(_transfer_id, _agent_params)
|
|
35
|
+
nil
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Where received files are written (direction `receive`).
|
|
40
|
+
# Override in agents that write on a remote system (node): the destination is then not a local path.
|
|
41
|
+
# @return [Boolean] `true` if the destination folder is on the local system: restrictions on local writes apply (see `Cli::Security`)
|
|
42
|
+
def local_destination?
|
|
43
|
+
true
|
|
23
44
|
end
|
|
24
45
|
|
|
25
46
|
# Wait for all sessions to terminate and return a typed Transfer::Result.
|
data/lib/aspera/agent/connect.rb
CHANGED
|
@@ -1,22 +1,29 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'base'
|
|
7
|
+
require_relative '../products/connect'
|
|
8
|
+
require_relative '../products/desktop'
|
|
9
|
+
require_relative '../products/other'
|
|
10
|
+
require_relative '../rest'
|
|
11
|
+
require_relative '../transfer/spec'
|
|
12
|
+
require_relative '../environment'
|
|
13
|
+
require_relative '../rainbow'
|
|
8
14
|
require 'securerandom'
|
|
9
|
-
require 'aspera/rainbow'
|
|
10
15
|
using Rainbow
|
|
11
16
|
|
|
12
17
|
module Aspera
|
|
13
18
|
module Agent
|
|
14
19
|
class Connect < Base
|
|
15
20
|
# try twice the main init url in sequence
|
|
16
|
-
CONNECT_START_URIS = ['fasp://initialize', 'fasp://initialize', 'aspera-drive://initialize', 'https://test-connect.ibmaspera.com/']
|
|
21
|
+
CONNECT_START_URIS = ['fasp://initialize', 'fasp://initialize', 'aspera-drive://initialize', 'https://test-connect.ibmaspera.com/'].freeze
|
|
17
22
|
# delay between each try to start connect
|
|
18
23
|
SLEEP_SEC_BETWEEN_RETRY = 5
|
|
19
|
-
|
|
24
|
+
# Connect is end of life in 2027, replaced by Aspera for Desktop
|
|
25
|
+
DEPRECATION_MESSAGE = "#{Products::Connect::APP_NAME} will be deprecated in 2027: use #{Products::Desktop::APP_NAME} instead (--transfer=desktop)".freeze
|
|
26
|
+
private_constant :CONNECT_START_URIS, :SLEEP_SEC_BETWEEN_RETRY, :DEPRECATION_MESSAGE
|
|
20
27
|
|
|
21
28
|
class << self
|
|
22
29
|
# Re-query a previously started transfer via the Connect REST API.
|
|
@@ -64,6 +71,7 @@ module Aspera
|
|
|
64
71
|
|
|
65
72
|
def initialize(**base_options)
|
|
66
73
|
super
|
|
74
|
+
Log.log.warn { DEPRECATION_MESSAGE.red }
|
|
67
75
|
@transfer_id = nil
|
|
68
76
|
@connect_settings = {
|
|
69
77
|
'app_id' => SecureRandom.uuid
|
|
@@ -139,10 +147,6 @@ module Aspera
|
|
|
139
147
|
Log.dump(:tr_info, tr_info, level: :trace1)
|
|
140
148
|
if tr_info['transfer_info'].is_a?(Hash)
|
|
141
149
|
transfer = tr_info['transfer_info']
|
|
142
|
-
if transfer.nil?
|
|
143
|
-
Log.log.warn('no session in Connect')
|
|
144
|
-
break
|
|
145
|
-
end
|
|
146
150
|
# TODO: get session id
|
|
147
151
|
case transfer['status']
|
|
148
152
|
when 'initiating', 'queued'
|
data/lib/aspera/agent/desktop.rb
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'base'
|
|
7
|
+
require_relative '../rest'
|
|
8
|
+
require_relative '../environment'
|
|
9
|
+
require_relative '../json_rpc/client'
|
|
10
|
+
require_relative '../products/desktop'
|
|
11
|
+
require_relative '../transfer/spec'
|
|
9
12
|
require 'securerandom'
|
|
10
13
|
|
|
11
14
|
module Aspera
|
|
@@ -13,7 +16,7 @@ module Aspera
|
|
|
13
16
|
# Client: Aspera for Desktop
|
|
14
17
|
class Desktop < Base
|
|
15
18
|
# try twice the main init url in sequence
|
|
16
|
-
START_URIS = ['aspera://', 'aspera://', 'aspera://']
|
|
19
|
+
START_URIS = ['aspera://', 'aspera://', 'aspera://'].freeze
|
|
17
20
|
# delay between each try to start the app
|
|
18
21
|
SLEEP_SEC_BETWEEN_RETRY = 5
|
|
19
22
|
private_constant :START_URIS, :SLEEP_SEC_BETWEEN_RETRY
|
data/lib/aspera/agent/direct.rb
CHANGED
|
@@ -1,20 +1,25 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'base'
|
|
7
|
+
require_relative '../ascp/installation'
|
|
8
|
+
require_relative '../ascp/management'
|
|
9
|
+
require_relative '../exec_spec'
|
|
10
|
+
require_relative '../temp_file_manager'
|
|
11
|
+
require_relative '../transfer/parameters'
|
|
12
|
+
require_relative '../transfer/error'
|
|
13
|
+
require_relative '../transfer/spec'
|
|
14
|
+
require_relative '../transfer/resumer'
|
|
15
|
+
require_relative '../log'
|
|
16
|
+
require_relative '../assert'
|
|
17
|
+
require_relative '../rainbow'
|
|
13
18
|
require 'socket'
|
|
14
19
|
require 'securerandom'
|
|
15
20
|
require 'shellwords'
|
|
16
21
|
require 'English'
|
|
17
|
-
require '
|
|
22
|
+
require 'time'
|
|
18
23
|
using Rainbow
|
|
19
24
|
|
|
20
25
|
module Aspera
|
|
@@ -27,7 +32,7 @@ module Aspera
|
|
|
27
32
|
# agent_params must contain '_agent_ref' pointing to the live Direct instance.
|
|
28
33
|
# @param transfer_id [String] job_id returned by start_transfer / last_job_id
|
|
29
34
|
# @param agent_params [Hash] connection parameters; '_agent_ref' is the live agent
|
|
30
|
-
# @return [Hash] status hash with at least 'status' key
|
|
35
|
+
# @return [Hash] status hash with at least 'status' key, and `bytes_transferred`, and `ended_at` when all sessions ended
|
|
31
36
|
def transfer_status(transfer_id, agent_params)
|
|
32
37
|
agent = agent_params['_agent_ref']
|
|
33
38
|
return {'status' => 'unknown', 'note' => 'direct agent not available (different process?)'} if agent.nil?
|
|
@@ -35,12 +40,14 @@ module Aspera
|
|
|
35
40
|
return {'status' => 'unknown', 'note' => "no sessions found for job #{transfer_id}"} if sessions.empty?
|
|
36
41
|
errors = sessions.filter_map { |s| s[:error] }
|
|
37
42
|
running = sessions.any? { |s| s[:thread]&.alive? }
|
|
43
|
+
result = {'bytes_transferred' => sessions.sum { |s| s[:last_size].to_i }}
|
|
44
|
+
result['ended_at'] = sessions.filter_map { |s| s[:ended_at] }.max unless running
|
|
38
45
|
if errors.any?
|
|
39
|
-
|
|
46
|
+
result.merge('status' => 'failed', 'error' => errors.first.message)
|
|
40
47
|
elsif running
|
|
41
|
-
|
|
48
|
+
result.merge('status' => 'running')
|
|
42
49
|
else
|
|
43
|
-
|
|
50
|
+
result.merge('status' => 'completed')
|
|
44
51
|
end
|
|
45
52
|
end
|
|
46
53
|
end
|
|
@@ -114,7 +121,7 @@ module Aspera
|
|
|
114
121
|
# Start `ascp` transfer(s) (non blocking), single or multi-session
|
|
115
122
|
# Session information added to @sessions
|
|
116
123
|
# @param transfer_spec [Hash] Aspera transfer specification
|
|
117
|
-
# @param token_regenerator [
|
|
124
|
+
# @param token_regenerator [Proc, nil] Returns a renewed value for the bearer token of the transfer
|
|
118
125
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
|
119
126
|
# clone transfer spec because we modify it (first level keys)
|
|
120
127
|
transfer_spec = transfer_spec.clone
|
|
@@ -171,6 +178,10 @@ module Aspera
|
|
|
171
178
|
exec_spec: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
|
|
172
179
|
}
|
|
173
180
|
|
|
181
|
+
# Temporary files of `ascp` (file list) are deleted when the last session ends,
|
|
182
|
+
# also after the end of the command (asynchronous transfer): hold them once per session, before any session starts
|
|
183
|
+
(multi_session_info.nil? ? 1 : multi_session_info[:count]).times { TempFileManager.instance.hold(session[:exec_spec].temp_files) }
|
|
184
|
+
|
|
174
185
|
if multi_session_info.nil?
|
|
175
186
|
Log.log.debug('Starting single session thread')
|
|
176
187
|
# single session for transfer : simple
|
|
@@ -266,6 +277,9 @@ module Aspera
|
|
|
266
277
|
rescue => e
|
|
267
278
|
session[:error] = e
|
|
268
279
|
raise if Log.log.debug? || !e.is_a?(Transfer::Error)
|
|
280
|
+
ensure
|
|
281
|
+
session[:ended_at] = Time.now.utc.iso8601
|
|
282
|
+
TempFileManager.instance.release(session[:exec_spec].temp_files)
|
|
269
283
|
end
|
|
270
284
|
Log.log.debug { "EXIT (#{Thread.current[:name]})" }
|
|
271
285
|
end
|
|
@@ -368,11 +382,11 @@ module Aspera
|
|
|
368
382
|
Log.log.trace1 { 'Graceful shutdown, DONE message received' }
|
|
369
383
|
when 'ERROR'
|
|
370
384
|
if /bearer token/i.match?(last_event['Description']) &&
|
|
371
|
-
session[:token_regenerator].
|
|
385
|
+
!session[:token_regenerator].nil?
|
|
372
386
|
# regenerate token here, expired, or error on it
|
|
373
387
|
# Note: in multi-session, each session will have a different one.
|
|
374
388
|
Log.log.warn('Regenerating token for transfer')
|
|
375
|
-
env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].
|
|
389
|
+
env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].call
|
|
376
390
|
end
|
|
377
391
|
raise Transfer::Error.new(last_event['Description'], code: last_event['Code'].to_i)
|
|
378
392
|
else
|
|
@@ -389,9 +403,9 @@ module Aspera
|
|
|
389
403
|
session.delete(:io)
|
|
390
404
|
# if command was successfully started, check its status
|
|
391
405
|
unless command_pid.nil?
|
|
392
|
-
Process.kill(:INT, command_pid) if @monitor && !Environment.instance.os.eql?(Environment::OS_WINDOWS)
|
|
406
|
+
::Process.kill(:INT, command_pid) if @monitor && !Environment.instance.os.eql?(Environment::OS_WINDOWS)
|
|
393
407
|
# collect process exit status or wait for termination
|
|
394
|
-
_, status = Process.wait2(command_pid)
|
|
408
|
+
_, status = ::Process.wait2(command_pid)
|
|
395
409
|
if capture_stderr
|
|
396
410
|
# process stderr of ascp
|
|
397
411
|
stderr_flag = false
|
|
@@ -452,12 +466,10 @@ module Aspera
|
|
|
452
466
|
total_size = session[:last_size] + session[:last_skipped]
|
|
453
467
|
notify_progress(:session_size, session_id: progress_id, info: total_size) if !session[:size_sent] && !total_size.zero?
|
|
454
468
|
notify_progress(:session_end, session_id: progress_id)
|
|
455
|
-
# cspell:disable
|
|
456
469
|
when 'SESSION'
|
|
457
470
|
when 'ARGSTOP'
|
|
458
471
|
when 'FILEERROR'
|
|
459
472
|
Log.log.error { "#{event['Type']} #{event['Description']}" }
|
|
460
|
-
# cspell:enable
|
|
461
473
|
else
|
|
462
474
|
Log.log.debug { "Unknown event type for progress: #{event['Type']}" }
|
|
463
475
|
end
|
data/lib/aspera/agent/factory.rb
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../log'
|
|
7
|
+
require_relative '../environment'
|
|
3
8
|
require 'singleton'
|
|
4
|
-
require 'aspera/log'
|
|
5
|
-
require 'aspera/environment'
|
|
6
9
|
module Aspera
|
|
7
10
|
module Agent
|
|
8
11
|
# Factory for Agents
|
|
@@ -20,13 +23,14 @@ module Aspera
|
|
|
20
23
|
Aspera::Agent.const_get(agent.to_s.capitalize).new(**options)
|
|
21
24
|
end
|
|
22
25
|
|
|
23
|
-
IGNORED_ITEMS = %i[factory base]
|
|
26
|
+
IGNORED_ITEMS = %i[factory base].freeze
|
|
24
27
|
# Available agents: :long : Capitalized name string, :short : single character symbol
|
|
25
28
|
ALL =
|
|
26
29
|
Dir.children(File.dirname(File.expand_path(__FILE__)))
|
|
27
30
|
.select { |file| file.end_with?(Environment::RB_EXT) }
|
|
28
31
|
.map { |file| File.basename(file, Environment::RB_EXT).to_sym }
|
|
29
|
-
.reject { |item| IGNORED_ITEMS.include?(item) }
|
|
32
|
+
.reject { |item| IGNORED_ITEMS.include?(item) }
|
|
33
|
+
.sort.to_h do |agent_sym|
|
|
30
34
|
[agent_sym, {
|
|
31
35
|
long: agent_sym.to_s.capitalize,
|
|
32
36
|
short: agent_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
|
data/lib/aspera/agent/httpgw.rb
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'base'
|
|
7
|
+
require_relative '../api/httpgw'
|
|
8
|
+
require_relative '../log'
|
|
9
|
+
require_relative '../assert'
|
|
8
10
|
require 'securerandom'
|
|
9
11
|
|
|
10
12
|
module Aspera
|
data/lib/aspera/agent/node.rb
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'base'
|
|
7
|
+
require_relative '../transfer/spec'
|
|
8
|
+
require_relative '../api/node'
|
|
9
|
+
require_relative '../log'
|
|
10
|
+
require_relative '../assert'
|
|
11
|
+
require_relative '../oauth/client/factory'
|
|
10
12
|
|
|
11
13
|
module Aspera
|
|
12
14
|
module Agent
|
|
@@ -79,6 +81,12 @@ module Aspera
|
|
|
79
81
|
@transfer_id = nil
|
|
80
82
|
end
|
|
81
83
|
|
|
84
|
+
# Received files are written on the node, not on the local system
|
|
85
|
+
# @return [Boolean] `false`
|
|
86
|
+
def local_destination?
|
|
87
|
+
false
|
|
88
|
+
end
|
|
89
|
+
|
|
82
90
|
# generic method
|
|
83
91
|
# :reek:UnusedParameters token_regenerator
|
|
84
92
|
def start_transfer(transfer_spec, token_regenerator: nil)
|