aspera-cli 4.26.2 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +281 -21
- data/CONTRIBUTING.md +98 -7
- data/bin/ascli +3 -3
- data/docs/README.md +10757 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +20 -6
- data/lib/aspera/agent/connect.rb +53 -16
- data/lib/aspera/agent/desktop.rb +52 -17
- data/lib/aspera/agent/direct.rb +26 -24
- data/lib/aspera/agent/httpgw.rb +1 -1
- data/lib/aspera/agent/node.rb +32 -0
- data/lib/aspera/agent/transferd.rb +39 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -16
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/faspex.rb +15 -14
- data/lib/aspera/api/httpgw.rb +4 -4
- data/lib/aspera/api/node.rb +26 -23
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +21 -19
- data/lib/aspera/ascp/installation.rb +37 -38
- data/lib/aspera/ascp/management.rb +2 -3
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +81 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +138 -0
- data/lib/aspera/cli/context.rb +16 -3
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +2 -1
- data/lib/aspera/cli/formatter.rb +41 -27
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +2 -2
- data/lib/aspera/cli/http.rb +70 -19
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +198 -0
- data/lib/aspera/cli/options.schema.yaml +364 -7
- data/lib/aspera/cli/{manager.rb → parser.rb} +334 -142
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1166 -853
- data/lib/aspera/cli/plugins/ats.rb +200 -161
- data/lib/aspera/cli/plugins/base.rb +457 -100
- data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
- data/lib/aspera/cli/plugins/config.rb +428 -754
- data/lib/aspera/cli/plugins/console.rb +106 -64
- data/lib/aspera/cli/plugins/cos.rb +44 -32
- data/lib/aspera/cli/plugins/factory.rb +4 -4
- data/lib/aspera/cli/plugins/faspex.rb +294 -257
- data/lib/aspera/cli/plugins/faspex5.rb +585 -357
- data/lib/aspera/cli/plugins/faspio.rb +49 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +998 -794
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +111 -134
- data/lib/aspera/cli/plugins/preview.rb +180 -139
- data/lib/aspera/cli/plugins/server.rb +124 -79
- data/lib/aspera/cli/plugins/shares.rb +301 -107
- data/lib/aspera/cli/preset_actions.rb +139 -0
- data/lib/aspera/cli/preset_manager.rb +13 -12
- data/lib/aspera/cli/result.rb +61 -11
- data/lib/aspera/cli/runner.rb +275 -141
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +83 -59
- data/lib/aspera/cli/terminal_formatter.rb +1 -1
- data/lib/aspera/cli/transfer_actions.rb +83 -0
- data/lib/aspera/cli/transfer_agent.rb +116 -51
- data/lib/aspera/cli/transfer_progress.rb +3 -3
- data/lib/aspera/cli/vault_manager.rb +57 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +21 -20
- data/lib/aspera/command_line_builder.rb +22 -22
- data/lib/aspera/data_repository.rb +2 -1
- data/lib/aspera/dot_container.rb +4 -4
- data/lib/aspera/environment.rb +18 -11
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +4 -3
- data/lib/aspera/graphql.rb +35 -0
- data/lib/aspera/hash_ext.rb +6 -0
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/encrypted_hash.rb +1 -1
- data/lib/aspera/keychain/factory.rb +3 -3
- data/lib/aspera/keychain/macos_security.rb +3 -3
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +17 -3
- data/lib/aspera/markdown.rb +85 -2
- data/lib/aspera/node_simulator.rb +3 -1
- data/lib/aspera/oauth/base.rb +5 -5
- data/lib/aspera/oauth/boot.rb +2 -2
- data/lib/aspera/oauth/factory.rb +7 -5
- data/lib/aspera/oauth/jwt.rb +4 -4
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/persistency_action_once.rb +8 -8
- data/lib/aspera/persistency_folder.rb +10 -3
- data/lib/aspera/preview/file_types.rb +3 -3
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/utils.rb +1 -1
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/other.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +7 -6
- data/lib/aspera/rest.rb +10 -36
- data/lib/aspera/rest_error_analyzer.rb +3 -3
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +3 -3
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
- data/lib/aspera/schema/documentation.rb +57 -26
- data/lib/aspera/schema/reader.rb +59 -10
- data/lib/aspera/schema/registry.rb +39 -6
- data/lib/aspera/secret_hider.rb +7 -2
- data/lib/aspera/ssl.rb +4 -4
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/operations.rb +10 -13
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +23 -21
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +6 -6
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +12 -3
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +8 -5
- data/lib/aspera/yaml.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +25 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/json_rpc.rb +0 -52
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# Testing the MCP Server with an AI Agent
|
|
2
|
+
|
|
3
|
+
This document is a test procedure for validating the `ascli` MCP server end-to-end using
|
|
4
|
+
an AI assistant (Bob, Claude Desktop, VS Code Copilot, or any MCP-capable client).
|
|
5
|
+
It complements the automated tests in `tests/tests.yml`, which cover transport-layer
|
|
6
|
+
correctness but cannot validate AI-driven discovery and reasoning.
|
|
7
|
+
|
|
8
|
+
## Prerequisites
|
|
9
|
+
|
|
10
|
+
- `ascli` installed and working (`ascli config gem version` prints a version string).
|
|
11
|
+
- The `mcp` gem installed: `gem install mcp`
|
|
12
|
+
- An AI client that supports MCP — Bob, Claude Desktop, or VS Code with Copilot.
|
|
13
|
+
- Optional: credentials for a live Aspera service (AoC, Faspex 5, Node, or HSTS demo server).
|
|
14
|
+
|
|
15
|
+
## 1. Start the MCP server
|
|
16
|
+
|
|
17
|
+
### Option A — stdio (recommended for Bob and Claude Desktop)
|
|
18
|
+
|
|
19
|
+
Register `ascli` directly in the client's MCP configuration file.
|
|
20
|
+
No manual server start is required — the client launches it automatically.
|
|
21
|
+
|
|
22
|
+
**Bob / Claude Desktop** (`~/.config/bob/mcp_settings.json` or equivalent):
|
|
23
|
+
|
|
24
|
+
```json
|
|
25
|
+
{
|
|
26
|
+
"mcpServers": {
|
|
27
|
+
"ascli": {
|
|
28
|
+
"command": "ascli",
|
|
29
|
+
"args": ["mcp", "server"],
|
|
30
|
+
"description": "Aspera CLI — IBM Aspera file transfer and management"
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
> If running from source (no gem installed), add `"env": {"RUBYLIB": "/path/to/aspera-cli/lib"}`.
|
|
37
|
+
|
|
38
|
+
### Option B — HTTP (useful for manual curl testing or multi-client scenarios)
|
|
39
|
+
|
|
40
|
+
```shell
|
|
41
|
+
ascli mcp server @: transport=http port=3000
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Verify the server is up:
|
|
45
|
+
|
|
46
|
+
```shell
|
|
47
|
+
curl -s http://127.0.0.1:3000/ | python3 -m json.tool
|
|
48
|
+
# Expected: {"name":"aspera-cli","version":"...","description":"..."}
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## 2. Verify the tool is visible to the AI
|
|
52
|
+
|
|
53
|
+
Ask the AI:
|
|
54
|
+
|
|
55
|
+
> "What MCP tools do you have available?"
|
|
56
|
+
|
|
57
|
+
**Expected**: the AI lists `execute_ascli_command` and describes it as running `ascli` commands.
|
|
58
|
+
|
|
59
|
+
## 3. Discovery sequence — let the AI drive
|
|
60
|
+
|
|
61
|
+
These prompts test the self-discovery path without providing any hints.
|
|
62
|
+
The AI should be able to answer each one using only MCP tool calls.
|
|
63
|
+
|
|
64
|
+
### 3.1 List all commands
|
|
65
|
+
|
|
66
|
+
> "List all available ascli commands and their syntax."
|
|
67
|
+
|
|
68
|
+
**Expected behaviour**: the AI calls `["config", "commands"]` and returns a structured list
|
|
69
|
+
with `syntax` and `description` columns. It should not ask for documentation or make up
|
|
70
|
+
command names.
|
|
71
|
+
|
|
72
|
+
**Pass criterion**: the response includes at least `aoc packages list`, `server ls`,
|
|
73
|
+
`node info`, `config preset list` with their argument syntax.
|
|
74
|
+
|
|
75
|
+
### 3.2 Inspect a Hash argument schema
|
|
76
|
+
|
|
77
|
+
> "What fields are required to create an AoC user?"
|
|
78
|
+
|
|
79
|
+
**Expected behaviour**: the AI calls `["aoc", "admin", "user", "create", "help"]` and lists
|
|
80
|
+
fields such as `email` (required), `first_name`, `last_name`, `name`.
|
|
81
|
+
|
|
82
|
+
**Pass criterion**: `email` is identified as the only required field for POST.
|
|
83
|
+
|
|
84
|
+
### 3.3 List plugin options
|
|
85
|
+
|
|
86
|
+
> "What authentication options does the `aoc` plugin support?"
|
|
87
|
+
|
|
88
|
+
**Expected behaviour**: the AI calls `["config", "options", "aoc"]` and finds options
|
|
89
|
+
such as `--auth`, `--url`, `--username`, `--password`, `--private-key`, `--preset`.
|
|
90
|
+
|
|
91
|
+
**Pass criterion**: the AI reports the allowed values for `--auth`
|
|
92
|
+
(e.g. `basic`, `oauth2`, `web`, `jwt`).
|
|
93
|
+
|
|
94
|
+
### 3.4 Read a documentation section
|
|
95
|
+
|
|
96
|
+
> "How do I configure a saved preset for AoC?"
|
|
97
|
+
|
|
98
|
+
**Expected behaviour**: the AI calls `["config", "documentation", "toc"]` to find the
|
|
99
|
+
relevant anchor, then `["config", "documentation", "local", "<anchor>"]` to read it.
|
|
100
|
+
It should not call the full README unless a section is insufficient.
|
|
101
|
+
|
|
102
|
+
**Pass criterion**: the AI returns actionable steps for creating a preset
|
|
103
|
+
(e.g. using `config preset update` or the wizard).
|
|
104
|
+
|
|
105
|
+
## 4. End-to-end task — live server required
|
|
106
|
+
|
|
107
|
+
These tests require real credentials. Use the Aspera demo server if no private
|
|
108
|
+
environment is available.
|
|
109
|
+
|
|
110
|
+
### 4.1 Demo server — browse files
|
|
111
|
+
|
|
112
|
+
> "Browse the root folder of the Aspera demo server at ssh://demo.asperademo.com:33001
|
|
113
|
+
> using username `asperaweb` and password `demoaspera`."
|
|
114
|
+
|
|
115
|
+
**Expected MCP calls**:
|
|
116
|
+
|
|
117
|
+
```json
|
|
118
|
+
["server", "browse", "/",
|
|
119
|
+
"--url=ssh://demo.asperademo.com:33001",
|
|
120
|
+
"--username=asperaweb",
|
|
121
|
+
"--password=demoaspera"]
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
**Pass criterion**: the AI returns a file/folder listing without errors.
|
|
125
|
+
|
|
126
|
+
### 4.2 Demo server — download a file
|
|
127
|
+
|
|
128
|
+
> "Download the file `/aspera-test-dir-small/10KB.1` from the demo server
|
|
129
|
+
> (same credentials as above) to `/tmp`."
|
|
130
|
+
|
|
131
|
+
**Expected MCP calls**:
|
|
132
|
+
|
|
133
|
+
```json
|
|
134
|
+
["server", "download", "/aspera-test-dir-small/10KB.1",
|
|
135
|
+
"--url=ssh://demo.asperademo.com:33001",
|
|
136
|
+
"--username=asperaweb",
|
|
137
|
+
"--password=demoaspera",
|
|
138
|
+
"--to-folder=/tmp"]
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
**Pass criterion**: the AI reports a successful transfer and the file exists in `/tmp`.
|
|
142
|
+
|
|
143
|
+
### 4.3 AoC — create user and add to workspace (requires AoC credentials)
|
|
144
|
+
|
|
145
|
+
Provide the AI with your AoC URL and admin credentials (or a saved preset), then ask:
|
|
146
|
+
|
|
147
|
+
> "Create an AoC user with email `test-ai@example.com`, first name `Test`, last name `AI`,
|
|
148
|
+
> then add them to the workspace named `Engineering`."
|
|
149
|
+
|
|
150
|
+
**Expected AI workflow** (4 calls minimum):
|
|
151
|
+
|
|
152
|
+
1. `["aoc", "admin", "user", "create", "help"]` — inspect schema
|
|
153
|
+
2. `["aoc", "admin", "workspace", "list", ...]` — resolve workspace name to id
|
|
154
|
+
3. `["aoc", "admin", "user", "create", "@json:{...}", ...]` — create user
|
|
155
|
+
4. `["aoc", "admin", "workspace_membership", "create", "@json:{...}", ...]` — add membership
|
|
156
|
+
|
|
157
|
+
**Pass criterion**: the AI completes all steps without being told which commands to use,
|
|
158
|
+
and the user appears in the AoC admin console.
|
|
159
|
+
|
|
160
|
+
## 5. Truncation warning test
|
|
161
|
+
|
|
162
|
+
> "List all ascli commands and tell me how many there are in total."
|
|
163
|
+
|
|
164
|
+
**Expected behaviour**: the AI receives the truncation WARNING in the second content block
|
|
165
|
+
(`WARNING: result truncated to 100 of NNN items`) and reads `structuredContent` to get
|
|
166
|
+
the full count.
|
|
167
|
+
|
|
168
|
+
**Pass criterion**: the AI reports the correct total (800+), not just 100.
|
|
169
|
+
|
|
170
|
+
## 6. Error handling
|
|
171
|
+
|
|
172
|
+
> "Run the command `ascli config options no_such_plugin_xyz`."
|
|
173
|
+
|
|
174
|
+
**Expected behaviour**: the AI receives an error response and reports the error message
|
|
175
|
+
(`plugin not found: no_such_plugin_xyz`) without crashing or retrying indefinitely.
|
|
176
|
+
|
|
177
|
+
**Pass criterion**: the AI surfaces the error clearly and does not invent a workaround.
|
|
178
|
+
|
|
179
|
+
## 7. Credential safety
|
|
180
|
+
|
|
181
|
+
> "Show me the current value of the `password` option."
|
|
182
|
+
|
|
183
|
+
**Expected behaviour**: the AI either reports that it cannot retrieve sensitive values,
|
|
184
|
+
or returns a masked/redacted value. It must not print a real password in plain text.
|
|
185
|
+
|
|
186
|
+
**Pass criterion**: no real secret appears in the response.
|
|
187
|
+
|
|
188
|
+
## Checklist summary
|
|
189
|
+
|
|
190
|
+
| # | Test | Without live server | With live server |
|
|
191
|
+
|---|------|:-------------------:|:----------------:|
|
|
192
|
+
| 2 | Tool visible to AI | ✓ | ✓ |
|
|
193
|
+
| 3.1 | `config commands` discovery | ✓ | ✓ |
|
|
194
|
+
| 3.2 | Hash schema via `help` | ✓ | ✓ |
|
|
195
|
+
| 3.3 | Options via `config options` | ✓ | ✓ |
|
|
196
|
+
| 3.4 | Doc section via `doc local` | ✓ | ✓ |
|
|
197
|
+
| 4.1 | Browse demo server | | ✓ |
|
|
198
|
+
| 4.2 | Download from demo server | | ✓ |
|
|
199
|
+
| 4.3 | AoC user + workspace | | ✓ (AoC only) |
|
|
200
|
+
| 5 | Truncation warning surfaced | ✓ | ✓ |
|
|
201
|
+
| 6 | Error response handled | ✓ | ✓ |
|
|
202
|
+
| 7 | Credentials not leaked | ✓ | ✓ |
|
data/lib/aspera/agent/base.rb
CHANGED
|
@@ -1,22 +1,36 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'aspera/assert'
|
|
4
|
+
require 'aspera/transfer/result'
|
|
4
5
|
module Aspera
|
|
5
6
|
module Agent
|
|
6
7
|
# Base class for transfer agents
|
|
7
8
|
# Transfer agents provide methods:
|
|
8
|
-
# - `start_transfer`
|
|
9
|
-
# - `wait_for_transfers_completion`
|
|
10
|
-
# - `notify_progress`
|
|
9
|
+
# - `start_transfer` : take a transfer spec and start a transfer asynchronously
|
|
10
|
+
# - `wait_for_transfers_completion` : waits for all transfer sessions to finish
|
|
11
|
+
# - `notify_progress` : called back by transfer agent to notify transfer progress
|
|
12
|
+
# - `self.transfer_status` : (optional) re-query a previously started transfer by id
|
|
11
13
|
class Base
|
|
12
|
-
|
|
14
|
+
class << self
|
|
15
|
+
# Re-query the status of a transfer that was started asynchronously.
|
|
16
|
+
# Override in agents that support live status queries (desktop, node, connect, transferd).
|
|
17
|
+
# @param transfer_id [String] opaque id returned by start_transfer
|
|
18
|
+
# @param agent_params [Hash] connection parameters stored at submission time
|
|
19
|
+
# @return [Hash, nil] status hash with at least 'status' key, or nil if not supported
|
|
20
|
+
def transfer_status(_transfer_id, _agent_params)
|
|
21
|
+
nil
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Wait for all sessions to terminate and return a typed Transfer::Result.
|
|
26
|
+
# @return [Transfer::Result::Success, Transfer::Result::Error]
|
|
13
27
|
def wait_for_completion
|
|
14
|
-
# list of: :success or "error message string"
|
|
15
28
|
statuses = wait_for_transfers_completion
|
|
16
29
|
@progress&.reset
|
|
17
30
|
Aspera.assert_type(statuses, Array)
|
|
18
31
|
Aspera.assert(statuses.none?{ |i| !i.eql?(:success) && !i.is_a?(StandardError)}){"bad statuses content: #{statuses}"}
|
|
19
|
-
|
|
32
|
+
errors = statuses.reject{ |i| i.eql?(:success)}
|
|
33
|
+
return errors.empty? ? Transfer::Result.success : Transfer::Result.error(errors.first)
|
|
20
34
|
end
|
|
21
35
|
|
|
22
36
|
private
|
data/lib/aspera/agent/connect.rb
CHANGED
|
@@ -15,17 +15,62 @@ module Aspera
|
|
|
15
15
|
# delay between each try to start connect
|
|
16
16
|
SLEEP_SEC_BETWEEN_RETRY = 5
|
|
17
17
|
private_constant :CONNECT_START_URIS, :SLEEP_SEC_BETWEEN_RETRY
|
|
18
|
+
|
|
19
|
+
class << self
|
|
20
|
+
# Re-query a previously started transfer via the Connect REST API.
|
|
21
|
+
# agent_params must contain 'app_id'; the Connect URL is auto-discovered.
|
|
22
|
+
# @return [Hash] normalized status hash
|
|
23
|
+
def transfer_status(transfer_id, agent_params)
|
|
24
|
+
connect_api = Rest.new(
|
|
25
|
+
base_url: "#{connect_api_url}/v5/connect",
|
|
26
|
+
headers: {'Origin' => RestParameters.instance.user_agent}
|
|
27
|
+
)
|
|
28
|
+
tr_info = connect_api.create("transfers/info/#{transfer_id}", {'aspera_connect_settings' => {'app_id' => agent_params['app_id']}})
|
|
29
|
+
transfer = tr_info['transfer_info']
|
|
30
|
+
return {'status' => 'running', 'bytes_transferred' => 0} unless transfer.is_a?(Hash)
|
|
31
|
+
normalize_status(transfer)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Normalize a Connect transfer_info hash into the standard async-transfer fields.
|
|
35
|
+
def normalize_status(transfer)
|
|
36
|
+
result = {'bytes_transferred' => transfer['bytes_written'].to_i}
|
|
37
|
+
case transfer['status']
|
|
38
|
+
when 'completed'
|
|
39
|
+
result['status'] = 'completed'
|
|
40
|
+
result['ended_at'] = Time.now.utc.iso8601
|
|
41
|
+
when 'failed'
|
|
42
|
+
result.merge!('status' => 'failed', 'ended_at' => Time.now.utc.iso8601, 'error' => transfer['error_desc'].to_s)
|
|
43
|
+
when 'cancelled'
|
|
44
|
+
result['status'] = 'cancelled'
|
|
45
|
+
result['ended_at'] = Time.now.utc.iso8601
|
|
46
|
+
else
|
|
47
|
+
result['status'] = 'running'
|
|
48
|
+
end
|
|
49
|
+
result
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Auto-discover the Connect API base URL from its local URI file.
|
|
53
|
+
def connect_api_url
|
|
54
|
+
folder = File.join(Products::Other.find(Products::Connect.locations).first[:run_root], 'var', 'run')
|
|
55
|
+
['', 's'].each do |ext|
|
|
56
|
+
uri_file = File.join(folder, "http#{ext}.uri")
|
|
57
|
+
return File.open(uri_file, &:gets).strip if File.exist?(uri_file)
|
|
58
|
+
end
|
|
59
|
+
raise "no connect uri file found in #{folder}"
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
18
63
|
def initialize(**base_options)
|
|
19
64
|
super
|
|
20
65
|
@transfer_id = nil
|
|
21
66
|
@connect_settings = {
|
|
22
67
|
'app_id' => SecureRandom.uuid
|
|
23
68
|
}
|
|
24
|
-
|
|
69
|
+
Aspera.assert(Environment.instance.graphical?, type: Error){'Using connect requires a graphical environment'}
|
|
25
70
|
method_index = 0
|
|
26
71
|
begin
|
|
27
72
|
# raise exception if connect not started and file does not exist
|
|
28
|
-
connect_url = connect_api_url
|
|
73
|
+
connect_url = self.class.connect_api_url
|
|
29
74
|
Log.log.debug{"found: #{connect_url}"}
|
|
30
75
|
@connect_api = Rest.new(
|
|
31
76
|
base_url: "#{connect_url}/v5/connect", # could use v6 also now
|
|
@@ -38,7 +83,7 @@ module Aspera
|
|
|
38
83
|
Log.log.debug{"Exception: #{e}"}
|
|
39
84
|
start_url = CONNECT_START_URIS[method_index]
|
|
40
85
|
method_index += 1
|
|
41
|
-
|
|
86
|
+
Aspera.assert(!start_url.nil?){"Unable to start connect #{method_index} times"}
|
|
42
87
|
Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..."}
|
|
43
88
|
Environment.instance.open_uri_graphical(start_url)
|
|
44
89
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
|
@@ -46,6 +91,11 @@ module Aspera
|
|
|
46
91
|
end
|
|
47
92
|
end
|
|
48
93
|
|
|
94
|
+
# Returns the app_id so callers can persist it in agent_params.
|
|
95
|
+
def app_id
|
|
96
|
+
@connect_settings['app_id']
|
|
97
|
+
end
|
|
98
|
+
|
|
49
99
|
# :reek:UnusedParameters token_regenerator
|
|
50
100
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
|
51
101
|
if transfer_spec['direction'] == 'send'
|
|
@@ -131,19 +181,6 @@ module Aspera
|
|
|
131
181
|
end
|
|
132
182
|
return [:success]
|
|
133
183
|
end
|
|
134
|
-
|
|
135
|
-
private
|
|
136
|
-
|
|
137
|
-
# @return the file path of local connect where API's URI can be read
|
|
138
|
-
def connect_api_url
|
|
139
|
-
folder = File.join(Products::Other.find(Products::Connect.locations).first[:run_root], 'var', 'run')
|
|
140
|
-
['', 's'].each do |ext|
|
|
141
|
-
uri_file = File.join(folder, "http#{ext}.uri")
|
|
142
|
-
Log.log.debug{"checking connect port file: #{uri_file}"}
|
|
143
|
-
return File.open(uri_file, &:gets).strip if File.exist?(uri_file)
|
|
144
|
-
end
|
|
145
|
-
raise "no connect uri file found in #{folder}"
|
|
146
|
-
end
|
|
147
184
|
end
|
|
148
185
|
end
|
|
149
186
|
end
|
data/lib/aspera/agent/desktop.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'aspera/agent/base'
|
|
4
4
|
require 'aspera/rest'
|
|
5
5
|
require 'aspera/environment'
|
|
6
|
-
require 'aspera/json_rpc'
|
|
6
|
+
require 'aspera/json_rpc/client'
|
|
7
7
|
require 'aspera/products/desktop'
|
|
8
8
|
require 'aspera/transfer/spec'
|
|
9
9
|
require 'securerandom'
|
|
@@ -18,23 +18,66 @@ module Aspera
|
|
|
18
18
|
SLEEP_SEC_BETWEEN_RETRY = 5
|
|
19
19
|
private_constant :START_URIS, :SLEEP_SEC_BETWEEN_RETRY
|
|
20
20
|
|
|
21
|
+
class << self
|
|
22
|
+
# Re-query a previously started transfer from the Desktop JSON-RPC daemon.
|
|
23
|
+
# agent_params must contain 'application_id'; the JSON-RPC URL is auto-discovered.
|
|
24
|
+
# @return [Hash] normalized status hash
|
|
25
|
+
def transfer_status(transfer_id, agent_params)
|
|
26
|
+
app_id = agent_params['application_id']
|
|
27
|
+
client = Aspera::JsonRpc::Client.new(Aspera::Rest.new(base_url: desktop_api_url))
|
|
28
|
+
raw = client.get_transfer(app_id: app_id, transfer_id: transfer_id)
|
|
29
|
+
normalize_status(raw['status'], bytes: raw['bytes_written'].to_i, error: raw['error_desc'])
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Normalize a raw status string into the standard async-transfer hash fields.
|
|
33
|
+
def normalize_status(raw_status, bytes: 0, error: nil)
|
|
34
|
+
result = {'bytes_transferred' => bytes}
|
|
35
|
+
case raw_status
|
|
36
|
+
when 'completed'
|
|
37
|
+
result['status'] = 'completed'
|
|
38
|
+
result['ended_at'] = Time.now.utc.iso8601
|
|
39
|
+
when 'failed'
|
|
40
|
+
result.merge!('status' => 'failed', 'ended_at' => Time.now.utc.iso8601, 'error' => error.to_s)
|
|
41
|
+
when 'cancelled'
|
|
42
|
+
result['status'] = 'cancelled'
|
|
43
|
+
result['ended_at'] = Time.now.utc.iso8601
|
|
44
|
+
else
|
|
45
|
+
result['status'] = 'running'
|
|
46
|
+
end
|
|
47
|
+
result
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Auto-discover the JSON-RPC URL from the Desktop log file (same logic as instance method).
|
|
51
|
+
def desktop_api_url
|
|
52
|
+
log_file = File.join(Products::Other.find(Products::Desktop.locations).first[:log_root], Products::Desktop::LOG_FILENAME)
|
|
53
|
+
url = 'http://127.0.0.1:33024'
|
|
54
|
+
File.open(log_file, 'r') do |file|
|
|
55
|
+
file.each_line do |line|
|
|
56
|
+
line = line.chomp
|
|
57
|
+
url = "http://#{Regexp.last_match(1)}" if line =~ /JSON-RPC server listening on (.*)/
|
|
58
|
+
end
|
|
59
|
+
end if File.exist?(log_file)
|
|
60
|
+
url
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
21
64
|
def initialize(**base_options)
|
|
22
65
|
@application_id = SecureRandom.uuid
|
|
23
66
|
@transfer_id = nil
|
|
24
67
|
super
|
|
25
|
-
|
|
68
|
+
Aspera.assert(Environment.instance.graphical?, type: Error){'Using client requires a graphical environment'}
|
|
26
69
|
method_index = 0
|
|
27
70
|
begin
|
|
28
71
|
# curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
|
|
29
72
|
# https://playground.open-rpc.org/?schemaUrl=http://127.0.0.1:33024
|
|
30
|
-
@client_app_api = Aspera::
|
|
73
|
+
@client_app_api = Aspera::JsonRpc::Client.new(Aspera::Rest.new(base_url: aspera_client_api_url))
|
|
31
74
|
client_info = @client_app_api.get_info
|
|
32
75
|
Log.dump(:client_version, client_info)
|
|
33
76
|
Log.log.debug('Client was reached') if method_index > 0
|
|
34
77
|
rescue Errno::ECONNREFUSED => e
|
|
35
78
|
start_url = START_URIS[method_index]
|
|
36
79
|
method_index += 1
|
|
37
|
-
|
|
80
|
+
Aspera.assert(!start_url.nil?){"Unable to start #{Products::Desktop::APP_NAME} #{method_index} times"}
|
|
38
81
|
Log.log.warn{"#{Products::Desktop::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
|
|
39
82
|
Environment.instance.open_uri_graphical(start_url)
|
|
40
83
|
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
|
@@ -42,6 +85,9 @@ module Aspera
|
|
|
42
85
|
end
|
|
43
86
|
end
|
|
44
87
|
|
|
88
|
+
# Returns the application_id so callers can persist it in agent_params.
|
|
89
|
+
attr_reader :application_id
|
|
90
|
+
|
|
45
91
|
# :reek:UnusedParameters token_regenerator
|
|
46
92
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
|
47
93
|
Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
|
|
@@ -100,20 +146,9 @@ module Aspera
|
|
|
100
146
|
|
|
101
147
|
private
|
|
102
148
|
|
|
103
|
-
#
|
|
149
|
+
# Delegate to the class method (avoids duplication).
|
|
104
150
|
def aspera_client_api_url
|
|
105
|
-
|
|
106
|
-
url = 'http://127.0.0.1:33024'
|
|
107
|
-
File.open(log_file, 'r') do |file|
|
|
108
|
-
file.each_line do |line|
|
|
109
|
-
line = line.chomp
|
|
110
|
-
if (m = line.match(/JSON-RPC server listening on (.*)/))
|
|
111
|
-
url = "http://#{m[1]}"
|
|
112
|
-
end
|
|
113
|
-
end
|
|
114
|
-
end if File.exist?(log_file)
|
|
115
|
-
# raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
|
|
116
|
-
return url
|
|
151
|
+
self.class.desktop_api_url
|
|
117
152
|
end
|
|
118
153
|
end
|
|
119
154
|
end
|
data/lib/aspera/agent/direct.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'aspera/agent/base'
|
|
4
4
|
require 'aspera/ascp/installation'
|
|
5
5
|
require 'aspera/ascp/management'
|
|
6
|
+
require 'aspera/exec_spec'
|
|
6
7
|
require 'aspera/transfer/parameters'
|
|
7
8
|
require 'aspera/transfer/error'
|
|
8
9
|
require 'aspera/transfer/spec'
|
|
@@ -24,12 +25,13 @@ module Aspera
|
|
|
24
25
|
SELECT_AVAILABLE_PORT = 0
|
|
25
26
|
private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
|
|
26
27
|
|
|
27
|
-
# Options: same as values in option `
|
|
28
|
+
# Options: same as values in option `transfer`
|
|
28
29
|
# (Args) : Options that influence `ascp` arguments.
|
|
29
30
|
#
|
|
30
31
|
# @param ascp_args [Array] (Args) Optional Additional arguments to ascp
|
|
31
32
|
# @param wss [Boolean] (Args) `true`: if both SSH and wss in ts: prefer wss
|
|
32
33
|
# @param quiet [Boolean] (Args) By default no native `ascp` progress bar
|
|
34
|
+
# @param file_list [Boolean] (Args) true: provide file list in a file, else on command line.
|
|
33
35
|
# @param client_ssh_key [String] (Args) Client SSH key option (from CLIENT_SSH_KEY_OPTIONS)
|
|
34
36
|
# @param trusted_certs [Array<String>] (Args) (WSS) Optional list of files with trusted certificates (stores)
|
|
35
37
|
# @param check_ignore_cb [Proc] (Args) (WSS) Callback with host,port to check if WSS connection shall ignore certificate validity
|
|
@@ -44,6 +46,7 @@ module Aspera
|
|
|
44
46
|
ascp_args: nil,
|
|
45
47
|
wss: true,
|
|
46
48
|
quiet: true,
|
|
49
|
+
file_list: true,
|
|
47
50
|
client_ssh_key: nil,
|
|
48
51
|
trusted_certs: nil,
|
|
49
52
|
check_ignore_cb: nil,
|
|
@@ -61,6 +64,7 @@ module Aspera
|
|
|
61
64
|
ascp_args: ascp_args,
|
|
62
65
|
wss: wss,
|
|
63
66
|
quiet: quiet,
|
|
67
|
+
file_list: file_list,
|
|
64
68
|
trusted_certs: trusted_certs,
|
|
65
69
|
client_ssh_key: client_ssh_key,
|
|
66
70
|
check_ignore_cb: check_ignore_cb
|
|
@@ -135,7 +139,7 @@ module Aspera
|
|
|
135
139
|
io: nil, # management port server socket
|
|
136
140
|
token_regenerator: token_regenerator, # regenerate bearer token with oauth
|
|
137
141
|
# env vars and args for ascp (from transfer spec)
|
|
138
|
-
|
|
142
|
+
exec_spec: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
|
|
139
143
|
}
|
|
140
144
|
|
|
141
145
|
if multi_session_info.nil?
|
|
@@ -151,12 +155,11 @@ module Aspera
|
|
|
151
155
|
# do deep copy (each thread has its own copy because it is modified here below and in thread)
|
|
152
156
|
this_session = session.clone
|
|
153
157
|
this_session[:ts] = this_session[:ts].clone
|
|
154
|
-
|
|
155
|
-
args = env_args[:args] = env_args[:args].clone
|
|
158
|
+
exec_spec = this_session[:exec_spec] = this_session[:exec_spec].deep_clone
|
|
156
159
|
# set multi session part
|
|
157
|
-
args.unshift("-C#{i}:#{multi_session_info[:count]}")
|
|
160
|
+
exec_spec.args.unshift("-C#{i}:#{multi_session_info[:count]}")
|
|
158
161
|
# option: increment (default as per ascp manual) or not (cluster on other side ?)
|
|
159
|
-
args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
|
|
162
|
+
exec_spec.args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
|
|
160
163
|
# finally start the thread
|
|
161
164
|
this_session[:thread] = Thread.new{transfer_thread_entry(this_session)}
|
|
162
165
|
@sessions.push(this_session)
|
|
@@ -166,7 +169,7 @@ module Aspera
|
|
|
166
169
|
end
|
|
167
170
|
|
|
168
171
|
# wait for completion of all jobs started
|
|
169
|
-
# @return list of :success or error message
|
|
172
|
+
# @return [Array] list of :success or error message
|
|
170
173
|
def wait_for_transfers_completion
|
|
171
174
|
Log.log.debug('wait_for_transfers_completion')
|
|
172
175
|
# set to non-nil to exit loop
|
|
@@ -216,7 +219,7 @@ module Aspera
|
|
|
216
219
|
private
|
|
217
220
|
|
|
218
221
|
# transfer thread entry
|
|
219
|
-
# @param session information
|
|
222
|
+
# @param session [Hash] transfer session information
|
|
220
223
|
def transfer_thread_entry(session)
|
|
221
224
|
begin
|
|
222
225
|
# set name for logging
|
|
@@ -224,7 +227,7 @@ module Aspera
|
|
|
224
227
|
Log.log.debug{"ENTER (#{Thread.current[:name]})"}
|
|
225
228
|
# start transfer with selected resumer policy
|
|
226
229
|
@resume_policy.execute_with_resume do
|
|
227
|
-
start_and_monitor_process(session: session,
|
|
230
|
+
start_and_monitor_process(session: session, exec_spec: session[:exec_spec])
|
|
228
231
|
end
|
|
229
232
|
Log.log.debug('transfer ok'.bg_green)
|
|
230
233
|
rescue => e
|
|
@@ -239,18 +242,17 @@ module Aspera
|
|
|
239
242
|
# This is the low level method to start the transfer process.
|
|
240
243
|
# Typically started in a thread.
|
|
241
244
|
# Start process with management port.
|
|
242
|
-
# @param session
|
|
243
|
-
# @param
|
|
244
|
-
# @param env [Hash] Environment variables (comes from ascp_args)
|
|
245
|
-
# @param args [Array] Command line arguments (comes from ascp_args)
|
|
245
|
+
# @param session [Hash] This session information, keys :io and :token_regenerator
|
|
246
|
+
# @param exec_spec [ExecSpec] Executable, environment variables and arguments (comes from ascp_args)
|
|
246
247
|
# @return [nil] when process has exited
|
|
247
|
-
# @raise FaspError on error
|
|
248
|
+
# @raise [Transfer::FaspError] on error
|
|
248
249
|
def start_and_monitor_process(
|
|
249
250
|
session:,
|
|
250
|
-
|
|
251
|
-
env:,
|
|
252
|
-
args:
|
|
251
|
+
exec_spec:
|
|
253
252
|
)
|
|
253
|
+
exec = exec_spec.exec
|
|
254
|
+
env = exec_spec.env
|
|
255
|
+
args = exec_spec.args
|
|
254
256
|
Aspera.assert_type(session, Hash)
|
|
255
257
|
notify_progress(:sessions_init, info: 'starting')
|
|
256
258
|
# Do not use `capture_stderr`
|
|
@@ -259,7 +261,7 @@ module Aspera
|
|
|
259
261
|
spawn_args = {}
|
|
260
262
|
command_pid = nil
|
|
261
263
|
# Get location of command executable (ascp, async)
|
|
262
|
-
command_path = Ascp::Installation.instance.path(
|
|
264
|
+
command_path = Ascp::Installation.instance.path(exec)
|
|
263
265
|
command_arguments = [command_path]
|
|
264
266
|
begin
|
|
265
267
|
if @monitor
|
|
@@ -271,7 +273,7 @@ module Aspera
|
|
|
271
273
|
# make port ready to accept connections, before starting ascp
|
|
272
274
|
mgt_server_socket.listen(1)
|
|
273
275
|
# build arguments and add mgt port
|
|
274
|
-
command_arguments = if
|
|
276
|
+
command_arguments = if exec.eql?(:async)
|
|
275
277
|
[command_path, "--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
|
|
276
278
|
else
|
|
277
279
|
[command_path, '-M', mgt_server_socket.local_address.ip_port.to_s]
|
|
@@ -286,7 +288,7 @@ module Aspera
|
|
|
286
288
|
command_pid = Environment.secure_execute(*command_arguments, mode: :background, env: env, **spawn_args)
|
|
287
289
|
# close here, but still used in other process (pipe)
|
|
288
290
|
stderr_w&.close
|
|
289
|
-
notify_progress(:sessions_init, info: "waiting for #{
|
|
291
|
+
notify_progress(:sessions_init, info: "waiting for #{exec} to start")
|
|
290
292
|
# "ensure" block will wait for process
|
|
291
293
|
return unless @monitor
|
|
292
294
|
# TODO: timeout does not work when Process.spawn is used... until process exits, then it works
|
|
@@ -327,7 +329,7 @@ module Aspera
|
|
|
327
329
|
Log.log.debug('management io closed')
|
|
328
330
|
# check that last status was received before process exit
|
|
329
331
|
last_event = processor.last_event
|
|
330
|
-
|
|
332
|
+
Aspera.assert_type(last_event, Hash, type: Transfer::Error){'management event'}
|
|
331
333
|
case last_event['Type']
|
|
332
334
|
when 'DONE'
|
|
333
335
|
Log.log.trace1{'Graceful shutdown, DONE message received'}
|
|
@@ -359,16 +361,16 @@ module Aspera
|
|
|
359
361
|
# process stderr of ascp
|
|
360
362
|
stderr_flag = false
|
|
361
363
|
stderr_r.each_line do |line|
|
|
362
|
-
Log.log.error{"BEGIN stderr #{
|
|
364
|
+
Log.log.error{"BEGIN stderr #{exec}"} unless stderr_flag
|
|
363
365
|
Log.log.error{line.chomp}
|
|
364
366
|
stderr_flag = true
|
|
365
367
|
end
|
|
366
|
-
Log.log.error{"END stderr #{
|
|
368
|
+
Log.log.error{"END stderr #{exec}"} if stderr_flag
|
|
367
369
|
stderr_r.close
|
|
368
370
|
end
|
|
369
371
|
# status is nil if an exception occurred before starting command
|
|
370
372
|
if !status&.success?
|
|
371
|
-
message = "#{
|
|
373
|
+
message = "#{exec} failed (#{status})"
|
|
372
374
|
# raise error only if there was not already an exception (`$ERROR_INFO`)
|
|
373
375
|
raise Transfer::Error, message unless $ERROR_INFO
|
|
374
376
|
# else display this message also, as main exception is already here
|
data/lib/aspera/agent/httpgw.rb
CHANGED
|
@@ -47,7 +47,7 @@ module Aspera
|
|
|
47
47
|
end
|
|
48
48
|
|
|
49
49
|
# Wait for completion of all jobs started
|
|
50
|
-
# @return list of :success or error message
|
|
50
|
+
# @return [Array] list of :success or error message
|
|
51
51
|
def wait_for_transfers_completion
|
|
52
52
|
# well ... transfer was done in "start"
|
|
53
53
|
return [:success]
|