aspera-cli 4.26.2 → 4.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +281 -21
  4. data/CONTRIBUTING.md +98 -7
  5. data/bin/ascli +3 -3
  6. data/docs/README.md +10757 -0
  7. data/docs/test-mcp-with-ai.md +202 -0
  8. data/lib/aspera/agent/base.rb +20 -6
  9. data/lib/aspera/agent/connect.rb +53 -16
  10. data/lib/aspera/agent/desktop.rb +52 -17
  11. data/lib/aspera/agent/direct.rb +26 -24
  12. data/lib/aspera/agent/httpgw.rb +1 -1
  13. data/lib/aspera/agent/node.rb +32 -0
  14. data/lib/aspera/agent/transferd.rb +39 -3
  15. data/lib/aspera/api/alee.rb +1 -1
  16. data/lib/aspera/api/aoc.rb +14 -16
  17. data/lib/aspera/api/cos_node.rb +1 -1
  18. data/lib/aspera/api/faspex.rb +15 -14
  19. data/lib/aspera/api/httpgw.rb +4 -4
  20. data/lib/aspera/api/node.rb +26 -23
  21. data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
  22. data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
  23. data/lib/aspera/ascmd.rb +21 -19
  24. data/lib/aspera/ascp/installation.rb +37 -38
  25. data/lib/aspera/ascp/management.rb +2 -3
  26. data/lib/aspera/cli/ascp_actions.rb +155 -0
  27. data/lib/aspera/cli/async_transfer_store.rb +81 -0
  28. data/lib/aspera/cli/bootstrapper.rb +197 -0
  29. data/lib/aspera/cli/command_registry.rb +145 -0
  30. data/lib/aspera/cli/command_spec.rb +138 -0
  31. data/lib/aspera/cli/context.rb +16 -3
  32. data/lib/aspera/cli/error.rb +16 -0
  33. data/lib/aspera/cli/extended_value.rb +2 -1
  34. data/lib/aspera/cli/formatter.rb +41 -27
  35. data/lib/aspera/cli/gem_checker.rb +65 -0
  36. data/lib/aspera/cli/hints.rb +2 -2
  37. data/lib/aspera/cli/http.rb +70 -19
  38. data/lib/aspera/cli/info.rb +2 -0
  39. data/lib/aspera/cli/mailer.rb +97 -0
  40. data/lib/aspera/cli/mcp_tool.rb +198 -0
  41. data/lib/aspera/cli/options.schema.yaml +364 -7
  42. data/lib/aspera/cli/{manager.rb → parser.rb} +334 -142
  43. data/lib/aspera/cli/plugins/alee.rb +20 -22
  44. data/lib/aspera/cli/plugins/aoc.rb +1166 -853
  45. data/lib/aspera/cli/plugins/ats.rb +200 -161
  46. data/lib/aspera/cli/plugins/base.rb +457 -100
  47. data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
  48. data/lib/aspera/cli/plugins/config.rb +428 -754
  49. data/lib/aspera/cli/plugins/console.rb +106 -64
  50. data/lib/aspera/cli/plugins/cos.rb +44 -32
  51. data/lib/aspera/cli/plugins/factory.rb +4 -4
  52. data/lib/aspera/cli/plugins/faspex.rb +294 -257
  53. data/lib/aspera/cli/plugins/faspex5.rb +585 -357
  54. data/lib/aspera/cli/plugins/faspio.rb +49 -51
  55. data/lib/aspera/cli/plugins/httpgw.rb +18 -25
  56. data/lib/aspera/cli/plugins/mcp.rb +279 -0
  57. data/lib/aspera/cli/plugins/node.rb +998 -794
  58. data/lib/aspera/cli/plugins/oauth.rb +7 -10
  59. data/lib/aspera/cli/plugins/orchestrator.rb +111 -134
  60. data/lib/aspera/cli/plugins/preview.rb +180 -139
  61. data/lib/aspera/cli/plugins/server.rb +124 -79
  62. data/lib/aspera/cli/plugins/shares.rb +301 -107
  63. data/lib/aspera/cli/preset_actions.rb +139 -0
  64. data/lib/aspera/cli/preset_manager.rb +13 -12
  65. data/lib/aspera/cli/result.rb +61 -11
  66. data/lib/aspera/cli/runner.rb +275 -141
  67. data/lib/aspera/cli/secret_finder.rb +40 -0
  68. data/lib/aspera/cli/special_values.rb +1 -0
  69. data/lib/aspera/cli/sync_actions.rb +83 -59
  70. data/lib/aspera/cli/terminal_formatter.rb +1 -1
  71. data/lib/aspera/cli/transfer_actions.rb +83 -0
  72. data/lib/aspera/cli/transfer_agent.rb +116 -51
  73. data/lib/aspera/cli/transfer_progress.rb +3 -3
  74. data/lib/aspera/cli/vault_manager.rb +57 -0
  75. data/lib/aspera/cli/version.rb +1 -1
  76. data/lib/aspera/cli/wizard.rb +21 -20
  77. data/lib/aspera/command_line_builder.rb +22 -22
  78. data/lib/aspera/data_repository.rb +2 -1
  79. data/lib/aspera/dot_container.rb +4 -4
  80. data/lib/aspera/environment.rb +18 -11
  81. data/lib/aspera/exec_spec.rb +13 -0
  82. data/lib/aspera/faspex_gw.rb +1 -1
  83. data/lib/aspera/faspex_postproc.rb +4 -3
  84. data/lib/aspera/graphql.rb +35 -0
  85. data/lib/aspera/hash_ext.rb +6 -0
  86. data/lib/aspera/json_rpc/client.rb +62 -0
  87. data/lib/aspera/json_rpc/version.rb +7 -0
  88. data/lib/aspera/keychain/encrypted_hash.rb +1 -1
  89. data/lib/aspera/keychain/factory.rb +3 -3
  90. data/lib/aspera/keychain/macos_security.rb +3 -3
  91. data/lib/aspera/link_header.rb +82 -0
  92. data/lib/aspera/log.rb +17 -3
  93. data/lib/aspera/markdown.rb +85 -2
  94. data/lib/aspera/node_simulator.rb +3 -1
  95. data/lib/aspera/oauth/base.rb +5 -5
  96. data/lib/aspera/oauth/boot.rb +2 -2
  97. data/lib/aspera/oauth/factory.rb +7 -5
  98. data/lib/aspera/oauth/jwt.rb +4 -4
  99. data/lib/aspera/oauth/web.rb +2 -2
  100. data/lib/aspera/persistency_action_once.rb +8 -8
  101. data/lib/aspera/persistency_folder.rb +10 -3
  102. data/lib/aspera/preview/file_types.rb +3 -3
  103. data/lib/aspera/preview/generator.rb +3 -3
  104. data/lib/aspera/preview/utils.rb +1 -1
  105. data/lib/aspera/products/connect.rb +1 -1
  106. data/lib/aspera/products/other.rb +1 -1
  107. data/lib/aspera/products/transferd.rb +1 -1
  108. data/lib/aspera/proxy_auto_config.rb +7 -6
  109. data/lib/aspera/rest.rb +10 -36
  110. data/lib/aspera/rest_error_analyzer.rb +3 -3
  111. data/lib/aspera/rest_errors_aspera.rb +0 -10
  112. data/lib/aspera/rest_list.rb +3 -3
  113. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
  114. data/lib/aspera/schema/documentation.rb +57 -26
  115. data/lib/aspera/schema/reader.rb +59 -10
  116. data/lib/aspera/schema/registry.rb +39 -6
  117. data/lib/aspera/secret_hider.rb +7 -2
  118. data/lib/aspera/ssl.rb +4 -4
  119. data/lib/aspera/sync/conf.schema.yaml +2 -2
  120. data/lib/aspera/sync/operations.rb +10 -13
  121. data/lib/aspera/temp_file_manager.rb +1 -1
  122. data/lib/aspera/timer_limiter.rb +1 -1
  123. data/lib/aspera/transfer/faux_file.rb +24 -11
  124. data/lib/aspera/transfer/parameters.rb +23 -21
  125. data/lib/aspera/transfer/result.rb +74 -0
  126. data/lib/aspera/transfer/resumer.rb +6 -6
  127. data/lib/aspera/transfer/spec.rb +18 -1
  128. data/lib/aspera/transfer/spec.schema.yaml +12 -3
  129. data/lib/aspera/uri_reader.rb +2 -2
  130. data/lib/aspera/web_auth.rb +2 -2
  131. data/lib/aspera/web_server_simple.rb +8 -5
  132. data/lib/aspera/yaml.rb +2 -1
  133. data.tar.gz.sig +0 -0
  134. metadata +25 -3
  135. metadata.gz.sig +0 -0
  136. data/lib/aspera/json_rpc.rb +0 -52
@@ -0,0 +1,202 @@
1
+ # Testing the MCP Server with an AI Agent
2
+
3
+ This document is a test procedure for validating the `ascli` MCP server end-to-end using
4
+ an AI assistant (Bob, Claude Desktop, VS Code Copilot, or any MCP-capable client).
5
+ It complements the automated tests in `tests/tests.yml`, which cover transport-layer
6
+ correctness but cannot validate AI-driven discovery and reasoning.
7
+
8
+ ## Prerequisites
9
+
10
+ - `ascli` installed and working (`ascli config gem version` prints a version string).
11
+ - The `mcp` gem installed: `gem install mcp`
12
+ - An AI client that supports MCP — Bob, Claude Desktop, or VS Code with Copilot.
13
+ - Optional: credentials for a live Aspera service (AoC, Faspex 5, Node, or HSTS demo server).
14
+
15
+ ## 1. Start the MCP server
16
+
17
+ ### Option A — stdio (recommended for Bob and Claude Desktop)
18
+
19
+ Register `ascli` directly in the client's MCP configuration file.
20
+ No manual server start is required — the client launches it automatically.
21
+
22
+ **Bob / Claude Desktop** (`~/.config/bob/mcp_settings.json` or equivalent):
23
+
24
+ ```json
25
+ {
26
+ "mcpServers": {
27
+ "ascli": {
28
+ "command": "ascli",
29
+ "args": ["mcp", "server"],
30
+ "description": "Aspera CLI — IBM Aspera file transfer and management"
31
+ }
32
+ }
33
+ }
34
+ ```
35
+
36
+ > If running from source (no gem installed), add `"env": {"RUBYLIB": "/path/to/aspera-cli/lib"}`.
37
+
38
+ ### Option B — HTTP (useful for manual curl testing or multi-client scenarios)
39
+
40
+ ```shell
41
+ ascli mcp server @: transport=http port=3000
42
+ ```
43
+
44
+ Verify the server is up:
45
+
46
+ ```shell
47
+ curl -s http://127.0.0.1:3000/ | python3 -m json.tool
48
+ # Expected: {"name":"aspera-cli","version":"...","description":"..."}
49
+ ```
50
+
51
+ ## 2. Verify the tool is visible to the AI
52
+
53
+ Ask the AI:
54
+
55
+ > "What MCP tools do you have available?"
56
+
57
+ **Expected**: the AI lists `execute_ascli_command` and describes it as running `ascli` commands.
58
+
59
+ ## 3. Discovery sequence — let the AI drive
60
+
61
+ These prompts test the self-discovery path without providing any hints.
62
+ The AI should be able to answer each one using only MCP tool calls.
63
+
64
+ ### 3.1 List all commands
65
+
66
+ > "List all available ascli commands and their syntax."
67
+
68
+ **Expected behaviour**: the AI calls `["config", "commands"]` and returns a structured list
69
+ with `syntax` and `description` columns. It should not ask for documentation or make up
70
+ command names.
71
+
72
+ **Pass criterion**: the response includes at least `aoc packages list`, `server ls`,
73
+ `node info`, `config preset list` with their argument syntax.
74
+
75
+ ### 3.2 Inspect a Hash argument schema
76
+
77
+ > "What fields are required to create an AoC user?"
78
+
79
+ **Expected behaviour**: the AI calls `["aoc", "admin", "user", "create", "help"]` and lists
80
+ fields such as `email` (required), `first_name`, `last_name`, `name`.
81
+
82
+ **Pass criterion**: `email` is identified as the only required field for POST.
83
+
84
+ ### 3.3 List plugin options
85
+
86
+ > "What authentication options does the `aoc` plugin support?"
87
+
88
+ **Expected behaviour**: the AI calls `["config", "options", "aoc"]` and finds options
89
+ such as `--auth`, `--url`, `--username`, `--password`, `--private-key`, `--preset`.
90
+
91
+ **Pass criterion**: the AI reports the allowed values for `--auth`
92
+ (e.g. `basic`, `oauth2`, `web`, `jwt`).
93
+
94
+ ### 3.4 Read a documentation section
95
+
96
+ > "How do I configure a saved preset for AoC?"
97
+
98
+ **Expected behaviour**: the AI calls `["config", "documentation", "toc"]` to find the
99
+ relevant anchor, then `["config", "documentation", "local", "<anchor>"]` to read it.
100
+ It should not call the full README unless a section is insufficient.
101
+
102
+ **Pass criterion**: the AI returns actionable steps for creating a preset
103
+ (e.g. using `config preset update` or the wizard).
104
+
105
+ ## 4. End-to-end task — live server required
106
+
107
+ These tests require real credentials. Use the Aspera demo server if no private
108
+ environment is available.
109
+
110
+ ### 4.1 Demo server — browse files
111
+
112
+ > "Browse the root folder of the Aspera demo server at ssh://demo.asperademo.com:33001
113
+ > using username `asperaweb` and password `demoaspera`."
114
+
115
+ **Expected MCP calls**:
116
+
117
+ ```json
118
+ ["server", "browse", "/",
119
+ "--url=ssh://demo.asperademo.com:33001",
120
+ "--username=asperaweb",
121
+ "--password=demoaspera"]
122
+ ```
123
+
124
+ **Pass criterion**: the AI returns a file/folder listing without errors.
125
+
126
+ ### 4.2 Demo server — download a file
127
+
128
+ > "Download the file `/aspera-test-dir-small/10KB.1` from the demo server
129
+ > (same credentials as above) to `/tmp`."
130
+
131
+ **Expected MCP calls**:
132
+
133
+ ```json
134
+ ["server", "download", "/aspera-test-dir-small/10KB.1",
135
+ "--url=ssh://demo.asperademo.com:33001",
136
+ "--username=asperaweb",
137
+ "--password=demoaspera",
138
+ "--to-folder=/tmp"]
139
+ ```
140
+
141
+ **Pass criterion**: the AI reports a successful transfer and the file exists in `/tmp`.
142
+
143
+ ### 4.3 AoC — create user and add to workspace (requires AoC credentials)
144
+
145
+ Provide the AI with your AoC URL and admin credentials (or a saved preset), then ask:
146
+
147
+ > "Create an AoC user with email `test-ai@example.com`, first name `Test`, last name `AI`,
148
+ > then add them to the workspace named `Engineering`."
149
+
150
+ **Expected AI workflow** (4 calls minimum):
151
+
152
+ 1. `["aoc", "admin", "user", "create", "help"]` — inspect schema
153
+ 2. `["aoc", "admin", "workspace", "list", ...]` — resolve workspace name to id
154
+ 3. `["aoc", "admin", "user", "create", "@json:{...}", ...]` — create user
155
+ 4. `["aoc", "admin", "workspace_membership", "create", "@json:{...}", ...]` — add membership
156
+
157
+ **Pass criterion**: the AI completes all steps without being told which commands to use,
158
+ and the user appears in the AoC admin console.
159
+
160
+ ## 5. Truncation warning test
161
+
162
+ > "List all ascli commands and tell me how many there are in total."
163
+
164
+ **Expected behaviour**: the AI receives the truncation WARNING in the second content block
165
+ (`WARNING: result truncated to 100 of NNN items`) and reads `structuredContent` to get
166
+ the full count.
167
+
168
+ **Pass criterion**: the AI reports the correct total (800+), not just 100.
169
+
170
+ ## 6. Error handling
171
+
172
+ > "Run the command `ascli config options no_such_plugin_xyz`."
173
+
174
+ **Expected behaviour**: the AI receives an error response and reports the error message
175
+ (`plugin not found: no_such_plugin_xyz`) without crashing or retrying indefinitely.
176
+
177
+ **Pass criterion**: the AI surfaces the error clearly and does not invent a workaround.
178
+
179
+ ## 7. Credential safety
180
+
181
+ > "Show me the current value of the `password` option."
182
+
183
+ **Expected behaviour**: the AI either reports that it cannot retrieve sensitive values,
184
+ or returns a masked/redacted value. It must not print a real password in plain text.
185
+
186
+ **Pass criterion**: no real secret appears in the response.
187
+
188
+ ## Checklist summary
189
+
190
+ | # | Test | Without live server | With live server |
191
+ |---|------|:-------------------:|:----------------:|
192
+ | 2 | Tool visible to AI | ✓ | ✓ |
193
+ | 3.1 | `config commands` discovery | ✓ | ✓ |
194
+ | 3.2 | Hash schema via `help` | ✓ | ✓ |
195
+ | 3.3 | Options via `config options` | ✓ | ✓ |
196
+ | 3.4 | Doc section via `doc local` | ✓ | ✓ |
197
+ | 4.1 | Browse demo server | | ✓ |
198
+ | 4.2 | Download from demo server | | ✓ |
199
+ | 4.3 | AoC user + workspace | | ✓ (AoC only) |
200
+ | 5 | Truncation warning surfaced | ✓ | ✓ |
201
+ | 6 | Error response handled | ✓ | ✓ |
202
+ | 7 | Credentials not leaked | ✓ | ✓ |
@@ -1,22 +1,36 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/assert'
4
+ require 'aspera/transfer/result'
4
5
  module Aspera
5
6
  module Agent
6
7
  # Base class for transfer agents
7
8
  # Transfer agents provide methods:
8
- # - `start_transfer` : take a transfer spec and start a transfer asynchronously
9
- # - `wait_for_transfers_completion` : waits for all transfer sessions to finish
10
- # - `notify_progress` : called back by transfer agent to notify transfer progress
9
+ # - `start_transfer` : take a transfer spec and start a transfer asynchronously
10
+ # - `wait_for_transfers_completion` : waits for all transfer sessions to finish
11
+ # - `notify_progress` : called back by transfer agent to notify transfer progress
12
+ # - `self.transfer_status` : (optional) re-query a previously started transfer by id
11
13
  class Base
12
- # Wait for all sessions to terminate and return the status of each session
14
+ class << self
15
+ # Re-query the status of a transfer that was started asynchronously.
16
+ # Override in agents that support live status queries (desktop, node, connect, transferd).
17
+ # @param transfer_id [String] opaque id returned by start_transfer
18
+ # @param agent_params [Hash] connection parameters stored at submission time
19
+ # @return [Hash, nil] status hash with at least 'status' key, or nil if not supported
20
+ def transfer_status(_transfer_id, _agent_params)
21
+ nil
22
+ end
23
+ end
24
+
25
+ # Wait for all sessions to terminate and return a typed Transfer::Result.
26
+ # @return [Transfer::Result::Success, Transfer::Result::Error]
13
27
  def wait_for_completion
14
- # list of: :success or "error message string"
15
28
  statuses = wait_for_transfers_completion
16
29
  @progress&.reset
17
30
  Aspera.assert_type(statuses, Array)
18
31
  Aspera.assert(statuses.none?{ |i| !i.eql?(:success) && !i.is_a?(StandardError)}){"bad statuses content: #{statuses}"}
19
- return statuses
32
+ errors = statuses.reject{ |i| i.eql?(:success)}
33
+ return errors.empty? ? Transfer::Result.success : Transfer::Result.error(errors.first)
20
34
  end
21
35
 
22
36
  private
@@ -15,17 +15,62 @@ module Aspera
15
15
  # delay between each try to start connect
16
16
  SLEEP_SEC_BETWEEN_RETRY = 5
17
17
  private_constant :CONNECT_START_URIS, :SLEEP_SEC_BETWEEN_RETRY
18
+
19
+ class << self
20
+ # Re-query a previously started transfer via the Connect REST API.
21
+ # agent_params must contain 'app_id'; the Connect URL is auto-discovered.
22
+ # @return [Hash] normalized status hash
23
+ def transfer_status(transfer_id, agent_params)
24
+ connect_api = Rest.new(
25
+ base_url: "#{connect_api_url}/v5/connect",
26
+ headers: {'Origin' => RestParameters.instance.user_agent}
27
+ )
28
+ tr_info = connect_api.create("transfers/info/#{transfer_id}", {'aspera_connect_settings' => {'app_id' => agent_params['app_id']}})
29
+ transfer = tr_info['transfer_info']
30
+ return {'status' => 'running', 'bytes_transferred' => 0} unless transfer.is_a?(Hash)
31
+ normalize_status(transfer)
32
+ end
33
+
34
+ # Normalize a Connect transfer_info hash into the standard async-transfer fields.
35
+ def normalize_status(transfer)
36
+ result = {'bytes_transferred' => transfer['bytes_written'].to_i}
37
+ case transfer['status']
38
+ when 'completed'
39
+ result['status'] = 'completed'
40
+ result['ended_at'] = Time.now.utc.iso8601
41
+ when 'failed'
42
+ result.merge!('status' => 'failed', 'ended_at' => Time.now.utc.iso8601, 'error' => transfer['error_desc'].to_s)
43
+ when 'cancelled'
44
+ result['status'] = 'cancelled'
45
+ result['ended_at'] = Time.now.utc.iso8601
46
+ else
47
+ result['status'] = 'running'
48
+ end
49
+ result
50
+ end
51
+
52
+ # Auto-discover the Connect API base URL from its local URI file.
53
+ def connect_api_url
54
+ folder = File.join(Products::Other.find(Products::Connect.locations).first[:run_root], 'var', 'run')
55
+ ['', 's'].each do |ext|
56
+ uri_file = File.join(folder, "http#{ext}.uri")
57
+ return File.open(uri_file, &:gets).strip if File.exist?(uri_file)
58
+ end
59
+ raise "no connect uri file found in #{folder}"
60
+ end
61
+ end
62
+
18
63
  def initialize(**base_options)
19
64
  super
20
65
  @transfer_id = nil
21
66
  @connect_settings = {
22
67
  'app_id' => SecureRandom.uuid
23
68
  }
24
- raise Error, 'Using connect requires a graphical environment' unless Environment.instance.graphical?
69
+ Aspera.assert(Environment.instance.graphical?, type: Error){'Using connect requires a graphical environment'}
25
70
  method_index = 0
26
71
  begin
27
72
  # raise exception if connect not started and file does not exist
28
- connect_url = connect_api_url
73
+ connect_url = self.class.connect_api_url
29
74
  Log.log.debug{"found: #{connect_url}"}
30
75
  @connect_api = Rest.new(
31
76
  base_url: "#{connect_url}/v5/connect", # could use v6 also now
@@ -38,7 +83,7 @@ module Aspera
38
83
  Log.log.debug{"Exception: #{e}"}
39
84
  start_url = CONNECT_START_URIS[method_index]
40
85
  method_index += 1
41
- raise StandardError, "Unable to start connect #{method_index} times" if start_url.nil?
86
+ Aspera.assert(!start_url.nil?){"Unable to start connect #{method_index} times"}
42
87
  Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..."}
43
88
  Environment.instance.open_uri_graphical(start_url)
44
89
  sleep(SLEEP_SEC_BETWEEN_RETRY)
@@ -46,6 +91,11 @@ module Aspera
46
91
  end
47
92
  end
48
93
 
94
+ # Returns the app_id so callers can persist it in agent_params.
95
+ def app_id
96
+ @connect_settings['app_id']
97
+ end
98
+
49
99
  # :reek:UnusedParameters token_regenerator
50
100
  def start_transfer(transfer_spec, token_regenerator: nil)
51
101
  if transfer_spec['direction'] == 'send'
@@ -131,19 +181,6 @@ module Aspera
131
181
  end
132
182
  return [:success]
133
183
  end
134
-
135
- private
136
-
137
- # @return the file path of local connect where API's URI can be read
138
- def connect_api_url
139
- folder = File.join(Products::Other.find(Products::Connect.locations).first[:run_root], 'var', 'run')
140
- ['', 's'].each do |ext|
141
- uri_file = File.join(folder, "http#{ext}.uri")
142
- Log.log.debug{"checking connect port file: #{uri_file}"}
143
- return File.open(uri_file, &:gets).strip if File.exist?(uri_file)
144
- end
145
- raise "no connect uri file found in #{folder}"
146
- end
147
184
  end
148
185
  end
149
186
  end
@@ -3,7 +3,7 @@
3
3
  require 'aspera/agent/base'
4
4
  require 'aspera/rest'
5
5
  require 'aspera/environment'
6
- require 'aspera/json_rpc'
6
+ require 'aspera/json_rpc/client'
7
7
  require 'aspera/products/desktop'
8
8
  require 'aspera/transfer/spec'
9
9
  require 'securerandom'
@@ -18,23 +18,66 @@ module Aspera
18
18
  SLEEP_SEC_BETWEEN_RETRY = 5
19
19
  private_constant :START_URIS, :SLEEP_SEC_BETWEEN_RETRY
20
20
 
21
+ class << self
22
+ # Re-query a previously started transfer from the Desktop JSON-RPC daemon.
23
+ # agent_params must contain 'application_id'; the JSON-RPC URL is auto-discovered.
24
+ # @return [Hash] normalized status hash
25
+ def transfer_status(transfer_id, agent_params)
26
+ app_id = agent_params['application_id']
27
+ client = Aspera::JsonRpc::Client.new(Aspera::Rest.new(base_url: desktop_api_url))
28
+ raw = client.get_transfer(app_id: app_id, transfer_id: transfer_id)
29
+ normalize_status(raw['status'], bytes: raw['bytes_written'].to_i, error: raw['error_desc'])
30
+ end
31
+
32
+ # Normalize a raw status string into the standard async-transfer hash fields.
33
+ def normalize_status(raw_status, bytes: 0, error: nil)
34
+ result = {'bytes_transferred' => bytes}
35
+ case raw_status
36
+ when 'completed'
37
+ result['status'] = 'completed'
38
+ result['ended_at'] = Time.now.utc.iso8601
39
+ when 'failed'
40
+ result.merge!('status' => 'failed', 'ended_at' => Time.now.utc.iso8601, 'error' => error.to_s)
41
+ when 'cancelled'
42
+ result['status'] = 'cancelled'
43
+ result['ended_at'] = Time.now.utc.iso8601
44
+ else
45
+ result['status'] = 'running'
46
+ end
47
+ result
48
+ end
49
+
50
+ # Auto-discover the JSON-RPC URL from the Desktop log file (same logic as instance method).
51
+ def desktop_api_url
52
+ log_file = File.join(Products::Other.find(Products::Desktop.locations).first[:log_root], Products::Desktop::LOG_FILENAME)
53
+ url = 'http://127.0.0.1:33024'
54
+ File.open(log_file, 'r') do |file|
55
+ file.each_line do |line|
56
+ line = line.chomp
57
+ url = "http://#{Regexp.last_match(1)}" if line =~ /JSON-RPC server listening on (.*)/
58
+ end
59
+ end if File.exist?(log_file)
60
+ url
61
+ end
62
+ end
63
+
21
64
  def initialize(**base_options)
22
65
  @application_id = SecureRandom.uuid
23
66
  @transfer_id = nil
24
67
  super
25
- raise Error, 'Using client requires a graphical environment' unless Environment.instance.graphical?
68
+ Aspera.assert(Environment.instance.graphical?, type: Error){'Using client requires a graphical environment'}
26
69
  method_index = 0
27
70
  begin
28
71
  # curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
29
72
  # https://playground.open-rpc.org/?schemaUrl=http://127.0.0.1:33024
30
- @client_app_api = Aspera::JsonRpcClient.new(Aspera::Rest.new(base_url: aspera_client_api_url))
73
+ @client_app_api = Aspera::JsonRpc::Client.new(Aspera::Rest.new(base_url: aspera_client_api_url))
31
74
  client_info = @client_app_api.get_info
32
75
  Log.dump(:client_version, client_info)
33
76
  Log.log.debug('Client was reached') if method_index > 0
34
77
  rescue Errno::ECONNREFUSED => e
35
78
  start_url = START_URIS[method_index]
36
79
  method_index += 1
37
- raise StandardError, "Unable to start #{Products::Desktop::APP_NAME} #{method_index} times" if start_url.nil?
80
+ Aspera.assert(!start_url.nil?){"Unable to start #{Products::Desktop::APP_NAME} #{method_index} times"}
38
81
  Log.log.warn{"#{Products::Desktop::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
39
82
  Environment.instance.open_uri_graphical(start_url)
40
83
  sleep(SLEEP_SEC_BETWEEN_RETRY)
@@ -42,6 +85,9 @@ module Aspera
42
85
  end
43
86
  end
44
87
 
88
+ # Returns the application_id so callers can persist it in agent_params.
89
+ attr_reader :application_id
90
+
45
91
  # :reek:UnusedParameters token_regenerator
46
92
  def start_transfer(transfer_spec, token_regenerator: nil)
47
93
  Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
@@ -100,20 +146,9 @@ module Aspera
100
146
 
101
147
  private
102
148
 
103
- # @return [String] the url where transferd is listening
149
+ # Delegate to the class method (avoids duplication).
104
150
  def aspera_client_api_url
105
- log_file = File.join(Products::Other.find(Products::Desktop.locations).first[:log_root], Products::Desktop::LOG_FILENAME)
106
- url = 'http://127.0.0.1:33024'
107
- File.open(log_file, 'r') do |file|
108
- file.each_line do |line|
109
- line = line.chomp
110
- if (m = line.match(/JSON-RPC server listening on (.*)/))
111
- url = "http://#{m[1]}"
112
- end
113
- end
114
- end if File.exist?(log_file)
115
- # raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
116
- return url
151
+ self.class.desktop_api_url
117
152
  end
118
153
  end
119
154
  end
@@ -3,6 +3,7 @@
3
3
  require 'aspera/agent/base'
4
4
  require 'aspera/ascp/installation'
5
5
  require 'aspera/ascp/management'
6
+ require 'aspera/exec_spec'
6
7
  require 'aspera/transfer/parameters'
7
8
  require 'aspera/transfer/error'
8
9
  require 'aspera/transfer/spec'
@@ -24,12 +25,13 @@ module Aspera
24
25
  SELECT_AVAILABLE_PORT = 0
25
26
  private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
26
27
 
27
- # Options: same as values in option `transfer_info`
28
+ # Options: same as values in option `transfer`
28
29
  # (Args) : Options that influence `ascp` arguments.
29
30
  #
30
31
  # @param ascp_args [Array] (Args) Optional Additional arguments to ascp
31
32
  # @param wss [Boolean] (Args) `true`: if both SSH and wss in ts: prefer wss
32
33
  # @param quiet [Boolean] (Args) By default no native `ascp` progress bar
34
+ # @param file_list [Boolean] (Args) true: provide file list in a file, else on command line.
33
35
  # @param client_ssh_key [String] (Args) Client SSH key option (from CLIENT_SSH_KEY_OPTIONS)
34
36
  # @param trusted_certs [Array<String>] (Args) (WSS) Optional list of files with trusted certificates (stores)
35
37
  # @param check_ignore_cb [Proc] (Args) (WSS) Callback with host,port to check if WSS connection shall ignore certificate validity
@@ -44,6 +46,7 @@ module Aspera
44
46
  ascp_args: nil,
45
47
  wss: true,
46
48
  quiet: true,
49
+ file_list: true,
47
50
  client_ssh_key: nil,
48
51
  trusted_certs: nil,
49
52
  check_ignore_cb: nil,
@@ -61,6 +64,7 @@ module Aspera
61
64
  ascp_args: ascp_args,
62
65
  wss: wss,
63
66
  quiet: quiet,
67
+ file_list: file_list,
64
68
  trusted_certs: trusted_certs,
65
69
  client_ssh_key: client_ssh_key,
66
70
  check_ignore_cb: check_ignore_cb
@@ -135,7 +139,7 @@ module Aspera
135
139
  io: nil, # management port server socket
136
140
  token_regenerator: token_regenerator, # regenerate bearer token with oauth
137
141
  # env vars and args for ascp (from transfer spec)
138
- env_args: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
142
+ exec_spec: Transfer::Parameters.new(transfer_spec, **@tr_opts).ascp_args
139
143
  }
140
144
 
141
145
  if multi_session_info.nil?
@@ -151,12 +155,11 @@ module Aspera
151
155
  # do deep copy (each thread has its own copy because it is modified here below and in thread)
152
156
  this_session = session.clone
153
157
  this_session[:ts] = this_session[:ts].clone
154
- env_args = this_session[:env_args] = this_session[:env_args].clone
155
- args = env_args[:args] = env_args[:args].clone
158
+ exec_spec = this_session[:exec_spec] = this_session[:exec_spec].deep_clone
156
159
  # set multi session part
157
- args.unshift("-C#{i}:#{multi_session_info[:count]}")
160
+ exec_spec.args.unshift("-C#{i}:#{multi_session_info[:count]}")
158
161
  # option: increment (default as per ascp manual) or not (cluster on other side ?)
159
- args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
162
+ exec_spec.args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
160
163
  # finally start the thread
161
164
  this_session[:thread] = Thread.new{transfer_thread_entry(this_session)}
162
165
  @sessions.push(this_session)
@@ -166,7 +169,7 @@ module Aspera
166
169
  end
167
170
 
168
171
  # wait for completion of all jobs started
169
- # @return list of :success or error message
172
+ # @return [Array] list of :success or error message
170
173
  def wait_for_transfers_completion
171
174
  Log.log.debug('wait_for_transfers_completion')
172
175
  # set to non-nil to exit loop
@@ -216,7 +219,7 @@ module Aspera
216
219
  private
217
220
 
218
221
  # transfer thread entry
219
- # @param session information
222
+ # @param session [Hash] transfer session information
220
223
  def transfer_thread_entry(session)
221
224
  begin
222
225
  # set name for logging
@@ -224,7 +227,7 @@ module Aspera
224
227
  Log.log.debug{"ENTER (#{Thread.current[:name]})"}
225
228
  # start transfer with selected resumer policy
226
229
  @resume_policy.execute_with_resume do
227
- start_and_monitor_process(session: session, **session[:env_args])
230
+ start_and_monitor_process(session: session, exec_spec: session[:exec_spec])
228
231
  end
229
232
  Log.log.debug('transfer ok'.bg_green)
230
233
  rescue => e
@@ -239,18 +242,17 @@ module Aspera
239
242
  # This is the low level method to start the transfer process.
240
243
  # Typically started in a thread.
241
244
  # Start process with management port.
242
- # @param session [Hash] This session information, keys :io and :token_regenerator
243
- # @param name [Symbol] Name of executable: :ascp, :ascp4 or :async (comes from ascp_args)
244
- # @param env [Hash] Environment variables (comes from ascp_args)
245
- # @param args [Array] Command line arguments (comes from ascp_args)
245
+ # @param session [Hash] This session information, keys :io and :token_regenerator
246
+ # @param exec_spec [ExecSpec] Executable, environment variables and arguments (comes from ascp_args)
246
247
  # @return [nil] when process has exited
247
- # @raise FaspError on error
248
+ # @raise [Transfer::FaspError] on error
248
249
  def start_and_monitor_process(
249
250
  session:,
250
- name:,
251
- env:,
252
- args:
251
+ exec_spec:
253
252
  )
253
+ exec = exec_spec.exec
254
+ env = exec_spec.env
255
+ args = exec_spec.args
254
256
  Aspera.assert_type(session, Hash)
255
257
  notify_progress(:sessions_init, info: 'starting')
256
258
  # Do not use `capture_stderr`
@@ -259,7 +261,7 @@ module Aspera
259
261
  spawn_args = {}
260
262
  command_pid = nil
261
263
  # Get location of command executable (ascp, async)
262
- command_path = Ascp::Installation.instance.path(name)
264
+ command_path = Ascp::Installation.instance.path(exec)
263
265
  command_arguments = [command_path]
264
266
  begin
265
267
  if @monitor
@@ -271,7 +273,7 @@ module Aspera
271
273
  # make port ready to accept connections, before starting ascp
272
274
  mgt_server_socket.listen(1)
273
275
  # build arguments and add mgt port
274
- command_arguments = if name.eql?(:async)
276
+ command_arguments = if exec.eql?(:async)
275
277
  [command_path, "--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
276
278
  else
277
279
  [command_path, '-M', mgt_server_socket.local_address.ip_port.to_s]
@@ -286,7 +288,7 @@ module Aspera
286
288
  command_pid = Environment.secure_execute(*command_arguments, mode: :background, env: env, **spawn_args)
287
289
  # close here, but still used in other process (pipe)
288
290
  stderr_w&.close
289
- notify_progress(:sessions_init, info: "waiting for #{name} to start")
291
+ notify_progress(:sessions_init, info: "waiting for #{exec} to start")
290
292
  # "ensure" block will wait for process
291
293
  return unless @monitor
292
294
  # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
@@ -327,7 +329,7 @@ module Aspera
327
329
  Log.log.debug('management io closed')
328
330
  # check that last status was received before process exit
329
331
  last_event = processor.last_event
330
- raise Transfer::Error, "No management event (#{last_event.class})" unless last_event.is_a?(Hash)
332
+ Aspera.assert_type(last_event, Hash, type: Transfer::Error){'management event'}
331
333
  case last_event['Type']
332
334
  when 'DONE'
333
335
  Log.log.trace1{'Graceful shutdown, DONE message received'}
@@ -359,16 +361,16 @@ module Aspera
359
361
  # process stderr of ascp
360
362
  stderr_flag = false
361
363
  stderr_r.each_line do |line|
362
- Log.log.error{"BEGIN stderr #{name}"} unless stderr_flag
364
+ Log.log.error{"BEGIN stderr #{exec}"} unless stderr_flag
363
365
  Log.log.error{line.chomp}
364
366
  stderr_flag = true
365
367
  end
366
- Log.log.error{"END stderr #{name}"} if stderr_flag
368
+ Log.log.error{"END stderr #{exec}"} if stderr_flag
367
369
  stderr_r.close
368
370
  end
369
371
  # status is nil if an exception occurred before starting command
370
372
  if !status&.success?
371
- message = "#{name} failed (#{status})"
373
+ message = "#{exec} failed (#{status})"
372
374
  # raise error only if there was not already an exception (`$ERROR_INFO`)
373
375
  raise Transfer::Error, message unless $ERROR_INFO
374
376
  # else display this message also, as main exception is already here
@@ -47,7 +47,7 @@ module Aspera
47
47
  end
48
48
 
49
49
  # Wait for completion of all jobs started
50
- # @return list of :success or error message
50
+ # @return [Array] list of :success or error message
51
51
  def wait_for_transfers_completion
52
52
  # well ... transfer was done in "start"
53
53
  return [:success]