aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -1,65 +1,186 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/environment'
4
- require 'aspera/agent/base'
5
- require 'aspera/products/transferd'
6
- require 'aspera/temp_file_manager'
7
- require 'aspera/transfer/spec'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../environment'
7
+ require_relative 'base'
8
+ require_relative '../products/transferd'
9
+ require_relative '../temp_file_manager'
10
+ require_relative '../transfer/error'
11
+ require_relative '../transfer/spec'
12
+ require_relative '../../transferd_services_pb'
8
13
  require 'json'
14
+ require 'time'
9
15
  require 'uri'
10
- require 'transferd_services_pb'
11
16
 
12
17
  module Aspera
13
18
  module Agent
14
19
  class Transferd < Base
20
+ # Sends the API key of the daemon (`authentication` of its configuration) in the metadata `api_key` of each call
21
+ class ApiKeyInterceptor < GRPC::ClientInterceptor
22
+ # @param api_key [String] API key
23
+ def initialize(api_key)
24
+ super()
25
+ @api_key = api_key
26
+ end
27
+
28
+ %i[request_response client_streamer server_streamer bidi_streamer].each do |rpc_type|
29
+ define_method(rpc_type) do |metadata: nil, **_rest, &block|
30
+ metadata['api_key'] = @api_key
31
+ block.call
32
+ end
33
+ end
34
+ end
35
+
15
36
  # https://github.com/grpc/grpc/blob/master/doc/naming.md
16
37
  # https://grpc.io/docs/guides/custom-name-resolution/
17
38
  LOCAL_SOCKET_ADDR = '127.0.0.1'
18
39
  PORT_SEP = ':'
19
40
  # port zero means select a random available high port
20
- AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
41
+ AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0".freeze
42
+ # Default timeout in seconds of the calls of `transfer_status` and `transfer_cancel` (`agent_params['timeout']`)
43
+ CALL_TIMEOUT_SEC = 10
44
+ # Status of the daemon → status of an asynchronous transfer (as other agents): other statuses are `running`
45
+ ASYNC_STATUS = {
46
+ COMPLETED: 'completed',
47
+ FAILED: 'failed',
48
+ CANCELED: 'cancelled',
49
+ UNKNOWN_STATUS: 'unknown'
50
+ }.freeze
51
+ # Statuses of the daemon after which a transfer does not change
52
+ FINAL_STATUSES = %i[COMPLETED FAILED CANCELED].freeze
53
+ # Time in the daemon is in microseconds, but some versions give milliseconds: values below this are milliseconds
54
+ MIN_USEC_EPOCH = 10**14
21
55
 
22
- private_constant :LOCAL_SOCKET_ADDR, :PORT_SEP, :AUTO_LOCAL_TCP_PORT
56
+ private_constant :LOCAL_SOCKET_ADDR, :PORT_SEP, :AUTO_LOCAL_TCP_PORT, :ASYNC_STATUS, :MIN_USEC_EPOCH
23
57
 
24
58
  class << self
25
- # Re-query a previously started transfer from a running transferd daemon.
26
- # agent_params must contain 'url' (the daemon endpoint, e.g. "127.0.0.1:12345").
27
- # @return [Hash] normalized status hash
59
+ # Client of a daemon
60
+ # @param url [String] Endpoint of the daemon, e.g. `127.0.0.1:55002`
61
+ # @param api_key [String, nil] API key, if the daemon has `authentication` enabled
62
+ # @param ca_file [String, nil] File with the certificate authority of the daemon (PEM), if the daemon has `tls` enabled
63
+ # @return [Transferd::Api::TransferService::Stub] gRPC client
64
+ def create_client(url, api_key: nil, ca_file: nil)
65
+ credentials = ca_file.nil? ? :this_channel_is_insecure : GRPC::Core::ChannelCredentials.new(File.read(ca_file))
66
+ return ::Transferd::Api::TransferService::Stub.new(url, credentials) if api_key.nil?
67
+ ::Transferd::Api::TransferService::Stub.new(url, credentials, interceptors: [ApiKeyInterceptor.new(api_key)])
68
+ end
69
+
70
+ # @param agent_params [Hash] `url`: endpoint of the daemon; optional `api_key`, `ca_file`
71
+ # @return [Transferd::Api::TransferService::Stub] Client of the daemon
72
+ def client(agent_params)
73
+ Aspera.assert(agent_params['url'].is_a?(String) && !agent_params['url'].empty?) { 'agent_params: missing url' }
74
+ create_client(agent_params['url'], api_key: agent_params['api_key'], ca_file: agent_params['ca_file'])
75
+ end
76
+
77
+ # Re-query a previously started transfer from a running transferd daemon (unary call `QueryTransfer`).
78
+ # The daemon keeps a transfer for some time after its end (`workers.transfer_max_age` of its configuration): then it is `unknown`.
79
+ # @param transfer_id [String] Identifier of transfer, given by the daemon (`transfer_id`)
80
+ # @param agent_params [Hash] `url`: endpoint of the daemon, e.g. `127.0.0.1:55002`; optional `timeout`: seconds, `api_key`, `ca_file`
81
+ # @return [Hash] Normalized status: `status` (`running`, `completed`, `failed`, `cancelled`, or `unknown` if the daemon cannot tell),
82
+ # `bytes_transferred`, and when known: `state` (status of the daemon), `bytes_written` (by the receiver, with the bytes of a resumed transfer),
83
+ # `files_completed`, `rate_kbps` (average), `target_rate_kbps`,
84
+ # `elapsed_sec`, `started_at`, `ended_at`, `error_code`, `error`
28
85
  def transfer_status(transfer_id, agent_params)
29
- client = ::Transferd::Api::TransferService::Stub.new(agent_params['url'], :this_channel_is_insecure)
30
- result = {'bytes_transferred' => 0}
31
- client.monitor_transfers(::Transferd::Api::RegistrationRequest.new(transferId: [transfer_id])) do |response|
32
- result = case response.status
33
- when :COMPLETED
34
- {
35
- 'status' => 'completed', 'ended_at' => Time.now.utc.iso8601,
36
- 'bytes_transferred' => response.transferInfo.bytesTransferred
37
- }
38
- when :FAILED, :CANCELED
39
- {
40
- 'status' => response.status == :FAILED ? 'failed' : 'cancelled',
41
- 'ended_at' => Time.now.utc.iso8601, 'bytes_transferred' => 0,
42
- 'error' => JSON.parse(response.message.to_s)['Description']
43
- }
44
- else
45
- {'status' => 'running', 'bytes_transferred' => response.transferInfo.bytesTransferred}
46
- end
47
- break
86
+ response = client(agent_params).query_transfer(::Transferd::Api::TransferInfoRequest.new(transferId: transfer_id), deadline: deadline(agent_params))
87
+ # An unknown transfer is not an exception: status is not set, and error has code 404
88
+ error = response.error&.description.to_s
89
+ return {'status' => 'unknown', 'bytes_transferred' => 0, 'error' => error} if response.status.eql?(:UNKNOWN_STATUS) && !error.empty?
90
+ normalize_status(response)
91
+ rescue GRPC::BadStatus => e
92
+ # Daemon not reachable, or timeout: the transfer may still run
93
+ {'status' => 'unknown', 'bytes_transferred' => 0, 'error' => "#{e.class.name.split('::').last}: #{e.details}"}
94
+ end
95
+
96
+ # Information on a running daemon (unary call `GetInfo`)
97
+ # @param agent_params [Hash] `url`: endpoint of the daemon; optional `timeout`: seconds, `api_key`, `ca_file`
98
+ # @return [Hash] Version of the API (`apiVersion`) and information on the instance (`info`)
99
+ # @raise [Transfer::Error] The daemon is not reachable, or gave an error
100
+ def info(agent_params)
101
+ response = client(agent_params).get_info(::Transferd::Api::InstanceInfoRequest.new, deadline: deadline(agent_params))
102
+ error = response.error&.description.to_s
103
+ Aspera.assert(error.empty?, type: Transfer::Error) { error }
104
+ JSON.parse(response.to_h.except(:error).to_json)
105
+ rescue GRPC::BadStatus => e
106
+ raise Transfer::Error, "#{e.class.name.split('::').last}: #{e.details}"
107
+ end
108
+
109
+ # Stop a transfer (call `StopTransfer`)
110
+ # @param transfer_id [String] Identifier of transfer, given by the daemon
111
+ # @param agent_params [Hash] `url`: endpoint of the daemon; optional `timeout`: seconds, `api_key`, `ca_file`
112
+ # @return [Boolean] `true`: stopped
113
+ # @raise [Transfer::Error] The daemon did not stop it: unknown or terminated transfer
114
+ def transfer_cancel(transfer_id, agent_params)
115
+ response = client(agent_params).stop_transfer(::Transferd::Api::StopTransferRequest.new(transferId: [transfer_id]), deadline: deadline(agent_params))
116
+ result = response.stopResult.find { |item| item.transferId.eql?(transfer_id) }
117
+ Aspera.assert(!result.nil?, type: Transfer::Error) { "transfer id=#{transfer_id}: no result of stop" }
118
+ Aspera.assert(result.stopped, type: Transfer::Error) { result.error&.description || "transfer id=#{transfer_id}: not stopped" }
119
+ true
120
+ end
121
+
122
+ # @param response [Transferd::Api::QueryTransferResponse, Transferd::Api::TransferResponse] Response of the daemon
123
+ # @return [Hash] Normalized status, see `transfer_status`
124
+ def normalize_status(response)
125
+ info = response.transferInfo
126
+ result = {
127
+ 'status' => ASYNC_STATUS.fetch(response.status, 'running'),
128
+ 'state' => response.status.to_s.downcase,
129
+ 'bytes_transferred' => info&.bytesTransferred.to_i
130
+ }
131
+ return result if info.nil?
132
+ result.merge!(
133
+ 'bytes_written' => info.bytesWritten,
134
+ 'files_completed' => info.filesCompleted,
135
+ 'rate_kbps' => info.averageRateKbps,
136
+ 'target_rate_kbps' => info.targetRateKbps,
137
+ 'elapsed_sec' => info.elapsedUsec / 1_000_000.0,
138
+ 'started_at' => epoch_time(info.startTimeUsec),
139
+ 'ended_at' => epoch_time(info.endTimeUsec)
140
+ )
141
+ if FINAL_STATUSES.include?(response.status)
142
+ result['ended_at'] ||= Time.now.utc.iso8601
143
+ else
144
+ result.delete('ended_at')
48
145
  end
49
- result
50
- rescue StandardError => e
51
- {'status' => 'running', 'bytes_transferred' => 0, 'error' => e.message}
146
+ error = info.errorDescription.to_s.strip
147
+ error = response.error&.description.to_s if error.empty?
148
+ unless error.empty?
149
+ result['error_code'] = info.errorCode
150
+ result['error'] = error
151
+ end
152
+ result.compact
153
+ end
154
+
155
+ private
156
+
157
+ # @param agent_params [Hash] Optional `timeout`: seconds
158
+ # @return [Time] Deadline of a call
159
+ def deadline(agent_params)
160
+ Time.now + (agent_params['timeout'] || CALL_TIMEOUT_SEC).to_f
161
+ end
162
+
163
+ # @param value [Integer] Time since epoch, in microseconds or milliseconds, 0 if not known
164
+ # @return [String, nil] Time in ISO 8601, `nil` if not known
165
+ def epoch_time(value)
166
+ return if value.nil? || value <= 0
167
+ seconds = value >= MIN_USEC_EPOCH ? value / 1_000_000.0 : value / 1000.0
168
+ Time.at(seconds).utc.iso8601
52
169
  end
53
170
  end
54
171
 
55
172
  # @param url [String] URL of the transfer manager daemon
56
173
  # @param start [Boolean] If `false`, expect that an external daemon is already running
57
174
  # @param stop [Boolean] If `false`, do not shutdown daemon on exit
175
+ # @param api_key [String, nil] API key of an external daemon with `authentication` enabled
176
+ # @param ca_file [String, nil] Certificate authority (PEM file) of an external daemon with `tls` enabled
58
177
  # @param base [Hash] Base class options
59
178
  def initialize(
60
- url: AUTO_LOCAL_TCP_PORT,
61
- start: true,
62
- stop: true,
179
+ url: AUTO_LOCAL_TCP_PORT,
180
+ start: true,
181
+ stop: true,
182
+ api_key: nil,
183
+ ca_file: nil,
63
184
  **base
64
185
  )
65
186
  super(**base)
@@ -76,14 +197,15 @@ module Aspera
76
197
  begin
77
198
  # no address: local bind
78
199
  daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{daemon_endpoint}" if daemon_endpoint.match?(/^#{PORT_SEP}[0-9]+$/o)
79
- # Create stub (without credentials)
80
- @transfer_client = ::Transferd::Api::TransferService::Stub.new(daemon_endpoint, :this_channel_is_insecure)
200
+ # Create stub: no credentials for a daemon started here, API key and TLS only for an external daemon
201
+ @transfer_client = self.class.create_client(daemon_endpoint, api_key: api_key, ca_file: ca_file)
81
202
  # Initiate actual connection
82
203
  get_info_response = @transfer_client.get_info(::Transferd::Api::InstanceInfoRequest.new)
83
204
  @daemon_endpoint = daemon_endpoint
84
205
  Log.log.debug { "Daemon info: #{get_info_response}" }
85
206
  Log.log.warn('Attached to existing daemon') unless @daemon_pid || !start || !@stop
86
- at_exit { shutdown }
207
+ # `shutdown` is registered with `at_exit` only when the daemon is started here:
208
+ # a long-running process (e.g. Orchestrator) that attaches to an external daemon for each transfer would accumulate them
87
209
  rescue GRPC::Unavailable => e
88
210
  # if transferd is external: do not start it, or other error
89
211
  raise if !start || !e.message.include?('failed to connect')
@@ -122,14 +244,14 @@ module Aspera
122
244
  # wait for process to initialize, max 2 seconds
123
245
  Timeout.timeout(2.0) do
124
246
  # this returns if process dies (within 2 seconds)
125
- _, status = Process.wait2(@daemon_pid)
247
+ _, status = ::Process.wait2(@daemon_pid)
126
248
  raise "Transfer daemon exited with status #{status.exitstatus}. Check files: #{log_stdout} and #{log_stderr}"
127
249
  end
128
250
  rescue Timeout::Error
129
251
  nil
130
252
  end
131
253
  Log.log.debug { "Daemon started with pid #{@daemon_pid}" }
132
- Process.detach(@daemon_pid) unless @stop
254
+ ::Process.detach(@daemon_pid) unless @stop
133
255
  at_exit { shutdown }
134
256
  # update port for next connection attempt (if auto high port was requested)
135
257
  daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
@@ -140,6 +262,8 @@ module Aspera
140
262
 
141
263
  # Actual endpoint the daemon is listening on (resolved after connect, e.g. when port 0 was used)
142
264
  attr_reader :daemon_endpoint
265
+ # Identifier of the last transfer started, given by the daemon: for `transfer_status` and `transfer_cancel`
266
+ attr_reader :transfer_id
143
267
  # gRPC client of the daemon
144
268
  attr_reader :transfer_client
145
269
 
@@ -206,8 +330,8 @@ module Aspera
206
330
  def stop_daemon
207
331
  if !@daemon_pid.nil?
208
332
  Log.log.debug("Stopping daemon #{@daemon_pid}")
209
- Process.kill(:KILL, @daemon_pid)
210
- _, status = Process.wait2(@daemon_pid)
333
+ ::Process.kill(:KILL, @daemon_pid)
334
+ _, status = ::Process.wait2(@daemon_pid)
211
335
  Log.log.debug("daemon stopped #{status}")
212
336
  @daemon_pid = nil
213
337
  end
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/api/aoc'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'aoc'
4
7
  module Aspera
5
8
  module Api
6
9
  # Aspera License Entitlement Engine API client
@@ -1,15 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'aspera/rest'
6
- require 'aspera/hash_ext'
7
- require 'aspera/data_repository'
8
- require 'aspera/transfer/spec'
9
- require 'aspera/api/node'
10
- require 'aspera/rest/list'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative '../rest'
9
+ require_relative '../hash_ext'
10
+ require_relative '../data_repository'
11
+ require_relative '../transfer/spec'
12
+ require_relative 'node'
13
+ require_relative '../rest/list'
14
+ require_relative '../rainbow'
11
15
  require 'base64'
12
- require 'aspera/rainbow'
13
16
  using Rainbow
14
17
 
15
18
  module Aspera
@@ -62,7 +65,7 @@ module Aspera
62
65
  # `''` Use default workspace if it is set, else none
63
66
  DEFAULT_WORKSPACE = ''
64
67
  # Production domain of AoC
65
- SAAS_DOMAIN_PROD = 'ibmaspera.com' # cspell:disable-line
68
+ SAAS_DOMAIN_PROD = 'ibmaspera.com'
66
69
  # To avoid infinite loop in pub link redirection
67
70
  MAX_AOC_URL_REDIRECT = 10
68
71
  CLIENT_ID_PREFIX = 'aspera.'
@@ -103,7 +106,7 @@ module Aspera
103
106
  # Scope `admin:all`
104
107
  ADMIN = 'admin:all'
105
108
  ADMIN_USER = 'admin-user:all'
106
- ADMIN_USER_USER = "#{ADMIN_USER}+#{USER}"
109
+ ADMIN_USER_USER = "#{ADMIN_USER}+#{USER}".freeze
107
110
  end
108
111
  API_V1 = 'api/v1'
109
112
 
@@ -1,7 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/rest'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../rest'
5
7
 
6
8
  module Aspera
7
9
  module Api
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'aspera/api/node'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative 'node'
6
9
  require 'xmlsimple'
7
10
 
8
11
  module Aspera
@@ -12,7 +15,7 @@ module Aspera
12
15
  IBM_CLOUD_TOKEN_URL = 'https://iam.cloud.ibm.com/identity'
13
16
  TOKEN_FIELD = 'delegated_refresh_token'
14
17
  FASP_INFO_KEYS = %w[ATSEndpoint AccessKey].freeze
15
- SERVICE_CREDS_PARAMS = %w[apikey resource_instance_id endpoints]
18
+ SERVICE_CREDS_PARAMS = %w[apikey resource_instance_id endpoints].freeze
16
19
  class << self
17
20
  def parameters_from_svc_credentials(service_credentials, bucket_region)
18
21
  # check necessary contents
@@ -62,7 +65,7 @@ module Aspera
62
65
  ret: :resp
63
66
  ).body
64
67
  ats_info = XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
65
- Log.dump(:ats_info, ats_info.reject { |k, _| k == 'AccessKey' })
68
+ Log.dump(:ats_info, ats_info)
66
69
  Aspera.assert_hash_all(ats_info, String, nil) { 'ats_info' }
67
70
  Aspera.assert((FASP_INFO_KEYS - ats_info.keys).empty?, 'ats_info missing required keys')
68
71
  Aspera.assert_hash_all(ats_info['AccessKey'], String, String) { 'ats_info' }
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
4
- require 'aspera/rest'
5
- require 'aspera/oauth/base'
6
- require 'aspera/rest/list'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+ require_relative '../rest'
8
+ require_relative '../oauth/client/base'
9
+ require_relative '../rest/list'
7
10
  require 'digest'
8
11
 
9
12
  module Aspera
@@ -94,7 +97,7 @@ module Aspera
94
97
  # states for jobs not in final state
95
98
  JOB_RUNNING = %w[queued working].freeze
96
99
  PATH_STANDARD_ROOT = '/aspera/faspex'
97
- PATH_API_DETECT = "#{PATH_API_V5}/#{PATH_HEALTH}"
100
+ PATH_API_DETECT = "#{PATH_API_V5}/#{PATH_HEALTH}".freeze
98
101
  HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
99
102
  HEADER_FASPEX_VERSION = 'X-IBM-Aspera'
100
103
  EMAIL_NOTIF_LIST = %w[
@@ -125,7 +128,7 @@ module Aspera
125
128
  sync_failed
126
129
  account_exist
127
130
  mfa_code
128
- ]
131
+ ].freeze
129
132
  class << self
130
133
  # @return [Boolean] true if the URL is a public link
131
134
  def public_link?(url)
@@ -146,7 +149,7 @@ module Aspera
146
149
  # @param root [String] Root path for API
147
150
  # @param password [String, nil] For :boot auth, the token copied directly from browser in developer mode
148
151
  # @param client_id [String, nil] For :web and :jwt auth, the client_id of web UI application
149
- # @param client_secret [String, nil] For :web auth, the client_secret of web UI application (not needed for :jwt)
152
+ # @param client_secret [String, nil] For :web auth, the client_secret of the API client, required if it has HTTP basic auth enabled (not needed for :jwt)
150
153
  # @param redirect_uri [String, nil] For :web auth, the redirect_uri of web UI application (must be the same as in application configuration)
151
154
  # @param username [String, nil] For :jwt auth, the username of the user to impersonate
152
155
  # @param private_key [String, nil] For :jwt auth, the private key to sign JWT token
@@ -215,9 +218,13 @@ module Aspera
215
218
  type: :oauth2,
216
219
  base_url: "#{url}/#{PATH_AUTH}",
217
220
  grant_method: :web,
221
+ # Faspex 5.0.16+ checks the code verifier, earlier versions ignore it
222
+ pkce: true,
218
223
  params: {
219
- client_id: client_id
220
- },
224
+ client_id: client_id,
225
+ # required when the API client has HTTP basic auth enabled, else ignored by Faspex
226
+ client_secret: client_secret
227
+ }.compact,
221
228
  redirect_uri: redirect_uri
222
229
  }
223
230
  }
@@ -1,15 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/rest'
5
- require 'aspera/transfer/faux_file'
6
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../rest'
8
+ require_relative '../transfer/faux_file'
9
+ require_relative '../assert'
10
+ require_relative '../rainbow'
7
11
  require 'net/protocol'
8
12
  require 'securerandom'
9
13
  require 'websocket'
10
14
  require 'base64'
11
15
  require 'json'
12
- require 'aspera/rainbow'
13
16
  using Rainbow
14
17
 
15
18
  # Throw exception on error, instead of error code
@@ -209,7 +212,7 @@ module Aspera
209
212
  fileIndex: file_index
210
213
  }
211
214
  file = file_to_send[:file]
212
- file = File.open(file) unless file.is_a?(Transfer::FauxFile) # rubocop:disable Style/FileOpen
215
+ file = File.open(file) unless file.is_a?(Transfer::FauxFile)
213
216
  begin
214
217
  until file.eof?
215
218
  slice_bin_data = file.read(@upload_chunk_size)
@@ -1,18 +1,22 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/link_header'
4
- require 'aspera/transfer/spec'
5
- require 'aspera/rest'
6
- require 'aspera/oauth'
7
- require 'aspera/log'
8
- require 'aspera/assert'
9
- require 'aspera/environment'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../link_header'
7
+ require_relative '../transfer/spec'
8
+ require_relative '../rest'
9
+ require_relative '../oauth/client/factory'
10
+ require_relative '../log'
11
+ require_relative '../assert'
12
+ require_relative '../secret/hider'
13
+ require_relative '../environment'
14
+ require_relative '../rainbow'
10
15
  require 'base64'
11
16
  require 'openssl'
12
17
  require 'pathname'
13
18
  require 'zlib'
14
19
  require 'net/ssh/buffer'
15
- require 'aspera/rainbow'
16
20
  using Rainbow
17
21
 
18
22
  module Aspera
@@ -173,12 +177,15 @@ module Aspera
173
177
  payload['auth_type'] ||= 'access_key'
174
178
  payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
175
179
  payload_json = JSON.generate(payload)
176
- return Base64.strict_encode64(Zlib::Deflate.deflate([
180
+ token = Base64.strict_encode64(Zlib::Deflate.deflate([
177
181
  payload_json,
178
182
  SIGNATURE_DELIMITER,
179
183
  Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
180
184
  ''
181
185
  ].join("\n")))
186
+ # Hidden wherever it appears (logs, results)
187
+ Secret::Hider.instance.add_secret_values(token)
188
+ return token
182
189
  end
183
190
 
184
191
  # Decode an Aspera Node bearer token
@@ -288,8 +295,10 @@ module Aspera
288
295
  if use_v4
289
296
  headers[HEADER_ACCEPT_VERSION] = '4.0'
290
297
  query['per_page'] = 1000 unless query.key?('per_page')
291
- elsif query.key?('per_page') && !query.key?('page')
292
- query['page'] = 0
298
+ else
299
+ # API v3 requires both `page` (first is 1) and `per_page`
300
+ query['page'] = 1 unless query.key?('page')
301
+ query['per_page'] = 1000 unless query.key?('per_page')
293
302
  end
294
303
  loop do
295
304
  Rest::Parameters.instance.spinner_cb.call(folder_items.count)
@@ -300,8 +309,9 @@ module Aspera
300
309
  break if iteration_token.nil? || iteration_token.empty?
301
310
  query['iteration_token'] = iteration_token
302
311
  else
303
- break if data['item_count'].eql?(0)
304
- query['offset'] += data['item_count']
312
+ # Last page: empty, or shorter than requested
313
+ break if data.empty? || (query.key?('per_page') && data.length < query['per_page'].to_i)
314
+ query['page'] = query['page'].to_i + 1
305
315
  end
306
316
  end
307
317
  rescue StandardError => e
@@ -446,11 +456,6 @@ module Aspera
446
456
  return find_state[:found]
447
457
  end
448
458
 
449
- # Generate a refreshed auth token
450
- def refreshed_transfer_token
451
- return oauth.authorization(refresh: true)
452
- end
453
-
454
459
  # Get a base download transfer spec (gen3)
455
460
  # @return [Hash] Base transfer spec
456
461
  def base_spec
@@ -480,7 +485,6 @@ module Aspera
480
485
  ak_token = Rest.basic_authorization(auth_params[:username], auth_params[:password])
481
486
  when :oauth2
482
487
  ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
483
- # TODO: token_generation_lambda = lambda{|do_refresh|oauth.authorization(refresh: do_refresh)}
484
488
  # Get bearer token, possibly use cache
485
489
  ak_token = oauth.authorization
486
490
  when :none
data/lib/aspera/ascmd.rb CHANGED
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore ascmd smode errstr zstr zmode zuid zgid zctime zatime zmtime fcount dcount btype blist codeset lc_ctype ascmdtypes
4
- require 'aspera/log'
5
- require 'aspera/assert'
6
- require 'aspera/rainbow'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:ignore smode errstr zstr zmode zuid zgid zctime zatime zmtime fcount dcount btype blist ascmdtypes
7
+ require_relative 'log'
8
+ require_relative 'assert'
9
+ require_relative 'rainbow'
7
10
  using Rainbow
8
11
 
9
12
  module Aspera
@@ -193,7 +196,7 @@ module Aspera
193
196
  num_bytes = type_name.eql?(:zstr) ? buffer.length : type_descr[:size]
194
197
  Aspera.assert(buffer.length >= num_bytes, 'not enough bytes')
195
198
  byte_array = buffer.shift(num_bytes)
196
- byte_array = [byte_array] unless byte_array.is_a?(Array)
199
+ byte_array = Array(byte_array)
197
200
  result = byte_array.pack('C*').unpack1(type_descr[:unpack])
198
201
  result.force_encoding('UTF-8') if type_name.eql?(:zstr)
199
202
  Log.log.trace1 { "#{' .' * indent_level}-> base:#{byte_array} -> #{result}" }