aspera-cli 4.28.0 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +143 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1633 -609
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +20 -20
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +71 -65
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +16 -8
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +12 -19
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -33
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +6 -1
  50. data/lib/aspera/cli/command_spec.rb +34 -7
  51. data/lib/aspera/cli/completion/ascli.bash +19 -5
  52. data/lib/aspera/cli/completion/ascli.fish +13 -2
  53. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  54. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  55. data/lib/aspera/cli/completion_actions.rb +303 -69
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +3 -0
  59. data/lib/aspera/cli/extended_value.rb +61 -27
  60. data/lib/aspera/cli/formatter.rb +29 -22
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -10
  63. data/lib/aspera/cli/http.rb +12 -9
  64. data/lib/aspera/cli/info.rb +12 -7
  65. data/lib/aspera/cli/mailer.rb +6 -3
  66. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  67. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  68. data/lib/aspera/cli/mcp/tool.rb +15 -14
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +65 -12
  73. data/lib/aspera/cli/options.schema.yaml +487 -63
  74. data/lib/aspera/cli/parser.rb +56 -32
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +244 -35
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +17 -12
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +198 -103
  81. data/lib/aspera/cli/plugins/console.rb +6 -3
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +21 -7
  84. data/lib/aspera/cli/plugins/faspex5.rb +28 -25
  85. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  86. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +28 -22
  88. data/lib/aspera/cli/plugins/node.rb +31 -30
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +63 -11
  93. data/lib/aspera/cli/plugins/shares.rb +86 -16
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +144 -30
  96. data/lib/aspera/cli/preset_manager.rb +52 -18
  97. data/lib/aspera/cli/prompt.rb +4 -1
  98. data/lib/aspera/cli/result.rb +12 -8
  99. data/lib/aspera/cli/runner.rb +69 -46
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +79 -9
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +3 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +29 -20
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +188 -42
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.rb +12 -10
  113. data/lib/aspera/cli/wizard.rb +42 -39
  114. data/lib/aspera/command_line_builder.rb +9 -5
  115. data/lib/aspera/command_line_converter.rb +4 -1
  116. data/lib/aspera/coverage.rb +103 -33
  117. data/lib/aspera/data_repository.rb +4 -1
  118. data/lib/aspera/dot_container.rb +4 -1
  119. data/lib/aspera/duration.rb +5 -2
  120. data/lib/aspera/environment.rb +21 -12
  121. data/lib/aspera/exec_spec.rb +3 -0
  122. data/lib/aspera/faspex_gw.rb +13 -9
  123. data/lib/aspera/faspex_postproc.rb +15 -12
  124. data/lib/aspera/formatter_interface.rb +4 -1
  125. data/lib/aspera/graphql.rb +5 -1
  126. data/lib/aspera/hash_ext.rb +3 -0
  127. data/lib/aspera/id_generator.rb +4 -2
  128. data/lib/aspera/json_rpc/client.rb +6 -5
  129. data/lib/aspera/json_rpc/version.rb +3 -0
  130. data/lib/aspera/line_logger.rb +4 -1
  131. data/lib/aspera/link_header.rb +4 -1
  132. data/lib/aspera/log.rb +18 -5
  133. data/lib/aspera/markdown.rb +5 -2
  134. data/lib/aspera/mime.rb +3 -0
  135. data/lib/aspera/nagios.rb +6 -4
  136. data/lib/aspera/node_emulator.rb +15 -13
  137. data/lib/aspera/oauth/client/base.rb +91 -70
  138. data/lib/aspera/oauth/client/boot.rb +7 -4
  139. data/lib/aspera/oauth/client/factory.rb +6 -3
  140. data/lib/aspera/oauth/client/generic.rb +5 -1
  141. data/lib/aspera/oauth/client/json_credentials.rb +4 -1
  142. data/lib/aspera/oauth/client/jwt.rb +6 -3
  143. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  144. data/lib/aspera/oauth/client/url_json.rb +4 -1
  145. data/lib/aspera/oauth/client/web.rb +39 -12
  146. data/lib/aspera/oauth/server/pkce.rb +6 -5
  147. data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
  148. data/lib/aspera/persistency_action_once.rb +5 -3
  149. data/lib/aspera/persistency_folder.rb +6 -3
  150. data/lib/aspera/preview/file_types.rb +6 -3
  151. data/lib/aspera/preview/generator.rb +9 -5
  152. data/lib/aspera/preview/options.schema.yaml +3 -3
  153. data/lib/aspera/preview/terminal.rb +8 -4
  154. data/lib/aspera/preview/utils.rb +6 -4
  155. data/lib/aspera/products/connect.rb +5 -2
  156. data/lib/aspera/products/desktop.rb +4 -1
  157. data/lib/aspera/products/other.rb +4 -2
  158. data/lib/aspera/products/transferd.rb +6 -2
  159. data/lib/aspera/products/transferd_service.rb +610 -0
  160. data/lib/aspera/proxy_auto_config.js +2 -2
  161. data/lib/aspera/proxy_auto_config.rb +5 -2
  162. data/lib/aspera/rainbow.rb +3 -0
  163. data/lib/aspera/rest/aspera_errors.rb +4 -1
  164. data/lib/aspera/rest/call_error.rb +3 -0
  165. data/lib/aspera/rest/client.rb +22 -29
  166. data/lib/aspera/rest/error_analyzer.rb +5 -2
  167. data/lib/aspera/rest/list.rb +8 -4
  168. data/lib/aspera/rest/parameters.rb +3 -0
  169. data/lib/aspera/rest/util.rb +8 -5
  170. data/lib/aspera/rest.rb +9 -6
  171. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  172. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  173. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  174. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  175. data/lib/aspera/schema/async_tables.yaml +11 -10
  176. data/lib/aspera/schema/documentation.rb +8 -2
  177. data/lib/aspera/schema/reader.rb +7 -0
  178. data/lib/aspera/schema/registry.rb +43 -28
  179. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  180. data/lib/aspera/schema/validator.rb +4 -1
  181. data/lib/aspera/sealed_hash.rb +166 -0
  182. data/lib/aspera/secret/hider.rb +201 -0
  183. data/lib/aspera/secret/reference.rb +150 -0
  184. data/lib/aspera/secret/resolver.rb +212 -0
  185. data/lib/aspera/secret/store/base.rb +153 -0
  186. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  187. data/lib/aspera/secret/store/factory.rb +103 -0
  188. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  189. data/lib/aspera/secret/store/keeper.rb +141 -0
  190. data/lib/aspera/secret/store/macos_security.rb +284 -0
  191. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  192. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  193. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  194. data/lib/aspera/secret/store/secret_service.rb +125 -0
  195. data/lib/aspera/secret/strength.rb +119 -0
  196. data/lib/aspera/servlet_helpers.rb +5 -2
  197. data/lib/aspera/ssh.rb +10 -7
  198. data/lib/aspera/ssl.rb +5 -2
  199. data/lib/aspera/string_ext.rb +3 -0
  200. data/lib/aspera/sync/args.schema.yaml +3 -0
  201. data/lib/aspera/sync/conf.schema.yaml +9 -4
  202. data/lib/aspera/sync/database.rb +3 -0
  203. data/lib/aspera/sync/operations.rb +16 -14
  204. data/lib/aspera/temp_file_manager.rb +8 -4
  205. data/lib/aspera/timer_limiter.rb +3 -0
  206. data/lib/aspera/transfer/error.rb +4 -1
  207. data/lib/aspera/transfer/faux_file.rb +4 -1
  208. data/lib/aspera/transfer/parameters.rb +17 -14
  209. data/lib/aspera/transfer/result.rb +3 -0
  210. data/lib/aspera/transfer/resumer.rb +6 -4
  211. data/lib/aspera/transfer/spec.rb +9 -22
  212. data/lib/aspera/transfer/units.rb +75 -0
  213. data/lib/aspera/uri_reader.rb +57 -7
  214. data/lib/aspera/web_auth/index.html.erb +6 -2
  215. data/lib/aspera/web_auth/paste.rb +109 -0
  216. data/lib/aspera/web_auth/receiver.rb +50 -0
  217. data/lib/aspera/web_auth/relay.rb +76 -0
  218. data/lib/aspera/web_auth/reply.html.erb +150 -0
  219. data/lib/aspera/web_auth/server.rb +47 -10
  220. data/lib/aspera/web_page.rb +7 -2
  221. data/lib/aspera/web_server_simple.rb +6 -3
  222. data/lib/aspera/yaml.rb +5 -1
  223. data/llms.txt +81 -0
  224. data/skills/ascli/SKILL.md +88 -0
  225. data.tar.gz.sig +0 -0
  226. metadata +72 -19
  227. metadata.gz.sig +0 -0
  228. data/lib/aspera/keychain/base.rb +0 -30
  229. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  230. data/lib/aspera/keychain/factory.rb +0 -58
  231. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  232. data/lib/aspera/keychain/macos_security.rb +0 -277
  233. data/lib/aspera/keychain/one_password_api.rb +0 -91
  234. data/lib/aspera/keychain/one_password_base.rb +0 -39
  235. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  236. data/lib/aspera/secret_hider.rb +0 -196
  237. data/lib/aspera/secret_strength.rb +0 -114
@@ -1,15 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'aspera/rest'
6
- require 'aspera/hash_ext'
7
- require 'aspera/data_repository'
8
- require 'aspera/transfer/spec'
9
- require 'aspera/api/node'
10
- require 'aspera/rest/list'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative '../rest'
9
+ require_relative '../hash_ext'
10
+ require_relative '../data_repository'
11
+ require_relative '../transfer/spec'
12
+ require_relative 'node'
13
+ require_relative '../rest/list'
14
+ require_relative '../rainbow'
11
15
  require 'base64'
12
- require 'aspera/rainbow'
13
16
  using Rainbow
14
17
 
15
18
  module Aspera
@@ -62,7 +65,7 @@ module Aspera
62
65
  # `''` Use default workspace if it is set, else none
63
66
  DEFAULT_WORKSPACE = ''
64
67
  # Production domain of AoC
65
- SAAS_DOMAIN_PROD = 'ibmaspera.com' # cspell:disable-line
68
+ SAAS_DOMAIN_PROD = 'ibmaspera.com'
66
69
  # To avoid infinite loop in pub link redirection
67
70
  MAX_AOC_URL_REDIRECT = 10
68
71
  CLIENT_ID_PREFIX = 'aspera.'
@@ -103,7 +106,7 @@ module Aspera
103
106
  # Scope `admin:all`
104
107
  ADMIN = 'admin:all'
105
108
  ADMIN_USER = 'admin-user:all'
106
- ADMIN_USER_USER = "#{ADMIN_USER}+#{USER}"
109
+ ADMIN_USER_USER = "#{ADMIN_USER}+#{USER}".freeze
107
110
  end
108
111
  API_V1 = 'api/v1'
109
112
 
@@ -1,7 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/rest'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../rest'
5
7
 
6
8
  module Aspera
7
9
  module Api
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'aspera/api/node'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative 'node'
6
9
  require 'xmlsimple'
7
10
 
8
11
  module Aspera
@@ -12,7 +15,7 @@ module Aspera
12
15
  IBM_CLOUD_TOKEN_URL = 'https://iam.cloud.ibm.com/identity'
13
16
  TOKEN_FIELD = 'delegated_refresh_token'
14
17
  FASP_INFO_KEYS = %w[ATSEndpoint AccessKey].freeze
15
- SERVICE_CREDS_PARAMS = %w[apikey resource_instance_id endpoints]
18
+ SERVICE_CREDS_PARAMS = %w[apikey resource_instance_id endpoints].freeze
16
19
  class << self
17
20
  def parameters_from_svc_credentials(service_credentials, bucket_region)
18
21
  # check necessary contents
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
4
- require 'aspera/rest'
5
- require 'aspera/oauth/client/base'
6
- require 'aspera/rest/list'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+ require_relative '../rest'
8
+ require_relative '../oauth/client/base'
9
+ require_relative '../rest/list'
7
10
  require 'digest'
8
11
 
9
12
  module Aspera
@@ -94,7 +97,7 @@ module Aspera
94
97
  # states for jobs not in final state
95
98
  JOB_RUNNING = %w[queued working].freeze
96
99
  PATH_STANDARD_ROOT = '/aspera/faspex'
97
- PATH_API_DETECT = "#{PATH_API_V5}/#{PATH_HEALTH}"
100
+ PATH_API_DETECT = "#{PATH_API_V5}/#{PATH_HEALTH}".freeze
98
101
  HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
99
102
  HEADER_FASPEX_VERSION = 'X-IBM-Aspera'
100
103
  EMAIL_NOTIF_LIST = %w[
@@ -125,7 +128,7 @@ module Aspera
125
128
  sync_failed
126
129
  account_exist
127
130
  mfa_code
128
- ]
131
+ ].freeze
129
132
  class << self
130
133
  # @return [Boolean] true if the URL is a public link
131
134
  def public_link?(url)
@@ -146,7 +149,7 @@ module Aspera
146
149
  # @param root [String] Root path for API
147
150
  # @param password [String, nil] For :boot auth, the token copied directly from browser in developer mode
148
151
  # @param client_id [String, nil] For :web and :jwt auth, the client_id of web UI application
149
- # @param client_secret [String, nil] For :web auth, the client_secret of web UI application (not needed for :jwt)
152
+ # @param client_secret [String, nil] For :web auth, the client_secret of the API client, required if it has HTTP basic auth enabled (not needed for :jwt)
150
153
  # @param redirect_uri [String, nil] For :web auth, the redirect_uri of web UI application (must be the same as in application configuration)
151
154
  # @param username [String, nil] For :jwt auth, the username of the user to impersonate
152
155
  # @param private_key [String, nil] For :jwt auth, the private key to sign JWT token
@@ -215,9 +218,13 @@ module Aspera
215
218
  type: :oauth2,
216
219
  base_url: "#{url}/#{PATH_AUTH}",
217
220
  grant_method: :web,
221
+ # Faspex 5.0.16+ checks the code verifier, earlier versions ignore it
222
+ pkce: true,
218
223
  params: {
219
- client_id: client_id
220
- },
224
+ client_id: client_id,
225
+ # required when the API client has HTTP basic auth enabled, else ignored by Faspex
226
+ client_secret: client_secret
227
+ }.compact,
221
228
  redirect_uri: redirect_uri
222
229
  }
223
230
  }
@@ -1,15 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/rest'
5
- require 'aspera/transfer/faux_file'
6
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../rest'
8
+ require_relative '../transfer/faux_file'
9
+ require_relative '../assert'
10
+ require_relative '../rainbow'
7
11
  require 'net/protocol'
8
12
  require 'securerandom'
9
13
  require 'websocket'
10
14
  require 'base64'
11
15
  require 'json'
12
- require 'aspera/rainbow'
13
16
  using Rainbow
14
17
 
15
18
  # Throw exception on error, instead of error code
@@ -209,7 +212,7 @@ module Aspera
209
212
  fileIndex: file_index
210
213
  }
211
214
  file = file_to_send[:file]
212
- file = File.open(file) unless file.is_a?(Transfer::FauxFile) # rubocop:disable Style/FileOpen
215
+ file = File.open(file) unless file.is_a?(Transfer::FauxFile)
213
216
  begin
214
217
  until file.eof?
215
218
  slice_bin_data = file.read(@upload_chunk_size)
@@ -1,19 +1,22 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/link_header'
4
- require 'aspera/transfer/spec'
5
- require 'aspera/rest'
6
- require 'aspera/oauth/client/factory'
7
- require 'aspera/log'
8
- require 'aspera/assert'
9
- require 'aspera/secret_hider'
10
- require 'aspera/environment'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../link_header'
7
+ require_relative '../transfer/spec'
8
+ require_relative '../rest'
9
+ require_relative '../oauth/client/factory'
10
+ require_relative '../log'
11
+ require_relative '../assert'
12
+ require_relative '../secret/hider'
13
+ require_relative '../environment'
14
+ require_relative '../rainbow'
11
15
  require 'base64'
12
16
  require 'openssl'
13
17
  require 'pathname'
14
18
  require 'zlib'
15
19
  require 'net/ssh/buffer'
16
- require 'aspera/rainbow'
17
20
  using Rainbow
18
21
 
19
22
  module Aspera
@@ -181,7 +184,7 @@ module Aspera
181
184
  ''
182
185
  ].join("\n")))
183
186
  # Hidden wherever it appears (logs, results)
184
- SecretHider.instance.add_secret_values(token)
187
+ Secret::Hider.instance.add_secret_values(token)
185
188
  return token
186
189
  end
187
190
 
@@ -292,8 +295,10 @@ module Aspera
292
295
  if use_v4
293
296
  headers[HEADER_ACCEPT_VERSION] = '4.0'
294
297
  query['per_page'] = 1000 unless query.key?('per_page')
295
- elsif query.key?('per_page') && !query.key?('page')
296
- query['page'] = 0
298
+ else
299
+ # API v3 requires both `page` (first is 1) and `per_page`
300
+ query['page'] = 1 unless query.key?('page')
301
+ query['per_page'] = 1000 unless query.key?('per_page')
297
302
  end
298
303
  loop do
299
304
  Rest::Parameters.instance.spinner_cb.call(folder_items.count)
@@ -304,8 +309,9 @@ module Aspera
304
309
  break if iteration_token.nil? || iteration_token.empty?
305
310
  query['iteration_token'] = iteration_token
306
311
  else
307
- break if data['item_count'].eql?(0)
308
- query['offset'] += data['item_count']
312
+ # Last page: empty, or shorter than requested
313
+ break if data.empty? || (query.key?('per_page') && data.length < query['per_page'].to_i)
314
+ query['page'] = query['page'].to_i + 1
309
315
  end
310
316
  end
311
317
  rescue StandardError => e
@@ -450,11 +456,6 @@ module Aspera
450
456
  return find_state[:found]
451
457
  end
452
458
 
453
- # Generate a refreshed auth token
454
- def refreshed_transfer_token
455
- return oauth.authorization(refresh: true)
456
- end
457
-
458
459
  # Get a base download transfer spec (gen3)
459
460
  # @return [Hash] Base transfer spec
460
461
  def base_spec
@@ -484,7 +485,6 @@ module Aspera
484
485
  ak_token = Rest.basic_authorization(auth_params[:username], auth_params[:password])
485
486
  when :oauth2
486
487
  ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
487
- # TODO: token_generation_lambda = lambda{|do_refresh|oauth.authorization(refresh: do_refresh)}
488
488
  # Get bearer token, possibly use cache
489
489
  ak_token = oauth.authorization
490
490
  when :none
data/lib/aspera/ascmd.rb CHANGED
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore ascmd smode errstr zstr zmode zuid zgid zctime zatime zmtime fcount dcount btype blist codeset lc_ctype ascmdtypes
4
- require 'aspera/log'
5
- require 'aspera/assert'
6
- require 'aspera/rainbow'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:ignore smode errstr zstr zmode zuid zgid zctime zatime zmtime fcount dcount btype blist ascmdtypes
7
+ require_relative 'log'
8
+ require_relative 'assert'
9
+ require_relative 'rainbow'
7
10
  using Rainbow
8
11
 
9
12
  module Aspera
@@ -193,7 +196,7 @@ module Aspera
193
196
  num_bytes = type_name.eql?(:zstr) ? buffer.length : type_descr[:size]
194
197
  Aspera.assert(buffer.length >= num_bytes, 'not enough bytes')
195
198
  byte_array = buffer.shift(num_bytes)
196
- byte_array = [byte_array] unless byte_array.is_a?(Array)
199
+ byte_array = Array(byte_array)
197
200
  result = byte_array.pack('C*').unpack1(type_descr[:unpack])
198
201
  result.force_encoding('UTF-8') if type_name.eql?(:zstr)
199
202
  Log.log.trace1 { "#{' .' * indent_level}-> base:#{byte_array} -> #{result}" }
@@ -1,27 +1,26 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore protobuf ckpt
4
- require 'aspera/yaml'
5
- require 'aspera/environment'
6
- require 'aspera/data_repository'
7
- require 'aspera/log'
8
- require 'aspera/rest'
9
- require 'aspera/uri_reader'
10
- require 'aspera/assert'
11
- require 'aspera/web_server_simple'
12
- require 'aspera/cli/info'
13
- require 'aspera/cli/version'
14
- require 'aspera/products/desktop'
15
- require 'aspera/products/connect'
16
- require 'aspera/products/transferd'
17
- require 'aspera/products/other'
18
- require 'English'
19
- require 'singleton'
20
- require 'xmlsimple'
21
- require 'base64'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:ignore ckpt
7
+ require_relative '../yaml'
8
+ require_relative '../environment'
9
+ require_relative '../data_repository'
10
+ require_relative '../log'
11
+ require_relative '../rest'
12
+ require_relative '../uri_reader'
13
+ require_relative '../assert'
14
+ require_relative '../web_server_simple'
15
+ require_relative '../hash_ext'
16
+ require_relative '../products/desktop'
17
+ require_relative '../products/connect'
18
+ require_relative '../products/transferd'
19
+ require_relative '../products/other'
22
20
  require 'fileutils'
23
21
  require 'openssl'
24
- require 'yaml'
22
+ require 'singleton'
23
+ require 'stringio'
25
24
  module Aspera
26
25
  module Ascp
27
26
  # Singleton that tells where to find ascp and other local resources (keys..) , using the "path(:name)" method.
@@ -57,18 +56,10 @@ module Aspera
57
56
  def sdk_locations
58
57
  Aspera.assert_type(@transferd_urls, Array) { 'locations_url' }
59
58
  location_urls = @transferd_urls.empty? ? TRANSFERD_ARCHIVE_LOCATION_URLS : @transferd_urls
60
- location_urls.each_with_index do |location_url, index|
61
- last = index.eql?(location_urls.length - 1)
62
- Log.log.debug { "Retrieving SDK locations from #{location_url}" }
63
- begin
64
- transferd_locations = last ? UriReader.read(location_url) : read_fail_fast(location_url)
65
- locations = Yaml.safe_load(transferd_locations)
66
- Aspera.assert_type(locations, Array) { "SDK locations from #{location_url}" }
67
- return locations
68
- rescue StandardError => e
69
- raise if last
70
- Log.log.warn { "Failed to retrieve SDK locations from #{location_url}: #{e.message}, trying next" }
71
- end
59
+ return UriReader.read_first(location_urls, what: 'SDK locations') do |content|
60
+ locations = Yaml.safe_load(content)
61
+ Aspera.assert_type(locations, Array) { 'SDK locations' }
62
+ locations
72
63
  end
73
64
  end
74
65
 
@@ -141,7 +132,6 @@ module Aspera
141
132
  file_key = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert_private_key.pem')
142
133
  file_cert = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert.pem')
143
134
  if !File.exist?(file_key) || !File.exist?(file_cert)
144
- require 'openssl'
145
135
  # create new self signed certificate for http fallback
146
136
  private_key = OpenSSL::PKey::RSA.new(4096)
147
137
  cert = WebServerSimple.self_signed_cert(private_key)
@@ -247,8 +237,10 @@ module Aspera
247
237
  return ascp_data
248
238
  end
249
239
 
250
- # @return [String] the url for download of SDK archive for the given platform and version
251
- def sdk_url_for_platform(platform: nil, version: nil)
240
+ # @param platform [nil, String] Platform, if nil: current one
241
+ # @param version [nil, String] Version, if nil: latest
242
+ # @return [Hash] the entry of SDK locations for the given platform and version
243
+ def sdk_location_for_platform(platform: nil, version: nil)
252
244
  all_locations = sdk_locations
253
245
  platform = Environment.instance.architecture if platform.nil?
254
246
  locations = all_locations.select { |l| l['platform'].eql?(platform) }
@@ -256,7 +248,20 @@ module Aspera
256
248
  version = locations.max_by { |entry| Gem::Version.new(entry['version']) }['version'] if version.nil?
257
249
  info = locations.select { |entry| entry['version'].eql?(version) }
258
250
  Aspera.assert(!info.empty?) { "No such version: #{version} for #{platform}" }
259
- return info.first['url']
251
+ return info.first
252
+ end
253
+
254
+ # @return [String] the url for download of SDK archive for the given platform and version
255
+ def sdk_url_for_platform(platform: nil, version: nil)
256
+ return sdk_location_for_platform(platform: platform, version: version)['url']
257
+ end
258
+
259
+ # @return [String, nil] Version of the SDK installed in the SDK folder (from its meta data file), or nil if not installed
260
+ def installed_sdk_version
261
+ return unless File.exist?(sdk_exe_path(:transferd))
262
+ meta_data_file = File.join(Products::Transferd.sdk_directory, Products::Other::INFO_META_FILE)
263
+ return unless File.exist?(meta_data_file)
264
+ return File.read(meta_data_file).scan(%r{<version>(.*?)</version>}).flatten.first
260
265
  end
261
266
 
262
267
  # @param url [String] URL or filename used to detect archive format
@@ -346,8 +351,8 @@ module Aspera
346
351
  extracted_files[file_basename] = entry_name
347
352
  FileUtils.mkdir_p(dest_folder)
348
353
  if link_target.nil?
349
- # Security: Check if destination already exists
350
- if File.exist?(dest_file)
354
+ # Security: Check if destination already exists, including a dangling symlink (`exist?` follows links)
355
+ if File.symlink?(dest_file) || File.exist?(dest_file)
351
356
  Log.log.warn { "Rejecting write to existing file or link: #{dest_file}" }
352
357
  next
353
358
  end
@@ -357,13 +362,13 @@ module Aspera
357
362
  FileUtils.mkdir_p(File.dirname(dest_file))
358
363
  # Check where the file would resolve to (handles existing symlinks in path)
359
364
  resolved_dest = File.realpath(File.dirname(dest_file))
360
- unless resolved_dest.start_with?(install_boundary)
365
+ unless inside_folder?(resolved_dest, install_boundary)
361
366
  Log.log.warn { "Rejecting file outside installation directory: #{dest_file} resolves to #{resolved_dest}" }
362
367
  next
363
368
  end
364
369
  rescue Errno::ENOENT
365
370
  # Directory doesn't exist yet, verify the intended path
366
- unless dest_file.start_with?(folder)
371
+ unless inside_folder?(File.expand_path(dest_file), File.expand_path(folder))
367
372
  Log.log.warn { "Rejecting file with path outside installation directory: #{dest_file}" }
368
373
  next
369
374
  end
@@ -372,7 +377,8 @@ module Aspera
372
377
  else
373
378
  # Security: Validate symlink target stays within installation boundary
374
379
  # Resolve the symlink target relative to its location
375
- link_dir = File.dirname(dest_file)
380
+ # (real path of the folder, so that links already on disk are resolved)
381
+ link_dir = File.realpath(File.dirname(dest_file))
376
382
  resolved_target = if link_target.start_with?('/')
377
383
  # Absolute symlink target
378
384
  link_target
@@ -381,7 +387,7 @@ module Aspera
381
387
  File.expand_path(link_target, link_dir)
382
388
  end
383
389
  # Check if resolved target would be outside installation directory
384
- unless resolved_target.start_with?(install_boundary)
390
+ unless inside_folder?(File.expand_path(resolved_target), install_boundary)
385
391
  Log.log.warn { "Rejecting symlink pointing outside installation directory: #{entry_name} -> #{link_target} (resolves to #{resolved_target})" }
386
392
  next
387
393
  end
@@ -393,8 +399,10 @@ module Aspera
393
399
  # Downloads, installs and configures the SDK, returns name, version and install folder
394
400
  # @param url [nil, String] URL to SDK archive, if nil: default url for version
395
401
  # @param version [nil, String] Specific version, if nil: latest version
402
+ # @param folder [nil, String] Install folder, if nil: the SDK folder (option `sdk_folder`), which is unchanged
396
403
  # @return [Array<String>] sdk name, sdk version, install folder
397
- def install_sdk(url: nil, version: nil)
404
+ def install_sdk(url: nil, version: nil, folder: nil)
405
+ return with_sdk_directory(folder) { install_sdk(url: url, version: version) } unless folder.nil?
398
406
  folder = Products::Transferd.sdk_directory
399
407
  download_sdk(folder: folder, url: url, version: version)
400
408
  # Ensure necessary files are there, or generate them, restrict file access on SDK executables
@@ -429,6 +437,25 @@ module Aspera
429
437
 
430
438
  private
431
439
 
440
+ # Run the block with another SDK folder (files of the SDK are located from it), then set the previous one back
441
+ # @param folder [String] Temporary SDK folder
442
+ def with_sdk_directory(folder)
443
+ previous = Products::Transferd.sdk_directory
444
+ begin
445
+ Products::Transferd.sdk_directory = folder
446
+ return yield
447
+ ensure
448
+ Products::Transferd.sdk_directory = previous
449
+ end
450
+ end
451
+
452
+ # @param path [String] Path to check
453
+ # @param boundary [String] Folder that must contain `path`
454
+ # @return [Boolean] `true` if `path` is `boundary` or is below it (a sibling sharing the name prefix is not)
455
+ def inside_folder?(path, boundary)
456
+ path.eql?(boundary) || path.start_with?(boundary.end_with?(File::SEPARATOR) ? boundary : "#{boundary}#{File::SEPARATOR}")
457
+ end
458
+
432
459
  DEFAULT_ASPERA_CONF = <<~END_OF_CONFIG_FILE
433
460
  <?xml version='1.0' encoding='UTF-8'?>
434
461
  <CONF version="2">
@@ -449,10 +476,8 @@ module Aspera
449
476
  'https://ibm.biz/sdk_location',
450
477
  'https://raw.githubusercontent.com/IBM/aspera-cli/refs/heads/main/docs/sdk_location.yaml'
451
478
  ].freeze
452
- # Timeout in seconds when another location URL remains to be tried
453
- LOCATION_FAIL_FAST_TIMEOUT = 10
454
479
  # filename for ascp with optional extension (Windows)
455
- private_constant :DEFAULT_ASPERA_CONF, :EXE_FILES, :SDK_FILES, :TRANSFERD_ARCHIVE_LOCATION_URLS, :LOCATION_FAIL_FAST_TIMEOUT
480
+ private_constant :DEFAULT_ASPERA_CONF, :EXE_FILES, :SDK_FILES, :TRANSFERD_ARCHIVE_LOCATION_URLS
456
481
 
457
482
  def initialize
458
483
  # cache for installed products found
@@ -465,25 +490,6 @@ module Aspera
465
490
  @ascp_folder = nil
466
491
  end
467
492
 
468
- # Read URL with a short timeout and no retry (temporarily changes global HTTP parameters)
469
- # @param url [String] URL to read
470
- # @return [String] content
471
- def read_fail_fast(url)
472
- parameters = Rest::Parameters.instance
473
- session_cb = parameters.session_cb
474
- retry_max = parameters.retry_max
475
- parameters.retry_max = 0
476
- parameters.session_cb = lambda do |http_session|
477
- session_cb&.call(http_session)
478
- http_session.open_timeout = [http_session.open_timeout, LOCATION_FAIL_FAST_TIMEOUT].compact.min
479
- http_session.read_timeout = [http_session.read_timeout, LOCATION_FAIL_FAST_TIMEOUT].compact.min
480
- end
481
- UriReader.read(url)
482
- ensure
483
- parameters.session_cb = session_cb
484
- parameters.retry_max = retry_max
485
- end
486
-
487
493
  # @param file_type [Symbol] one of EXE_FILES
488
494
  # @return [String] path to executable in SDK folder
489
495
  def sdk_exe_path(file_type)