aspera-cli 4.28.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +217 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +122 -26
  10. data/SECURITY.md +84 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1895 -632
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +29 -22
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +95 -67
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +370 -0
  41. data/lib/aspera/ascp/log_locations.rb +163 -0
  42. data/lib/aspera/ascp/management.rb +16 -8
  43. data/lib/aspera/ascp/session_errors.rb +217 -0
  44. data/lib/aspera/assert.rb +4 -0
  45. data/lib/aspera/cli/ascp_actions.rb +12 -19
  46. data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
  47. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  48. data/lib/aspera/cli/bootstrapper.rb +38 -33
  49. data/lib/aspera/cli/command_line.rb +63 -8
  50. data/lib/aspera/cli/command_registry.rb +9 -1
  51. data/lib/aspera/cli/command_spec.rb +47 -7
  52. data/lib/aspera/cli/completion/ascli.bash +19 -5
  53. data/lib/aspera/cli/completion/ascli.fish +13 -2
  54. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  55. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  56. data/lib/aspera/cli/completion_actions.rb +303 -69
  57. data/lib/aspera/cli/context.rb +9 -4
  58. data/lib/aspera/cli/deprecation.rb +5 -2
  59. data/lib/aspera/cli/error.rb +6 -0
  60. data/lib/aspera/cli/extended_value.rb +66 -31
  61. data/lib/aspera/cli/formatter.rb +43 -30
  62. data/lib/aspera/cli/gem_checker.rb +9 -6
  63. data/lib/aspera/cli/hints.rb +22 -10
  64. data/lib/aspera/cli/http.rb +38 -10
  65. data/lib/aspera/cli/info.rb +12 -7
  66. data/lib/aspera/cli/mailer.rb +6 -3
  67. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  68. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  69. data/lib/aspera/cli/mcp/tool.rb +20 -17
  70. data/lib/aspera/cli/option_declarator.rb +10 -4
  71. data/lib/aspera/cli/option_registry.rb +6 -3
  72. data/lib/aspera/cli/option_types.rb +17 -2
  73. data/lib/aspera/cli/option_value.rb +77 -18
  74. data/lib/aspera/cli/options.schema.yaml +641 -80
  75. data/lib/aspera/cli/parser.rb +71 -35
  76. data/lib/aspera/cli/plugins/alee.rb +7 -4
  77. data/lib/aspera/cli/plugins/aoc.rb +248 -37
  78. data/lib/aspera/cli/plugins/ats.rb +10 -9
  79. data/lib/aspera/cli/plugins/base.rb +50 -18
  80. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  81. data/lib/aspera/cli/plugins/config.rb +210 -104
  82. data/lib/aspera/cli/plugins/console.rb +6 -3
  83. data/lib/aspera/cli/plugins/cos.rb +8 -4
  84. data/lib/aspera/cli/plugins/factory.rb +21 -7
  85. data/lib/aspera/cli/plugins/faspex5.rb +31 -26
  86. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  87. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  88. data/lib/aspera/cli/plugins/mcp.rb +42 -22
  89. data/lib/aspera/cli/plugins/node.rb +39 -38
  90. data/lib/aspera/cli/plugins/oauth.rb +23 -6
  91. data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
  92. data/lib/aspera/cli/plugins/preview.rb +72 -45
  93. data/lib/aspera/cli/plugins/server.rb +66 -14
  94. data/lib/aspera/cli/plugins/shares.rb +87 -18
  95. data/lib/aspera/cli/plugins/transferd.rb +228 -0
  96. data/lib/aspera/cli/poll.rb +236 -0
  97. data/lib/aspera/cli/preset_actions.rb +144 -30
  98. data/lib/aspera/cli/preset_manager.rb +113 -35
  99. data/lib/aspera/cli/prompt.rb +157 -9
  100. data/lib/aspera/cli/result.rb +12 -8
  101. data/lib/aspera/cli/runner.rb +89 -48
  102. data/lib/aspera/cli/secret_finder.rb +4 -1
  103. data/lib/aspera/cli/security.rb +89 -9
  104. data/lib/aspera/cli/special_values.rb +4 -1
  105. data/lib/aspera/cli/sync_actions.rb +15 -10
  106. data/lib/aspera/cli/table_fitter.rb +3 -0
  107. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  108. data/lib/aspera/cli/transfer_actions.rb +11 -8
  109. data/lib/aspera/cli/transfer_agent.rb +43 -20
  110. data/lib/aspera/cli/transfer_progress.rb +14 -45
  111. data/lib/aspera/cli/vault_actions.rb +45 -0
  112. data/lib/aspera/cli/vault_manager.rb +188 -42
  113. data/lib/aspera/cli/version.rb +4 -1
  114. data/lib/aspera/cli/web_secret.rb +12 -10
  115. data/lib/aspera/cli/wizard.rb +42 -39
  116. data/lib/aspera/command_line_builder.rb +9 -5
  117. data/lib/aspera/command_line_converter.rb +4 -1
  118. data/lib/aspera/coverage.rb +103 -33
  119. data/lib/aspera/data_repository.rb +4 -1
  120. data/lib/aspera/dot_container.rb +4 -1
  121. data/lib/aspera/environment.rb +23 -12
  122. data/lib/aspera/exec_spec.rb +3 -0
  123. data/lib/aspera/faspex_gw.rb +13 -9
  124. data/lib/aspera/faspex_postproc.rb +15 -12
  125. data/lib/aspera/formatter_interface.rb +4 -1
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +20 -5
  129. data/lib/aspera/inflate.rb +43 -0
  130. data/lib/aspera/json_rpc/client.rb +6 -5
  131. data/lib/aspera/json_rpc/version.rb +3 -0
  132. data/lib/aspera/line_logger.rb +4 -1
  133. data/lib/aspera/link_header.rb +4 -1
  134. data/lib/aspera/log.rb +18 -5
  135. data/lib/aspera/markdown.rb +5 -2
  136. data/lib/aspera/mime.rb +3 -0
  137. data/lib/aspera/nagios.rb +6 -4
  138. data/lib/aspera/node_emulator.rb +15 -13
  139. data/lib/aspera/oauth/client/base.rb +124 -73
  140. data/lib/aspera/oauth/client/boot.rb +7 -4
  141. data/lib/aspera/oauth/client/factory.rb +41 -18
  142. data/lib/aspera/oauth/client/generic.rb +5 -1
  143. data/lib/aspera/oauth/client/json_credentials.rb +7 -3
  144. data/lib/aspera/oauth/client/jwt.rb +6 -3
  145. data/lib/aspera/oauth/client/shares_web.rb +94 -0
  146. data/lib/aspera/oauth/client/url_json.rb +7 -3
  147. data/lib/aspera/oauth/client/web.rb +40 -13
  148. data/lib/aspera/oauth/server/pkce.rb +6 -5
  149. data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +16 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +17 -2
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +3 -0
  165. data/lib/aspera/rest/aspera_errors.rb +4 -1
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +45 -34
  168. data/lib/aspera/rest/error_analyzer.rb +5 -2
  169. data/lib/aspera/rest/list.rb +8 -4
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +8 -5
  172. data/lib/aspera/rest.rb +9 -6
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  175. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  176. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  177. data/lib/aspera/schema/async_tables.yaml +11 -10
  178. data/lib/aspera/schema/documentation.rb +8 -2
  179. data/lib/aspera/schema/reader.rb +7 -0
  180. data/lib/aspera/schema/registry.rb +44 -28
  181. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  182. data/lib/aspera/schema/validator.rb +4 -1
  183. data/lib/aspera/sealed_hash.rb +150 -0
  184. data/lib/aspera/secret/hider.rb +213 -0
  185. data/lib/aspera/secret/reference.rb +150 -0
  186. data/lib/aspera/secret/resolver.rb +212 -0
  187. data/lib/aspera/secret/store/base.rb +153 -0
  188. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  189. data/lib/aspera/secret/store/factory.rb +103 -0
  190. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  191. data/lib/aspera/secret/store/keeper.rb +141 -0
  192. data/lib/aspera/secret/store/macos_security.rb +284 -0
  193. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  194. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  195. data/lib/aspera/secret/store/one_password_cli.rb +112 -0
  196. data/lib/aspera/secret/store/secret_service.rb +125 -0
  197. data/lib/aspera/secret/strength.rb +119 -0
  198. data/lib/aspera/servlet_helpers.rb +5 -2
  199. data/lib/aspera/ssh.rb +10 -7
  200. data/lib/aspera/ssl.rb +5 -2
  201. data/lib/aspera/string_ext.rb +3 -0
  202. data/lib/aspera/sync/args.schema.yaml +3 -0
  203. data/lib/aspera/sync/conf.schema.yaml +9 -4
  204. data/lib/aspera/sync/database.rb +3 -0
  205. data/lib/aspera/sync/operations.rb +17 -15
  206. data/lib/aspera/temp_file_manager.rb +8 -4
  207. data/lib/aspera/timer_limiter.rb +3 -0
  208. data/lib/aspera/transfer/error.rb +4 -1
  209. data/lib/aspera/transfer/faux_file.rb +4 -1
  210. data/lib/aspera/transfer/parameters.rb +17 -14
  211. data/lib/aspera/transfer/result.rb +23 -0
  212. data/lib/aspera/transfer/resumer.rb +6 -4
  213. data/lib/aspera/transfer/spec.rb +9 -22
  214. data/lib/aspera/units/decimal.rb +67 -0
  215. data/lib/aspera/units/duration.rb +109 -0
  216. data/lib/aspera/uri_reader.rb +64 -9
  217. data/lib/aspera/web_auth/index.html.erb +6 -2
  218. data/lib/aspera/web_auth/paste.rb +109 -0
  219. data/lib/aspera/web_auth/receiver.rb +50 -0
  220. data/lib/aspera/web_auth/relay.rb +76 -0
  221. data/lib/aspera/web_auth/reply.html.erb +150 -0
  222. data/lib/aspera/web_auth/server.rb +47 -10
  223. data/lib/aspera/web_page.rb +7 -2
  224. data/lib/aspera/web_server_simple.rb +6 -3
  225. data/lib/aspera/yaml.rb +5 -1
  226. data/llms-install.md +88 -0
  227. data/llms.txt +87 -0
  228. data/skills/ascli/SKILL.md +94 -0
  229. data.tar.gz.sig +0 -0
  230. metadata +77 -20
  231. metadata.gz.sig +0 -0
  232. data/lib/aspera/duration.rb +0 -42
  233. data/lib/aspera/keychain/base.rb +0 -30
  234. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  235. data/lib/aspera/keychain/factory.rb +0 -58
  236. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  237. data/lib/aspera/keychain/macos_security.rb +0 -277
  238. data/lib/aspera/keychain/one_password_api.rb +0 -91
  239. data/lib/aspera/keychain/one_password_base.rb +0 -39
  240. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  241. data/lib/aspera/secret_hider.rb +0 -196
  242. data/lib/aspera/secret_strength.rb +0 -114
@@ -1,15 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'aspera/rest'
6
- require 'aspera/hash_ext'
7
- require 'aspera/data_repository'
8
- require 'aspera/transfer/spec'
9
- require 'aspera/api/node'
10
- require 'aspera/rest/list'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative '../rest'
9
+ require_relative '../hash_ext'
10
+ require_relative '../data_repository'
11
+ require_relative '../transfer/spec'
12
+ require_relative 'node'
13
+ require_relative '../rest/list'
14
+ require_relative '../rainbow'
11
15
  require 'base64'
12
- require 'aspera/rainbow'
13
16
  using Rainbow
14
17
 
15
18
  module Aspera
@@ -62,7 +65,7 @@ module Aspera
62
65
  # `''` Use default workspace if it is set, else none
63
66
  DEFAULT_WORKSPACE = ''
64
67
  # Production domain of AoC
65
- SAAS_DOMAIN_PROD = 'ibmaspera.com' # cspell:disable-line
68
+ SAAS_DOMAIN_PROD = 'ibmaspera.com'
66
69
  # To avoid infinite loop in pub link redirection
67
70
  MAX_AOC_URL_REDIRECT = 10
68
71
  CLIENT_ID_PREFIX = 'aspera.'
@@ -103,7 +106,7 @@ module Aspera
103
106
  # Scope `admin:all`
104
107
  ADMIN = 'admin:all'
105
108
  ADMIN_USER = 'admin-user:all'
106
- ADMIN_USER_USER = "#{ADMIN_USER}+#{USER}"
109
+ ADMIN_USER_USER = "#{ADMIN_USER}+#{USER}".freeze
107
110
  end
108
111
  API_V1 = 'api/v1'
109
112
 
@@ -1,7 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/rest'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../rest'
5
7
 
6
8
  module Aspera
7
9
  module Api
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'aspera/api/node'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative 'node'
6
9
  require 'xmlsimple'
7
10
 
8
11
  module Aspera
@@ -12,7 +15,7 @@ module Aspera
12
15
  IBM_CLOUD_TOKEN_URL = 'https://iam.cloud.ibm.com/identity'
13
16
  TOKEN_FIELD = 'delegated_refresh_token'
14
17
  FASP_INFO_KEYS = %w[ATSEndpoint AccessKey].freeze
15
- SERVICE_CREDS_PARAMS = %w[apikey resource_instance_id endpoints]
18
+ SERVICE_CREDS_PARAMS = %w[apikey resource_instance_id endpoints].freeze
16
19
  class << self
17
20
  def parameters_from_svc_credentials(service_credentials, bucket_region)
18
21
  # check necessary contents
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
4
- require 'aspera/rest'
5
- require 'aspera/oauth/client/base'
6
- require 'aspera/rest/list'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+ require_relative '../rest'
8
+ require_relative '../oauth/client/base'
9
+ require_relative '../rest/list'
7
10
  require 'digest'
8
11
 
9
12
  module Aspera
@@ -94,7 +97,7 @@ module Aspera
94
97
  # states for jobs not in final state
95
98
  JOB_RUNNING = %w[queued working].freeze
96
99
  PATH_STANDARD_ROOT = '/aspera/faspex'
97
- PATH_API_DETECT = "#{PATH_API_V5}/#{PATH_HEALTH}"
100
+ PATH_API_DETECT = "#{PATH_API_V5}/#{PATH_HEALTH}".freeze
98
101
  HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
99
102
  HEADER_FASPEX_VERSION = 'X-IBM-Aspera'
100
103
  EMAIL_NOTIF_LIST = %w[
@@ -125,7 +128,7 @@ module Aspera
125
128
  sync_failed
126
129
  account_exist
127
130
  mfa_code
128
- ]
131
+ ].freeze
129
132
  class << self
130
133
  # @return [Boolean] true if the URL is a public link
131
134
  def public_link?(url)
@@ -146,7 +149,7 @@ module Aspera
146
149
  # @param root [String] Root path for API
147
150
  # @param password [String, nil] For :boot auth, the token copied directly from browser in developer mode
148
151
  # @param client_id [String, nil] For :web and :jwt auth, the client_id of web UI application
149
- # @param client_secret [String, nil] For :web auth, the client_secret of web UI application (not needed for :jwt)
152
+ # @param client_secret [String, nil] For :web auth, the client_secret of the API client, required if it has HTTP basic auth enabled (not needed for :jwt)
150
153
  # @param redirect_uri [String, nil] For :web auth, the redirect_uri of web UI application (must be the same as in application configuration)
151
154
  # @param username [String, nil] For :jwt auth, the username of the user to impersonate
152
155
  # @param private_key [String, nil] For :jwt auth, the private key to sign JWT token
@@ -215,9 +218,13 @@ module Aspera
215
218
  type: :oauth2,
216
219
  base_url: "#{url}/#{PATH_AUTH}",
217
220
  grant_method: :web,
221
+ # Faspex 5.0.16+ checks the code verifier, earlier versions ignore it
222
+ pkce: true,
218
223
  params: {
219
- client_id: client_id
220
- },
224
+ client_id: client_id,
225
+ # required when the API client has HTTP basic auth enabled, else ignored by Faspex
226
+ client_secret: client_secret
227
+ }.compact,
221
228
  redirect_uri: redirect_uri
222
229
  }
223
230
  }
@@ -1,15 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/rest'
5
- require 'aspera/transfer/faux_file'
6
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../rest'
8
+ require_relative '../transfer/faux_file'
9
+ require_relative '../assert'
10
+ require_relative '../rainbow'
7
11
  require 'net/protocol'
8
12
  require 'securerandom'
9
13
  require 'websocket'
10
14
  require 'base64'
11
15
  require 'json'
12
- require 'aspera/rainbow'
13
16
  using Rainbow
14
17
 
15
18
  # Throw exception on error, instead of error code
@@ -209,7 +212,7 @@ module Aspera
209
212
  fileIndex: file_index
210
213
  }
211
214
  file = file_to_send[:file]
212
- file = File.open(file) unless file.is_a?(Transfer::FauxFile) # rubocop:disable Style/FileOpen
215
+ file = File.open(file) unless file.is_a?(Transfer::FauxFile)
213
216
  begin
214
217
  until file.eof?
215
218
  slice_bin_data = file.read(@upload_chunk_size)
@@ -1,19 +1,23 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/link_header'
4
- require 'aspera/transfer/spec'
5
- require 'aspera/rest'
6
- require 'aspera/oauth/client/factory'
7
- require 'aspera/log'
8
- require 'aspera/assert'
9
- require 'aspera/secret_hider'
10
- require 'aspera/environment'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../link_header'
7
+ require_relative '../transfer/spec'
8
+ require_relative '../rest'
9
+ require_relative '../oauth/client/factory'
10
+ require_relative '../log'
11
+ require_relative '../assert'
12
+ require_relative '../inflate'
13
+ require_relative '../secret/hider'
14
+ require_relative '../environment'
15
+ require_relative '../rainbow'
11
16
  require 'base64'
12
17
  require 'openssl'
13
18
  require 'pathname'
14
19
  require 'zlib'
15
20
  require 'net/ssh/buffer'
16
- require 'aspera/rainbow'
17
21
  using Rainbow
18
22
 
19
23
  module Aspera
@@ -70,7 +74,9 @@ module Aspera
70
74
  SIGNATURE_DELIMITER = '==SIGNATURE=='
71
75
  # Default validity when generating a bearer token "manually"
72
76
  BEARER_TOKEN_VALIDITY_DEFAULT = 86400
73
- private_constant :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT
77
+ # Maximum size of a decoded bearer token (a few kB): it is given by the user, a client, or a server
78
+ BEARER_TOKEN_MAX_INFLATED_SIZE = 65_536
79
+ private_constant :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT, :BEARER_TOKEN_MAX_INFLATED_SIZE
74
80
  # Node API permissions: delete list mkdir preview read rename write
75
81
  ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
76
82
  # Special HTTP Headers
@@ -181,13 +187,17 @@ module Aspera
181
187
  ''
182
188
  ].join("\n")))
183
189
  # Hidden wherever it appears (logs, results)
184
- SecretHider.instance.add_secret_values(token)
190
+ Secret::Hider.instance.add_secret_values(token)
185
191
  return token
186
192
  end
187
193
 
188
194
  # Decode an Aspera Node bearer token
195
+ # @param token [String] Base64 of deflated payload and signature
196
+ # @return [Hash] Payload
197
+ # @raise [ParameterError] If the decoded token is too large
189
198
  def decode_bearer_token(token)
190
- return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
199
+ inflated = Inflate.limited(Base64.decode64(token), max_size: BEARER_TOKEN_MAX_INFLATED_SIZE, what: 'Node bearer token')
200
+ return JSON.parse(inflated.partition(SIGNATURE_DELIMITER).first)
191
201
  end
192
202
 
193
203
  # @return [Hash] Headers to call node API with access key and auth
@@ -292,8 +302,10 @@ module Aspera
292
302
  if use_v4
293
303
  headers[HEADER_ACCEPT_VERSION] = '4.0'
294
304
  query['per_page'] = 1000 unless query.key?('per_page')
295
- elsif query.key?('per_page') && !query.key?('page')
296
- query['page'] = 0
305
+ else
306
+ # API v3 requires both `page` (first is 1) and `per_page`
307
+ query['page'] = 1 unless query.key?('page')
308
+ query['per_page'] = 1000 unless query.key?('per_page')
297
309
  end
298
310
  loop do
299
311
  Rest::Parameters.instance.spinner_cb.call(folder_items.count)
@@ -304,8 +316,9 @@ module Aspera
304
316
  break if iteration_token.nil? || iteration_token.empty?
305
317
  query['iteration_token'] = iteration_token
306
318
  else
307
- break if data['item_count'].eql?(0)
308
- query['offset'] += data['item_count']
319
+ # Last page: empty, or shorter than requested
320
+ break if data.empty? || (query.key?('per_page') && data.length < query['per_page'].to_i)
321
+ query['page'] = query['page'].to_i + 1
309
322
  end
310
323
  end
311
324
  rescue StandardError => e
@@ -450,11 +463,6 @@ module Aspera
450
463
  return find_state[:found]
451
464
  end
452
465
 
453
- # Generate a refreshed auth token
454
- def refreshed_transfer_token
455
- return oauth.authorization(refresh: true)
456
- end
457
-
458
466
  # Get a base download transfer spec (gen3)
459
467
  # @return [Hash] Base transfer spec
460
468
  def base_spec
@@ -484,7 +492,6 @@ module Aspera
484
492
  ak_token = Rest.basic_authorization(auth_params[:username], auth_params[:password])
485
493
  when :oauth2
486
494
  ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
487
- # TODO: token_generation_lambda = lambda{|do_refresh|oauth.authorization(refresh: do_refresh)}
488
495
  # Get bearer token, possibly use cache
489
496
  ak_token = oauth.authorization
490
497
  when :none
data/lib/aspera/ascmd.rb CHANGED
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore ascmd smode errstr zstr zmode zuid zgid zctime zatime zmtime fcount dcount btype blist codeset lc_ctype ascmdtypes
4
- require 'aspera/log'
5
- require 'aspera/assert'
6
- require 'aspera/rainbow'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:ignore smode errstr zstr zmode zuid zgid zctime zatime zmtime fcount dcount btype blist ascmdtypes
7
+ require_relative 'log'
8
+ require_relative 'assert'
9
+ require_relative 'rainbow'
7
10
  using Rainbow
8
11
 
9
12
  module Aspera
@@ -193,7 +196,7 @@ module Aspera
193
196
  num_bytes = type_name.eql?(:zstr) ? buffer.length : type_descr[:size]
194
197
  Aspera.assert(buffer.length >= num_bytes, 'not enough bytes')
195
198
  byte_array = buffer.shift(num_bytes)
196
- byte_array = [byte_array] unless byte_array.is_a?(Array)
199
+ byte_array = Array(byte_array)
197
200
  result = byte_array.pack('C*').unpack1(type_descr[:unpack])
198
201
  result.force_encoding('UTF-8') if type_name.eql?(:zstr)
199
202
  Log.log.trace1 { "#{' .' * indent_level}-> base:#{byte_array} -> #{result}" }
@@ -1,27 +1,26 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore protobuf ckpt
4
- require 'aspera/yaml'
5
- require 'aspera/environment'
6
- require 'aspera/data_repository'
7
- require 'aspera/log'
8
- require 'aspera/rest'
9
- require 'aspera/uri_reader'
10
- require 'aspera/assert'
11
- require 'aspera/web_server_simple'
12
- require 'aspera/cli/info'
13
- require 'aspera/cli/version'
14
- require 'aspera/products/desktop'
15
- require 'aspera/products/connect'
16
- require 'aspera/products/transferd'
17
- require 'aspera/products/other'
18
- require 'English'
19
- require 'singleton'
20
- require 'xmlsimple'
21
- require 'base64'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:ignore ckpt
7
+ require_relative '../yaml'
8
+ require_relative '../environment'
9
+ require_relative '../data_repository'
10
+ require_relative '../log'
11
+ require_relative '../rest'
12
+ require_relative '../uri_reader'
13
+ require_relative '../assert'
14
+ require_relative '../web_server_simple'
15
+ require_relative '../hash_ext'
16
+ require_relative '../products/desktop'
17
+ require_relative '../products/connect'
18
+ require_relative '../products/transferd'
19
+ require_relative '../products/other'
22
20
  require 'fileutils'
23
21
  require 'openssl'
24
- require 'yaml'
22
+ require 'singleton'
23
+ require 'stringio'
25
24
  module Aspera
26
25
  module Ascp
27
26
  # Singleton that tells where to find ascp and other local resources (keys..) , using the "path(:name)" method.
@@ -53,22 +52,17 @@ module Aspera
53
52
  # Loads YAML from cloud with locations of SDK archives for all platforms
54
53
  # Tries each URL of option `locations_url` (default if empty) in order, until one provides a valid list.
55
54
  # All but the last URL are read with a short timeout and no retry, so that fallback is quick.
55
+ # The request tells which application reads the list, and what it downloads (header `Referer`, see `referer_url`):
56
+ # the default URL counts requests per `Referer` (download statistics).
57
+ # @param download [Hash, nil] SDK about to be downloaded (`sdk`: version, `platform`), `nil` if the list is only read
56
58
  # @return [Array<Hash>] location structure
57
- def sdk_locations
59
+ def sdk_locations(download: nil)
58
60
  Aspera.assert_type(@transferd_urls, Array) { 'locations_url' }
59
61
  location_urls = @transferd_urls.empty? ? TRANSFERD_ARCHIVE_LOCATION_URLS : @transferd_urls
60
- location_urls.each_with_index do |location_url, index|
61
- last = index.eql?(location_urls.length - 1)
62
- Log.log.debug { "Retrieving SDK locations from #{location_url}" }
63
- begin
64
- transferd_locations = last ? UriReader.read(location_url) : read_fail_fast(location_url)
65
- locations = Yaml.safe_load(transferd_locations)
66
- Aspera.assert_type(locations, Array) { "SDK locations from #{location_url}" }
67
- return locations
68
- rescue StandardError => e
69
- raise if last
70
- Log.log.warn { "Failed to retrieve SDK locations from #{location_url}: #{e.message}, trying next" }
71
- end
62
+ return UriReader.read_first(location_urls, what: 'SDK locations', headers: locations_headers(download)) do |content|
63
+ locations = Yaml.safe_load(content)
64
+ Aspera.assert_type(locations, Array) { 'SDK locations' }
65
+ locations
72
66
  end
73
67
  end
74
68
 
@@ -141,7 +135,6 @@ module Aspera
141
135
  file_key = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert_private_key.pem')
142
136
  file_cert = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert.pem')
143
137
  if !File.exist?(file_key) || !File.exist?(file_cert)
144
- require 'openssl'
145
138
  # create new self signed certificate for http fallback
146
139
  private_key = OpenSSL::PKey::RSA.new(4096)
147
140
  cert = WebServerSimple.self_signed_cert(private_key)
@@ -244,19 +237,39 @@ module Aspera
244
237
  ascp_data = file_paths
245
238
  ascp_data.merge!(ascp_info_from_log)
246
239
  ascp_data.merge!(ascp_info_from_file)
240
+ # `ascp` is run with `-L-` to read its paths: report where it logs without `-L`
241
+ if ascp_data['log'].eql?('-')
242
+ require_relative 'log_locations'
243
+ ascp_data['log'] = LogLocations.ascp_log_location(path(:ascp), root: ascp_data['root'], conf_file: ascp_data['conf'])
244
+ end
247
245
  return ascp_data
248
246
  end
249
247
 
250
- # @return [String] the url for download of SDK archive for the given platform and version
251
- def sdk_url_for_platform(platform: nil, version: nil)
252
- all_locations = sdk_locations
248
+ # @param platform [nil, String] Platform, if nil: current one
249
+ # @param version [nil, String] Version, if nil: latest
250
+ # @return [Hash] the entry of SDK locations for the given platform and version
251
+ def sdk_location_for_platform(platform: nil, version: nil)
253
252
  platform = Environment.instance.architecture if platform.nil?
253
+ all_locations = sdk_locations(download: {sdk: version || LATEST_VERSION, platform: platform})
254
254
  locations = all_locations.select { |l| l['platform'].eql?(platform) }
255
255
  Aspera.assert(!locations.empty?) { "No SDK for platform: #{platform}, available: #{all_locations.map { |i| i['platform'] }.uniq}" }
256
256
  version = locations.max_by { |entry| Gem::Version.new(entry['version']) }['version'] if version.nil?
257
257
  info = locations.select { |entry| entry['version'].eql?(version) }
258
258
  Aspera.assert(!info.empty?) { "No such version: #{version} for #{platform}" }
259
- return info.first['url']
259
+ return info.first
260
+ end
261
+
262
+ # @return [String] the url for download of SDK archive for the given platform and version
263
+ def sdk_url_for_platform(platform: nil, version: nil)
264
+ return sdk_location_for_platform(platform: platform, version: version)['url']
265
+ end
266
+
267
+ # @return [String, nil] Version of the SDK installed in the SDK folder (from its meta data file), or nil if not installed
268
+ def installed_sdk_version
269
+ return unless File.exist?(sdk_exe_path(:transferd))
270
+ meta_data_file = File.join(Products::Transferd.sdk_directory, Products::Other::INFO_META_FILE)
271
+ return unless File.exist?(meta_data_file)
272
+ return File.read(meta_data_file).scan(%r{<version>(.*?)</version>}).flatten.first
260
273
  end
261
274
 
262
275
  # @param url [String] URL or filename used to detect archive format
@@ -346,8 +359,8 @@ module Aspera
346
359
  extracted_files[file_basename] = entry_name
347
360
  FileUtils.mkdir_p(dest_folder)
348
361
  if link_target.nil?
349
- # Security: Check if destination already exists
350
- if File.exist?(dest_file)
362
+ # Security: Check if destination already exists, including a dangling symlink (`exist?` follows links)
363
+ if File.symlink?(dest_file) || File.exist?(dest_file)
351
364
  Log.log.warn { "Rejecting write to existing file or link: #{dest_file}" }
352
365
  next
353
366
  end
@@ -357,13 +370,13 @@ module Aspera
357
370
  FileUtils.mkdir_p(File.dirname(dest_file))
358
371
  # Check where the file would resolve to (handles existing symlinks in path)
359
372
  resolved_dest = File.realpath(File.dirname(dest_file))
360
- unless resolved_dest.start_with?(install_boundary)
373
+ unless inside_folder?(resolved_dest, install_boundary)
361
374
  Log.log.warn { "Rejecting file outside installation directory: #{dest_file} resolves to #{resolved_dest}" }
362
375
  next
363
376
  end
364
377
  rescue Errno::ENOENT
365
378
  # Directory doesn't exist yet, verify the intended path
366
- unless dest_file.start_with?(folder)
379
+ unless inside_folder?(File.expand_path(dest_file), File.expand_path(folder))
367
380
  Log.log.warn { "Rejecting file with path outside installation directory: #{dest_file}" }
368
381
  next
369
382
  end
@@ -372,7 +385,8 @@ module Aspera
372
385
  else
373
386
  # Security: Validate symlink target stays within installation boundary
374
387
  # Resolve the symlink target relative to its location
375
- link_dir = File.dirname(dest_file)
388
+ # (real path of the folder, so that links already on disk are resolved)
389
+ link_dir = File.realpath(File.dirname(dest_file))
376
390
  resolved_target = if link_target.start_with?('/')
377
391
  # Absolute symlink target
378
392
  link_target
@@ -381,7 +395,7 @@ module Aspera
381
395
  File.expand_path(link_target, link_dir)
382
396
  end
383
397
  # Check if resolved target would be outside installation directory
384
- unless resolved_target.start_with?(install_boundary)
398
+ unless inside_folder?(File.expand_path(resolved_target), install_boundary)
385
399
  Log.log.warn { "Rejecting symlink pointing outside installation directory: #{entry_name} -> #{link_target} (resolves to #{resolved_target})" }
386
400
  next
387
401
  end
@@ -393,8 +407,10 @@ module Aspera
393
407
  # Downloads, installs and configures the SDK, returns name, version and install folder
394
408
  # @param url [nil, String] URL to SDK archive, if nil: default url for version
395
409
  # @param version [nil, String] Specific version, if nil: latest version
410
+ # @param folder [nil, String] Install folder, if nil: the SDK folder (option `sdk_folder`), which is unchanged
396
411
  # @return [Array<String>] sdk name, sdk version, install folder
397
- def install_sdk(url: nil, version: nil)
412
+ def install_sdk(url: nil, version: nil, folder: nil)
413
+ return with_sdk_directory(folder) { install_sdk(url: url, version: version) } unless folder.nil?
398
414
  folder = Products::Transferd.sdk_directory
399
415
  download_sdk(folder: folder, url: url, version: version)
400
416
  # Ensure necessary files are there, or generate them, restrict file access on SDK executables
@@ -427,8 +443,37 @@ module Aspera
427
443
  # @return [Array<String>] URLs of SDK locations, tried in order
428
444
  attr_accessor :transferd_urls
429
445
 
446
+ # @return [String, nil] URL that identifies the application and its version, sent in header `Referer` when SDK locations are read, `nil`: not sent
447
+ attr_accessor :referer_url
448
+
430
449
  private
431
450
 
451
+ # @param download [Hash, nil] See `sdk_locations`
452
+ # @return [Hash] HTTP headers of the request for SDK locations
453
+ def locations_headers(download)
454
+ return {} if @referer_url.nil?
455
+ {'Referer' => download.nil? ? @referer_url : "#{@referer_url}?#{URI.encode_www_form(download)}"}
456
+ end
457
+
458
+ # Run the block with another SDK folder (files of the SDK are located from it), then set the previous one back
459
+ # @param folder [String] Temporary SDK folder
460
+ def with_sdk_directory(folder)
461
+ previous = Products::Transferd.sdk_directory
462
+ begin
463
+ Products::Transferd.sdk_directory = folder
464
+ return yield
465
+ ensure
466
+ Products::Transferd.sdk_directory = previous
467
+ end
468
+ end
469
+
470
+ # @param path [String] Path to check
471
+ # @param boundary [String] Folder that must contain `path`
472
+ # @return [Boolean] `true` if `path` is `boundary` or is below it (a sibling sharing the name prefix is not)
473
+ def inside_folder?(path, boundary)
474
+ path.eql?(boundary) || path.start_with?(boundary.end_with?(File::SEPARATOR) ? boundary : "#{boundary}#{File::SEPARATOR}")
475
+ end
476
+
432
477
  DEFAULT_ASPERA_CONF = <<~END_OF_CONFIG_FILE
433
478
  <?xml version='1.0' encoding='UTF-8'?>
434
479
  <CONF version="2">
@@ -449,41 +494,24 @@ module Aspera
449
494
  'https://ibm.biz/sdk_location',
450
495
  'https://raw.githubusercontent.com/IBM/aspera-cli/refs/heads/main/docs/sdk_location.yaml'
451
496
  ].freeze
452
- # Timeout in seconds when another location URL remains to be tried
453
- LOCATION_FAIL_FAST_TIMEOUT = 10
497
+ # Version reported in statistics when the latest version is requested (not known before the list is read)
498
+ LATEST_VERSION = 'latest'
454
499
  # filename for ascp with optional extension (Windows)
455
- private_constant :DEFAULT_ASPERA_CONF, :EXE_FILES, :SDK_FILES, :TRANSFERD_ARCHIVE_LOCATION_URLS, :LOCATION_FAIL_FAST_TIMEOUT
500
+ private_constant :DEFAULT_ASPERA_CONF, :EXE_FILES, :SDK_FILES, :TRANSFERD_ARCHIVE_LOCATION_URLS, :LATEST_VERSION
456
501
 
457
502
  def initialize
458
503
  # cache for installed products found
459
504
  @found_products = nil
460
505
  # URLs of SDK locations, tried in order
461
506
  @transferd_urls = TRANSFERD_ARCHIVE_LOCATION_URLS
507
+ # set by the application, `nil`: no `Referer` sent
508
+ @referer_url = nil
462
509
  # product selected with option `sdk_folder`, or nil to use SDK folder
463
510
  @ascp_product = nil
464
511
  # cache for folder of `ascp` of selected product
465
512
  @ascp_folder = nil
466
513
  end
467
514
 
468
- # Read URL with a short timeout and no retry (temporarily changes global HTTP parameters)
469
- # @param url [String] URL to read
470
- # @return [String] content
471
- def read_fail_fast(url)
472
- parameters = Rest::Parameters.instance
473
- session_cb = parameters.session_cb
474
- retry_max = parameters.retry_max
475
- parameters.retry_max = 0
476
- parameters.session_cb = lambda do |http_session|
477
- session_cb&.call(http_session)
478
- http_session.open_timeout = [http_session.open_timeout, LOCATION_FAIL_FAST_TIMEOUT].compact.min
479
- http_session.read_timeout = [http_session.read_timeout, LOCATION_FAIL_FAST_TIMEOUT].compact.min
480
- end
481
- UriReader.read(url)
482
- ensure
483
- parameters.session_cb = session_cb
484
- parameters.retry_max = retry_max
485
- end
486
-
487
515
  # @param file_type [Symbol] one of EXE_FILES
488
516
  # @return [String] path to executable in SDK folder
489
517
  def sdk_exe_path(file_type)