aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -1,17 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore precalc noxfer euid gettime
4
- require 'aspera/assert'
5
- require 'aspera/mime'
6
- require 'aspera/ascp/installation'
7
- require 'aspera/agent/transferd'
8
- require 'aspera/transfer/spec'
9
- require 'aspera/transfer/error'
10
- require 'aspera/secret_hider'
11
- require 'aspera/string_ext'
12
- require 'aspera/log'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'ascp/installation'
8
+ require_relative 'agent/transferd'
9
+ require_relative 'transfer/spec'
10
+ require_relative 'transfer/error'
11
+ require_relative 'secret/hider'
12
+ require_relative 'string_ext'
13
+ require_relative 'log'
14
+ require_relative 'servlet_helpers'
13
15
  require 'webrick'
14
- require 'openssl'
15
16
  require 'etc'
16
17
  require 'securerandom'
17
18
  require 'json'
@@ -296,7 +297,7 @@ module Aspera
296
297
  @node_id = SecureRandom.uuid
297
298
  @cluster_id = SecureRandom.uuid
298
299
  # transferd runs as the user of the emulator
299
- @transfer_user = Etc.getpwuid(Process.euid)&.name || Etc.getlogin.to_s
300
+ @transfer_user = Etc.getpwuid(::Process.euid)&.name || Etc.getlogin.to_s
300
301
  end
301
302
 
302
303
  # Get the node information
@@ -319,59 +320,25 @@ module Aspera
319
320
  Aspera.assert(in_docroot?(File.realpath(folder_path))) { "Path outside docroot: #{path}" }
320
321
  Aspera.assert(File.directory?(folder_path)) { "Not a folder: #{path}" }
321
322
  folder_virtual = virtual_path(folder_path)
322
-
323
- # Build self structure
324
- folder_stat = File.stat(folder_path)
325
- structure = {
326
- 'self' => {
327
- 'path' => folder_virtual,
328
- 'basename' => File.basename(folder_path),
329
- 'type' => 'directory',
330
- # Node API reports 0 for folders
331
- 'size' => 0,
332
- 'mtime' => folder_stat.mtime.utc.iso8601,
333
- 'permissions' => permissions(folder_path)
334
- },
335
- 'items' => []
336
- }
337
-
338
- # Iterate over folder contents
339
- Dir.foreach(folder_path) do |entry|
340
- next if entry == '.' || entry == '..' # Skip current and parent directory
341
-
342
- item_path = File.join(folder_path, entry)
343
- item_type = File.ftype(item_path) rescue 'unknown' # Get the type of file
344
- item_type = FILE_TYPES.fetch(item_type, item_type)
345
- item_stat = File.lstat(item_path) # Use lstat to handle symbolic links correctly
346
-
347
- item = {
348
- 'path' => File.join(folder_virtual, entry),
349
- 'basename' => entry,
350
- 'type' => item_type,
351
- 'size' => item_type.eql?('directory') ? 0 : item_stat.size,
352
- 'mtime' => item_stat.mtime.utc.iso8601,
353
- 'permissions' => permissions(item_path)
354
- }
355
-
356
- # Add additional details for specific types
357
- case item_type
358
- when 'file'
359
- item['partial_file'] = false
360
- when 'symbolic_link'
361
- item['target'] = link_target(item_path)
362
- when 'unknown'
363
- item['note'] = 'File type could not be determined'
323
+ # stable order for paging: only the items of the page are read
324
+ names = Dir.children(folder_path).sort
325
+ items = names.drop(skip).first(count || names.length).map do |name|
326
+ item_path = File.join(folder_path, name)
327
+ # a symbolic link is not followed
328
+ item = item_info(item_path, File.join(folder_virtual, name), File.lstat(item_path))
329
+ case item['type']
330
+ when 'file' then item['partial_file'] = false
331
+ when 'symbolic_link' then item['target'] = link_target(item_path)
332
+ when 'unknown' then item['note'] = 'File type could not be determined'
364
333
  end
365
-
366
- structure['items'] << item
334
+ item
367
335
  end
368
-
369
- # stable order for paging
370
- all_items = structure['items'].sort_by { |item| item['basename'] }
371
- structure['items'] = all_items.drop(skip).first(count || all_items.length)
372
- structure['item_count'] = structure['items'].length
373
- structure['total_count'] = all_items.length
374
- structure
336
+ {
337
+ 'self' => item_info(folder_path, folder_virtual, File.stat(folder_path)),
338
+ 'items' => items,
339
+ 'item_count' => items.length,
340
+ 'total_count' => names.length
341
+ }
375
342
  end
376
343
 
377
344
  # Start a transfer, and monitor it in a thread until terminated
@@ -379,8 +346,8 @@ module Aspera
379
346
  # @return [String] transfer id
380
347
  def start(transfer_spec)
381
348
  Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
382
- # deep copy, returned in transfer `start_spec`
383
- start_spec = SecretHider.instance.deep_remove_secret(JSON.parse(transfer_spec.to_json))
349
+ # copy without secrets, returned in transfer `start_spec`
350
+ start_spec = Secret::Hider.instance.hidden_copy(transfer_spec)
384
351
  Log.dump(:start_spec, start_spec)
385
352
  request = ::Transferd::Api::TransferRequest.new(
386
353
  transferType: ::Transferd::Api::TransferType::FILE_REGULAR,
@@ -469,7 +436,7 @@ module Aspera
469
436
 
470
437
  # @return [Float] current time in seconds, for durations
471
438
  def monotonic_time
472
- Process.clock_gettime(Process::CLOCK_MONOTONIC)
439
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
473
440
  end
474
441
 
475
442
  # Convert a Node API path to a real path
@@ -478,10 +445,23 @@ module Aspera
478
445
  def real_path(path)
479
446
  # `./` prevents expansion of `~`
480
447
  real = File.expand_path("./#{path.to_s.sub(%r{\A/+}, '')}", @docroot)
481
- Aspera.assert(in_docroot?(real)) { "Path outside docroot: #{path}" }
448
+ # symbolic links shall not lead out of the docroot
449
+ resolved = resolved_path(real) if in_docroot?(real)
450
+ Aspera.assert(!resolved.nil? && in_docroot?(resolved)) { "Path outside docroot: #{path}" }
482
451
  real
483
452
  end
484
453
 
454
+ # Resolve symbolic links of the existing part of a path: a destination may not exist yet
455
+ # @param real [String] absolute path, without `..`, within the docroot
456
+ # @return [String, nil] real path of the longest existing part of `real`, `nil` if it is a dangling symbolic link
457
+ def resolved_path(real)
458
+ # stops at the docroot, which exists
459
+ real = File.dirname(real) until File.exist?(real) || File.symlink?(real)
460
+ File.realpath(real)
461
+ rescue Errno::ENOENT, Errno::ELOOP
462
+ nil
463
+ end
464
+
485
465
  # Convert a real path to a Node API path
486
466
  # @param real [String] real path within the docroot
487
467
  # @return [String] path relative to the docroot, with leading `/`
@@ -496,6 +476,23 @@ module Aspera
496
476
  real.eql?(@docroot) || real.start_with?(@docroot.end_with?('/') ? @docroot : "#{@docroot}/")
497
477
  end
498
478
 
479
+ # Get the Node API fields of a folder or of an item of a folder
480
+ # @param real [String] real path within the docroot
481
+ # @param virtual [String] Node API path
482
+ # @param stat [File::Stat] information on the item
483
+ # @return [Hash] Node API fields common to a folder and its items
484
+ def item_info(real, virtual, stat)
485
+ {
486
+ 'path' => virtual,
487
+ 'basename' => File.basename(real),
488
+ 'type' => FILE_TYPES.fetch(stat.ftype, stat.ftype),
489
+ # Node API reports 0 for folders
490
+ 'size' => stat.directory? ? 0 : stat.size,
491
+ 'mtime' => stat.mtime.utc.iso8601,
492
+ 'permissions' => permissions(real)
493
+ }
494
+ end
495
+
499
496
  # Get the Node API permissions of an item
500
497
  # @param real [String] real path within the docroot
501
498
  # @return [Array<Hash>] Node API permissions, from the access of the emulator on the item
@@ -581,6 +578,8 @@ module Aspera
581
578
  # Answers a subset of the Node API, transfers are delegated to a NodeEmulator
582
579
  # a new instance is created for each request
583
580
  class NodeEmulatorServlet < WEBrick::HTTPServlet::AbstractServlet
581
+ include ServletHelpers
582
+
584
583
  PATH_TRANSFERS = '/ops/transfers'
585
584
  PATH_ONE_TRANSFER = %r{/ops/transfers/(.+)$}
586
585
  PATH_BROWSE = '/files/browse'
@@ -589,6 +588,8 @@ module Aspera
589
588
  CANCEL_STATUSES = %w[canceled cancelled stopped].freeze
590
589
  # `PUT` fields modified by transferd
591
590
  MODIFIABLE = %w[target_rate_kbps min_rate_kbps rate_policy].freeze
591
+ # HTTP methods with a JSON body
592
+ JSON_METHODS = %w[POST PUT].freeze
592
593
  # Create the servlet, with optional Basic authentication
593
594
  # @param server [WEBrick::HTTPServer]
594
595
  # @param config [Hash] `username` and `password` (Basic authentication expected from clients, optional)
@@ -599,14 +600,12 @@ module Aspera
599
600
  @expected_auth = "#{config[:username]}:#{config[:password]}" unless config[:username].nil?
600
601
  end
601
602
 
602
- # Check authentication, dispatch to `do_<verb>`, and send errors in Node API format
603
+ # Refuse requests of web pages, check authentication, dispatch to `do_<verb>`, and send errors in Node API format
603
604
  # @param request [WEBrick::HTTPRequest]
604
605
  # @param response [WEBrick::HTTPResponse]
605
606
  def service(request, response)
606
- unless authorized?(request)
607
- response['WWW-Authenticate'] = %Q(Basic realm="#{REALM}")
608
- return set_error(request, response, 401, 'Invalid or missing credentials')
609
- end
607
+ check_request(request, json: JSON_METHODS.include?(request.request_method))
608
+ return set_error(request, response, 401, 'Invalid or missing credentials', headers: {'WWW-Authenticate' => %Q(Basic realm="#{REALM}")}) unless authorized?(request)
610
609
  super
611
610
  rescue WEBrick::HTTPStatus::Error => e
612
611
  set_error(request, response, e.code, e.message)
@@ -623,40 +622,35 @@ module Aspera
623
622
  # @param request [WEBrick::HTTPRequest]
624
623
  # @param response [WEBrick::HTTPResponse]
625
624
  def do_POST(request, response)
626
- case request.path
627
- when PATH_TRANSFERS
628
- set_json_response(request, response, @simulator.transfer(@simulator.start(JSON.parse(request.body))))
629
- when PATH_BROWSE
630
- req = JSON.parse(request.body)
631
- set_json_response(request, response, @simulator.browse(req['path'] || '/', skip: req['skip'].to_i, count: req['count']&.to_i))
632
- else
633
- set_error(request, response, 404, "Unknown path: #{request.path}")
634
- end
625
+ json =
626
+ case request.path
627
+ when PATH_TRANSFERS
628
+ @simulator.transfer(@simulator.start(JSON.parse(request.body.to_s)))
629
+ when PATH_BROWSE
630
+ req = JSON.parse(request.body.to_s)
631
+ @simulator.browse(req['path'] || '/', skip: req['skip'].to_i, count: req['count']&.to_i)
632
+ else raise WEBrick::HTTPStatus::NotFound, "Unknown path: #{request.path}"
633
+ end
634
+ set_json_response(request, response, json)
635
635
  end
636
636
 
637
637
  # Get the node information, the list of transfers, or one transfer
638
638
  # @param request [WEBrick::HTTPRequest]
639
639
  # @param response [WEBrick::HTTPResponse]
640
640
  def do_GET(request, response)
641
- case request.path
642
- when '/info'
643
- set_json_response(request, response, @simulator.info)
644
- when PATH_TRANSFERS
645
- set_json_response(request, response, @simulator.transfers(
646
- active_only: query_boolean(request, 'active_only'),
647
- direction: request.query['direction']&.to_s,
648
- count: query_positive(request, 'count')
649
- ))
650
- when PATH_ONE_TRANSFER
651
- transfer = @simulator.transfer(request.path.match(PATH_ONE_TRANSFER)[1])
652
- if transfer.nil?
653
- set_error(request, response, 404, 'Unknown transfer')
654
- else
655
- set_json_response(request, response, transfer)
641
+ json =
642
+ case request.path
643
+ when '/info' then @simulator.info
644
+ when PATH_TRANSFERS
645
+ @simulator.transfers(
646
+ active_only: query_boolean(request, 'active_only'),
647
+ direction: request.query['direction']&.to_s,
648
+ count: query_positive(request, 'count')
649
+ )
650
+ when PATH_ONE_TRANSFER then known_transfer(@simulator.transfer(transfer_id(request)))
651
+ else raise WEBrick::HTTPStatus::NotFound, "Unknown path: #{request.path}"
656
652
  end
657
- else
658
- set_error(request, response, 404, "Unknown path: #{request.path}")
659
- end
653
+ set_json_response(request, response, json)
660
654
  end
661
655
 
662
656
  # Modify a transfer, or cancel it with `status`
@@ -676,7 +670,7 @@ module Aspera
676
670
  raise WEBrick::HTTPStatus::BadRequest, "Unsupported status: #{status}" unless CANCEL_STATUSES.include?(status)
677
671
  found = @simulator.cancel(id)
678
672
  end
679
- return set_error(request, response, 404, 'Unknown transfer') unless found
673
+ known_transfer(found)
680
674
  set_json_response(request, response, @simulator.transfer(id))
681
675
  end
682
676
 
@@ -684,7 +678,7 @@ module Aspera
684
678
  # @param request [WEBrick::HTTPRequest]
685
679
  # @param response [WEBrick::HTTPResponse]
686
680
  def do_CANCEL(request, response)
687
- return set_error(request, response, 404, 'Unknown transfer') unless @simulator.cancel(transfer_id(request))
681
+ known_transfer(@simulator.cancel(transfer_id(request)))
688
682
  response.status = 204
689
683
  end
690
684
 
@@ -700,6 +694,14 @@ module Aspera
700
694
  match[1]
701
695
  end
702
696
 
697
+ # @param found [Object] Result of the emulator for a transfer: `nil` or `false` if unknown
698
+ # @return [Object] found
699
+ # @raise [WEBrick::HTTPStatus::NotFound] if the transfer is unknown
700
+ def known_transfer(found)
701
+ raise WEBrick::HTTPStatus::NotFound, 'Unknown transfer' unless found
702
+ found
703
+ end
704
+
703
705
  # Read a boolean query parameter
704
706
  # @param request [WEBrick::HTTPRequest]
705
707
  # @param name [String] name of the query parameter
@@ -730,18 +732,16 @@ module Aspera
730
732
  # @param response [WEBrick::HTTPResponse]
731
733
  # @param code [Integer] HTTP status code
732
734
  # @param message [String] message for the user
733
- def set_error(request, response, code, message)
734
- set_json_response(request, response, {error: {code: code, reason: WEBrick::HTTPStatus.reason_phrase(code), user_message: message}}, code: code)
735
+ # @param headers [Hash{String => String}] Other headers of the response
736
+ def set_error(request, response, code, message, headers: {})
737
+ set_json_response(request, response, {error: {code: code, reason: WEBrick::HTTPStatus.reason_phrase(code), user_message: message}}, code: code, headers: headers)
735
738
  end
736
739
 
737
740
  # Check the Basic credentials of the request
738
741
  # @param request [WEBrick::HTTPRequest]
739
742
  # @return [Boolean] `true` if the request has the expected Basic credentials, or if none are expected
740
743
  def authorized?(request)
741
- return true if @expected_auth.nil?
742
- scheme, value = request['Authorization'].to_s.split(' ', 2)
743
- # constant time comparison
744
- scheme.to_s.casecmp?('Basic') && OpenSSL.secure_compare(value.to_s.unpack1('m'), @expected_auth)
744
+ @expected_auth.nil? || basic_authorized?(request, @expected_auth)
745
745
  end
746
746
 
747
747
  # Set the status and the JSON body of the response
@@ -749,11 +749,10 @@ module Aspera
749
749
  # @param response [WEBrick::HTTPResponse]
750
750
  # @param json [Hash, Array] body of the response
751
751
  # @param code [Integer] HTTP status code
752
- def set_json_response(request, response, json, code: 200)
753
- response.status = code
754
- response['Content-Type'] = Mime::JSON
755
- response.body = json.to_json
756
- Log.log.trace1 { Log.obj_dump("response for #{request.request_method} #{request.path}", json) }
752
+ # @param headers [Hash{String => String}] Other headers of the response
753
+ def set_json_response(request, response, json, code: 200, headers: {})
754
+ json_response(response, code, json, headers: headers)
755
+ Log.dump("response for #{request.request_method} #{request.path}", json, level: :trace1)
757
756
  end
758
757
  end
759
758
  end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'factory'
7
+ require_relative '../../log'
8
+ require_relative '../../assert'
9
+ require_relative '../../secret/hider'
10
+ require_relative '../../mime'
11
+ require_relative '../../rainbow'
12
+ require 'date'
13
+ using Rainbow
14
+
15
+ module Aspera
16
+ module OAuth
17
+ # OAuth 2 client for the REST client
18
+ # Generate bearer token
19
+ # Bearer tokens are cached in memory and in a file cache for later re-use
20
+ # OAuth 2.0 Authorization Framework: https://tools.ietf.org/html/rfc6749
21
+ # Bearer Token Usage: https://tools.ietf.org/html/rfc6750
22
+ class Base
23
+ Aspera.require_method!(:create_token)
24
+ # @param params [Hash] Parameters for token creation (client_id, client_secret, scope, etc...)
25
+ # @param path_token [String] API end point to create a token from base URL
26
+ # @param use_query [Boolean] Provide parameters in query instead of body
27
+ # @param token_field [String] Field in result that contains the token
28
+ # @param cache_ids [Array] List of unique identifiers for cache id generation
29
+ # @param rest_params [Hash] Parameters for REST
30
+ def initialize(
31
+ params: {},
32
+ path_token: 'token',
33
+ use_query: false,
34
+ token_field: Factory::TOKEN_FIELD,
35
+ cache_ids: [],
36
+ **rest_params
37
+ )
38
+ Aspera.assert_type(params, Hash)
39
+ Aspera.assert_type(cache_ids, Array)
40
+ # This is the OAuth API
41
+ @api = Rest::Client.new(**rest_params)
42
+ @params = params.dup.freeze
43
+ @path_token = path_token
44
+ @token_field = token_field
45
+ @use_query = use_query
46
+ @token_cache_id = Factory.cache_id(@api.base_url, self.class, *cache_ids, @params[:scope])
47
+ end
48
+
49
+ # The OAuth API Object
50
+ attr_reader :api
51
+ # Parameters to generate token
52
+ attr_reader :params
53
+ # Sub path to generate token
54
+ attr_reader :path_token
55
+
56
+ # Parameters for token creation call
57
+ # @param include_secret [Boolean] Add secret in default call parameters
58
+ # @param other_params [Hash] Other parameters to merge
59
+ # @return [Hash] Token creation parameters
60
+ def base_parameters(include_secret: false, **other_params)
61
+ (include_secret ? @params : @params.except(:client_secret)).merge(other_params)
62
+ end
63
+
64
+ # Helper method to create token as per RFC
65
+ # @param include_secret [Boolean] Add secret in call parameters
66
+ # @param other_params [Hash] Additional parameter to base parameters
67
+ # @return [Net::HTTPResponse] Raw HTTP response with token
68
+ # @raise [Rest::CallError] If not 2XX code
69
+ def create_token_base(include_secret: false, **other_params)
70
+ Log.log.debug { 'Generating a new token'.bg(:green) }
71
+ creation_params = base_parameters(include_secret: include_secret, **other_params)
72
+ return @api.create(@path_token, nil, query: creation_params, ret: :resp) if @use_query
73
+ @api.create(@path_token, creation_params, content_type: Mime::WWW, ret: :resp)
74
+ end
75
+
76
+ # Get the value of the HTTP `Authorization` header for this OAuth client, i.e. `Bearer <token>`.
77
+ # The bearer token is obtained through {#token} (cached, refreshed or generated).
78
+ # @return [String] Value suitable for Authorization header
79
+ def authorization
80
+ OAuth::Factory.bearer_authorization(token)
81
+ end
82
+
83
+ # Get the value of the HTTP `Authorization` header with a renewed token, i.e. `Bearer <token>`.
84
+ # To be used when the server rejected the current token.
85
+ # The bearer token is obtained through {#renew_token}.
86
+ # @return [String] Value suitable for Authorization header
87
+ def renewed_authorization
88
+ OAuth::Factory.bearer_authorization(renew_token)
89
+ end
90
+
91
+ # Get a OAuth v2 token: the cached one, unless it expires soon, else a renewed one.
92
+ # Checking expiration in advance helps a transfer agent which cannot renew the token itself
93
+ # (`direct` agent is equipped with renewal code).
94
+ # If the server rejects the token (API returns 4xx), call {#renew_token}.
95
+ # @return [String] The bearer token
96
+ def token
97
+ token_info = cached_token_info
98
+ unless token_info.nil?
99
+ return bearer_token(token_info[:data]) unless token_info[:expired]
100
+ Log.log.trace1 { 'Cached token expires soon' }
101
+ end
102
+ renew_token
103
+ end
104
+
105
+ # Get a new OAuth v2 token, discarding the cached one.
106
+ # Use the refresh token of the cached token, if any, else generate a new token.
107
+ # @return [String] The bearer token
108
+ # @raise [StandardError] Error of the generation of a new token
109
+ def renew_token
110
+ bearer_token(refreshed_token_data(discard_cached_token) || save_token_data(create_token))
111
+ end
112
+
113
+ private
114
+
115
+ # @return [Hash, nil] Token information from cache (see {Factory#get_token_info}), or `nil` if none
116
+ def cached_token_info
117
+ Factory.instance.get_token_info(@token_cache_id, token_field: @token_field)
118
+ end
119
+
120
+ # Delete the cached token
121
+ # @return [String, nil] Its refresh token, or `nil` if none
122
+ def discard_cached_token
123
+ token_info = cached_token_info
124
+ return if token_info.nil?
125
+ Factory.instance.persist_mgr.delete(@token_cache_id)
126
+ refresh_token = token_info[:data]['refresh_token']
127
+ refresh_token unless refresh_token.eql?('not_supported')
128
+ end
129
+
130
+ # Get a new token using a refresh token.
131
+ # NOTE: AoC admin token has no refresh token, and lives by default 1800 secs
132
+ # @param refresh_token [String, nil] Refresh token, or `nil` if none
133
+ # @return [Hash{String => Object}, nil] Token creation response, or `nil` if no refresh token or if refresh failed
134
+ def refreshed_token_data(refresh_token)
135
+ return if refresh_token.nil?
136
+ save_token_data(create_token_base(include_secret: true, grant_type: 'refresh_token', refresh_token: refresh_token))
137
+ rescue StandardError => e
138
+ # Refresh token can fail: a new token will be generated
139
+ Log.log.warn { "Refresh failed: #{e}" }
140
+ nil
141
+ end
142
+
143
+ # Cache the token creation response
144
+ # @param http [Net::HTTPResponse] Response of the token creation (see {#create_token_base})
145
+ # @return [Hash{String => Object}] Token creation response: `@token_field`, optional `refresh_token`, `expires_in`, ...
146
+ def save_token_data(http)
147
+ token_data = JSON.parse(http.body)
148
+ Factory.instance.persist_mgr.put(@token_cache_id, http.body)
149
+ # Logs decoded token information
150
+ cached_token_info
151
+ token_data
152
+ end
153
+
154
+ # @param token_data [Hash{String => Object}] Token creation response
155
+ # @return [String] The bearer token, registered as secret: hidden wherever it appears (logs, results)
156
+ def bearer_token(token_data)
157
+ Aspera.assert(token_data.key?(@token_field)) { "API error: No such field in answer: #{@token_field}" }
158
+ Secret::Hider.instance.add_secret_values(token_data[@token_field], token_data['refresh_token'])
159
+ token_data[@token_field]
160
+ end
161
+ end
162
+ end
163
+ end
@@ -1,7 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/oauth/base'
4
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'base'
7
+ require_relative '../../assert'
8
+ require_relative '../../environment'
5
9
  require 'json'
6
10
 
7
11
  module Aspera
@@ -12,7 +16,7 @@ module Aspera
12
16
  class Boot < Base
13
17
  # @param cookie [String, nil] Raw cookie string (--password), nil to rely on existing cache
14
18
  # @param username [String, nil] Expected subject; if provided, must match token's `sub` claim
15
- # @param **base_params [Hash] Forwarded to Base (base_url:, params: {client_id:, scope:}, etc.)
19
+ # @param base_params [Hash] Forwarded to Base (base_url:, params: {client_id:, scope:}, etc.)
16
20
  def initialize(cookie: nil, username: nil, **base_params)
17
21
  if cookie.nil?
18
22
  # No cookie: rely on existing cache, identified by username if provided
@@ -35,7 +39,7 @@ module Aspera
35
39
 
36
40
  # Should never be reached: if cache and refresh are both exhausted, re-authenticate via browser
37
41
  def create_token
38
- Aspera.report_error(AssertError, 'Boot: token expired and no refresh available — re-authenticate in browser')
42
+ Aspera.report_error(AssertError, 'Boot: token expired and no refresh available. Re-authenticate in browser')
39
43
  end
40
44
  end
41
45
  Factory.instance.register_token_creator(Boot)
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/id_generator'
4
- require 'aspera/assert'
5
- require 'aspera/string_ext'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../../id_generator'
7
+ require_relative '../../assert'
8
+ require_relative '../../string_ext'
6
9
  require 'singleton'
7
10
  require 'base64'
8
11
  module Aspera
@@ -193,6 +196,13 @@ module Aspera
193
196
  def create(**parameters)
194
197
  Aspera.assert_type(parameters, Hash)
195
198
  id = parameters[:grant_method]
199
+ unless @token_type_classes.key?(id)
200
+ begin
201
+ require "aspera/oauth/client/#{id}"
202
+ rescue LoadError
203
+ # ignored, handled by assert
204
+ end
205
+ end
196
206
  Aspera.assert(@token_type_classes.key?(id)) { "token grant method unknown: '#{id}'" }
197
207
  create_parameters = parameters.reject { |k, _v| k.eql?(:grant_method) }
198
208
  @token_type_classes[id].new(**create_parameters)
@@ -1,6 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/oauth/base'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'base'
7
+ require_relative 'factory'
4
8
 
5
9
  module Aspera
6
10
  module OAuth
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/oauth/base'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'base'
4
7
  require 'digest'
5
8
 
6
9
  module Aspera
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/oauth/base'
4
- require 'aspera/environment'
5
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'base'
7
+ require_relative '../../environment'
8
+ require_relative '../../assert'
6
9
  require 'securerandom'
7
10
  require 'openssl'
8
11
  module Aspera