aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -1,7 +1,52 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  module Cli
8
+ # Exit status of the process, by category of error
9
+ module ExitCode
10
+ SUCCESS = 0
11
+ ERROR = 1
12
+ USAGE = 2
13
+ AUTH = 3
14
+ NOT_FOUND = 4
15
+ TRANSFER = 5
16
+ NETWORK = 6
17
+ API = 7
18
+ # 128 + SIGINT, as shells do
19
+ INTERRUPTED = 130
20
+ # For help and documentation
21
+ DESCRIPTIONS = {
22
+ SUCCESS => 'Success',
23
+ ERROR => 'Other error',
24
+ USAGE => 'Command line error: unknown command or option, invalid or missing argument',
25
+ AUTH => 'Authentication or authorization failure: SSH, HTTP 401 or 403, OAuth grant',
26
+ NOT_FOUND => 'Resource not found, or several resources match the identifier, HTTP 404',
27
+ TRANSFER => 'Transfer failure',
28
+ NETWORK => 'Network or TLS failure: name resolution, connection, timeout, certificate',
29
+ API => 'Other error returned by a remote API',
30
+ INTERRUPTED => 'Interrupted by user (Ctrl-C)'
31
+ }.freeze
32
+ # OAuth 2.0 error codes of a refused authentication (RFC 6749, RFC 6750)
33
+ OAUTH_AUTH_ERRORS = %w[invalid_client invalid_grant unauthorized_client access_denied invalid_token insufficient_scope].freeze
34
+ private_constant :OAUTH_AUTH_ERRORS
35
+
36
+ class << self
37
+ # @param error [Rest::CallError] Failed API call
38
+ # @return [Integer] Exit code for the HTTP status, or OAuth error
39
+ def for_call_error(error)
40
+ return AUTH if error.data.is_a?(Hash) && OAUTH_AUTH_ERRORS.include?(error.data['error'])
41
+ case error.response&.code
42
+ when '401', '403' then AUTH
43
+ when '404' then NOT_FOUND
44
+ else API
45
+ end
46
+ end
47
+ end
48
+ end
49
+
5
50
  # CLI base exception
6
51
  class Error < StandardError; end
7
52
 
@@ -1,18 +1,23 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore csvt jsonpp stdbin
4
- require 'aspera/uri_reader'
5
- require 'aspera/environment'
6
- require 'aspera/yaml'
7
- require 'aspera/log'
8
- require 'aspera/assert'
9
- require 'aspera/cli/error'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../uri_reader'
7
+ require_relative '../environment'
8
+ require_relative '../yaml'
9
+ require_relative '../log'
10
+ require_relative '../assert'
11
+ require_relative '../secret/hider'
12
+ require_relative 'error'
13
+ require_relative 'prompt'
14
+ require_relative 'security'
15
+ require_relative '../rainbow'
10
16
  require 'json'
11
17
  require 'base64'
12
18
  require 'zlib'
13
19
  require 'csv'
14
20
  require 'singleton'
15
- require 'aspera/rainbow'
16
21
  using Rainbow
17
22
 
18
23
  module Aspera
@@ -26,7 +31,7 @@ module Aspera
26
31
  include Singleton
27
32
 
28
33
  # First is default
29
- DEFAULT_DECODERS = %i[none json ruby yaml]
34
+ DEFAULT_DECODERS = %i[none json ruby yaml].freeze
30
35
 
31
36
  class << self
32
37
  # Decode comma separated table text
@@ -73,6 +78,16 @@ module Aspera
73
78
  Aspera.assert(value.empty?, type: BadArgument) { "no value allowed for extended value type: #{ext_type}" }
74
79
  end
75
80
 
81
+ # Ask the user for a secret: on a web page if the user interface is graphical (option `ui`), else on terminal
82
+ # @param prompt [String] Name of the secret, empty for default
83
+ # @return [String] The secret
84
+ def read_secret(prompt)
85
+ prompt = 'secret' if prompt.empty?
86
+ return Prompt.prompt_user_input(prompt, sensitive: true) unless Environment.instance.url_method.eql?(:graphical)
87
+ require_relative 'web_secret'
88
+ WebSecret.ask(prompt)
89
+ end
90
+
76
91
  def read_stdin(mode)
77
92
  case mode
78
93
  when '' then $stdin.read
@@ -95,7 +110,7 @@ module Aspera
95
110
  base64: lambda { |i| Base64.decode64(i) },
96
111
  csvt: lambda { |i| ExtendedValue.decode_csvt(i) },
97
112
  env: lambda { |i| ENV.fetch(i, nil) },
98
- file: lambda { |i| File.read(File.expand_path(i)) },
113
+ file: lambda { |i| File.read(Security.instance.readable_path(i)) },
99
114
  uri: lambda { |i| UriReader.read(i) },
100
115
  json: lambda { |i| ExtendedValue.JSON_parse(i) },
101
116
  lines: lambda { |i| i.split("\n") },
@@ -105,24 +120,48 @@ module Aspera
105
120
  re: lambda { |i| Regexp.new(i, Regexp::MULTILINE) },
106
121
  ruby: lambda { |i| Environment.secure_eval(i, __FILE__, __LINE__) },
107
122
  s: lambda(&:to_s),
108
- secret: lambda { |i| prompt = i.empty? ? 'secret' : i; $stdin.getpass("#{prompt}> ") }, # rubocop:disable Style/Semicolon
123
+ secret: lambda { |i| ExtendedValue.read_secret(i) },
109
124
  stdin: lambda { |i| ExtendedValue.read_stdin(i) },
110
125
  yaml: lambda { |i| YAML.safe_load(i) },
111
126
  zlib: lambda { |i| Zlib::Inflate.inflate(i) },
112
127
  extend: lambda { |i| ExtendedValue.instance.evaluate_extend(i) }
113
128
  }
129
+ # Descriptions of handlers, for shell completion
130
+ # @type [Hash{Symbol => String}]
131
+ @descriptions = {
132
+ val: 'Value as is: modifiers on the right are not evaluated',
133
+ base64: 'Decode Base64',
134
+ csvt: 'Decode titled CSV: Array of Hash',
135
+ env: 'Value of environment variable',
136
+ file: 'Content of local file',
137
+ uri: 'Content of URL: http:, https:, data:, file:',
138
+ json: 'Decode JSON',
139
+ lines: 'Split lines: Array',
140
+ list: 'Split on first character: Array',
141
+ none: 'Null value',
142
+ path: 'Local path, with ~ expanded',
143
+ re: 'Regular expression',
144
+ ruby: 'Evaluate Ruby code',
145
+ s: 'Convert to String',
146
+ secret: 'Ask secret, with prompt',
147
+ stdin: 'Read standard input: empty, bin or chomp',
148
+ yaml: 'Decode YAML',
149
+ zlib: 'Decompress zlib',
150
+ extend: 'Evaluate embedded @<modifier>:<value>@'
151
+ }
114
152
  @regex_single = nil
115
- @regex_extend = nil
153
+ @regex_inner = nil
116
154
  @default_decoder = nil
117
155
  update_regex
118
156
  end
119
157
 
120
158
  # Update the Regex to match an extended value based on @handlers
121
- # Anchored on the whole value (`\A`, `\z`), not on lines: a multi-line value is extended only if it starts with a modifier.
159
+ # Single: anchored on the whole value (`\A`, `\z`), not on lines: a multi-line value is extended only if it starts with a modifier.
160
+ # Inner: an extended value embedded in a text, for `@extend:`
122
161
  def update_regex
123
- handler_regex = "#{MARKER_START}(#{modifiers.join('|')})#{MARKER_END}"
162
+ handler_regex = "#{MARKER_START}(#{Regexp.union(modifiers.map(&:to_s)).source})#{MARKER_END}"
124
163
  @regex_single = Regexp.new("\\A#{handler_regex}(.*)\\z", Regexp::MULTILINE)
125
- @regex_extend = Regexp.new("\\A(.*)#{handler_regex}([^#{MARKER_IN_END}]*)#{MARKER_IN_END}(.*)\\z", Regexp::MULTILINE)
164
+ @regex_inner = Regexp.new("#{handler_regex}([^#{MARKER_IN_END}]*)#{MARKER_IN_END}")
126
165
  end
127
166
 
128
167
  public
@@ -139,12 +178,19 @@ module Aspera
139
178
  # List of Extended Value methods
140
179
  def modifiers; @handlers.keys; end
141
180
 
142
- # Add a new handler
143
- def on(name, &block)
181
+ # @param name [Symbol] Extended Value method
182
+ # @return [String, nil] description of the method, for shell completion
183
+ def description(name) = @descriptions[name]
184
+
185
+ # Add a new handler, or replace one
186
+ # @param name [Symbol] method, used as `@<name>:`
187
+ # @param description [String, nil] description, for shell completion; `nil` keeps the current one
188
+ def on(name, description: nil, &block)
144
189
  Aspera.assert_type(name, Symbol) { 'name' }
145
190
  Aspera.assert(block) { "block required when registering handler for #{name}" }
146
191
  Log.log.debug { "Setting handler for #{name}" }
147
192
  @handlers[name] = block
193
+ @descriptions[name] = description unless description.nil?
148
194
  update_regex
149
195
  end
150
196
 
@@ -170,23 +216,34 @@ module Aspera
170
216
  break if SPECIAL_HANDLERS.include?(handler)
171
217
  end
172
218
  Log.log.trace1 { "evaluating: #{handlers_reversed}, value: #{value.class}:#{value}" }
173
- handlers_reversed.each do |handler|
219
+ handlers_reversed.each_with_index do |handler, index|
220
+ Security.instance.check_extended_value(handler)
221
+ # The value read by the previous handler may be a secret
222
+ Security.instance.check_secret_processing("@#{handler}") { Secret::Hider.instance.contains_secret?(value) } if index.positive?
174
223
  value = @handlers[handler].call(value)
224
+ # Values read from a secret source are hidden wherever they appear: logs, results
225
+ if SECRET_HANDLERS.include?(handler)
226
+ value.is_a?(String) ? Secret::Hider.instance.add_secret_values(value) : Secret::Hider.instance.add_secrets_in(value)
227
+ end
175
228
  rescue => e
176
- raise BadArgument, "Evaluation of #{handler} for #{context}: #{e.message}"
229
+ raise BadArgument, "Evaluation of #{MARKER_START}#{handler}#{MARKER_END} for #{context}: #{e.message}"
177
230
  end
178
231
  return value
179
232
  end
180
233
 
181
234
  # Find inner extended values
235
+ # Evaluated values are inserted as is: not evaluated again
182
236
  # Only used in above lambda
183
237
  def evaluate_extend(value)
184
- while (m = value.match(@regex_extend))
185
- sub_value = "@#{m[2]}:#{m[3]}"
238
+ value.gsub(@regex_inner) do
239
+ m = Regexp.last_match
240
+ sub_value = "@#{m[1]}:#{m[2]}"
186
241
  Log.log.debug { "evaluating #{sub_value}" }
187
- value = "#{m[1]}#{evaluate(sub_value, context: 'composite extended value')}#{m[4]}"
242
+ sub_value = evaluate(sub_value, context: 'composite extended value')
243
+ # In a text, a secret is not the whole value
244
+ Security.instance.check_secret_processing('@extend') { Secret::Hider.instance.contains_secret?(sub_value) }
245
+ sub_value.to_s
188
246
  end
189
- return value
190
247
  end
191
248
  # marker "@"
192
249
  MARKER_START = '@'
@@ -199,10 +256,12 @@ module Aspera
199
256
  # :extend includes processing of other handlers in itself
200
257
  # :val keeps the value intact
201
258
  SPECIAL_HANDLERS = %i[extend val].freeze
259
+ # Handlers that read a secret
260
+ SECRET_HANDLERS = %i[secret vault].freeze
202
261
 
203
262
  # Array and Hash types:
204
263
  DEFAULT_PARSER_TYPES = [Array, Hash].freeze
205
- private_constant :MARKER_START, :MARKER_END, :MARKER_IN_END, :SPECIAL_HANDLERS, :DEFAULT_PARSER_TYPES
264
+ private_constant :MARKER_START, :MARKER_END, :MARKER_IN_END, :SPECIAL_HANDLERS, :SECRET_HANDLERS, :DEFAULT_PARSER_TYPES
206
265
  end
207
266
  end
208
267
  end
@@ -1,24 +1,26 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:ignore jsonpp
4
- require 'aspera/schema/registry'
5
- require 'aspera/cli/option_declarator'
6
- require 'aspera/cli/special_values'
7
- require 'aspera/cli/terminal_formatter'
8
- require 'aspera/preview/terminal'
9
- require 'aspera/secret_hider'
10
- require 'aspera/environment'
11
- require 'aspera/log'
12
- require 'aspera/assert'
13
- require 'aspera/markdown'
14
- require 'aspera/dot_container'
15
- require 'terminal-table'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../schema/registry'
7
+ require_relative 'option_declarator'
8
+ require_relative 'special_values'
9
+ require_relative 'terminal_formatter'
10
+ require_relative 'table_fitter'
11
+ require_relative '../preview/terminal'
12
+ require_relative '../secret/hider'
13
+ require_relative '../environment'
14
+ require_relative '../log'
15
+ require_relative '../assert'
16
+ require_relative '../markdown'
17
+ require_relative '../dot_container'
18
+ require_relative '../rainbow'
16
19
  require 'tty-spinner'
20
+ require 'json'
17
21
  require 'yaml'
18
22
  require 'pp'
19
23
  require 'csv'
20
- require 'word_wrap'
21
- require 'aspera/rainbow'
22
24
  using Rainbow
23
25
 
24
26
  module Aspera
@@ -37,13 +39,15 @@ module Aspera
37
39
  SINGLE_OBJECT_COLUMN_NAMES = %i[field value].freeze
38
40
  # Terminal: vertical separator for list of strings.
39
41
  STR_LST_SEP_VERT = "\n"
42
+ # Sub-options of option `out`, for help and errors
43
+ OUT_SUB_OPTIONS = 'format, level, file, fields, select, query, table[.pivot], flat, secrets, colors, utf8, img'
40
44
 
41
- private_constant :FIELDS_LESS, :DISPLAY_FORMATS, :DISPLAY_LEVELS, :SINGLE_OBJECT_COLUMN_NAMES, :STR_LST_SEP_VERT
45
+ private_constant :FIELDS_LESS, :DISPLAY_FORMATS, :DISPLAY_LEVELS, :SINGLE_OBJECT_COLUMN_NAMES, :STR_LST_SEP_VERT, :OUT_SUB_OPTIONS
42
46
 
43
- option :out, description: 'Output rendering options (dot-notation: format, level, file, fields, select, table[.pivot], flat, secrets, colors, utf8, img)', schema: Schema::Registry::OUT_OPTIONS
47
+ option :out, description: "Output rendering options (dot-notation: #{OUT_SUB_OPTIONS})", schema: Schema::Registry::OUT_OPTIONS
44
48
  option :display, description: 'Output only some information', allowed: DISPLAY_LEVELS, default: :data, deprecation: {last: '4.27.0', message: 'use --out.level'}
45
49
  option :format, description: 'Output format (also: --out.format)', allowed: DISPLAY_FORMATS, default: :table
46
- option :output, description: 'Destination for results', deprecation: {last: '4.27.0', message: 'use --out.file'}
50
+ option :output, description: 'Destination for results', deprecation: {last: '4.27.0', message: 'use --out.file'}, complete: :file, security: :low
47
51
  option :fields, description: "Comma separated list of: fields, or #{SpecialValues::ALL}, or #{SpecialValues::DEF} (also: --out.fields)", allowed: [String, Array, Regexp, Proc], default: SpecialValues::DEF
48
52
  option :select, description: 'Select only some items in lists: column, value (also: --out.select)', allowed: [Hash, Proc]
49
53
  option :table_style, description: '(Table) Display style', allowed: [Hash], deprecation: {last: '4.27.0', message: 'use --out.table'}
@@ -56,35 +60,23 @@ module Aspera
56
60
 
57
61
  class << self
58
62
  # Replace special values with a readable version on terminal
63
+ # @param input_hash [Hash] Object to display (not modified)
64
+ # @param string_list_separator [String] Separator of the items of a list of strings
65
+ # @return [Hash] Copy of input_hash with special values replaced, also in nested objects
59
66
  def replace_specific_for_terminal(input_hash, string_list_separator)
60
- hash_to_process = [input_hash]
61
- until hash_to_process.empty?
62
- current = hash_to_process.pop
63
- current.each do |key, value|
64
- case value
65
- when NilClass
66
- current[key] = TerminalFormatter.special_format('null')
67
- when String
68
- current[key] = TerminalFormatter.special_format('empty string') if value.empty?
69
- when Proc
70
- current[key] = TerminalFormatter.special_format('lambda')
71
- when Array
72
- if value.empty?
73
- current[key] = TerminalFormatter.special_format('empty list')
74
- elsif value.all?(String)
75
- current[key] = value.join(string_list_separator)
76
- else
77
- value.each do |item|
78
- hash_to_process.push(item) if item.is_a?(Hash)
79
- end
80
- end
81
- when Hash
82
- if value.empty?
83
- current[key] = TerminalFormatter.special_format('empty dict')
84
- else
85
- hash_to_process.push(value)
86
- end
67
+ input_hash.transform_values do |value|
68
+ case value
69
+ when NilClass then TerminalFormatter.special_format('null')
70
+ when String then value.empty? ? TerminalFormatter.special_format('empty string') : value
71
+ when Proc then TerminalFormatter.special_format('lambda')
72
+ when Array
73
+ if value.empty? then TerminalFormatter.special_format('empty list')
74
+ elsif value.all?(String) then value.join(string_list_separator)
75
+ else value.map { |item| item.is_a?(Hash) ? replace_specific_for_terminal(item, string_list_separator) : item }
87
76
  end
77
+ when Hash
78
+ value.empty? ? TerminalFormatter.special_format('empty dict') : replace_specific_for_terminal(value, string_list_separator)
79
+ else value
88
80
  end
89
81
  end
90
82
  end
@@ -102,8 +94,12 @@ module Aspera
102
94
  @options = {}
103
95
  # Last value of option `out`
104
96
  @out_dispatched = {}
105
- # Colors auto-detected by Rainbow at load time, restored when `--out.colors` is reset
97
+ # Colors auto-detected at load time (aspera/rainbow), restored when `--out.colors` is reset
106
98
  @colors_detected = Rainbow.enabled
99
+ # Value of `--out.colors`: `true`, `false`, or `nil` for detection
100
+ @colors = nil
101
+ # `true` if results are written to a file (`--out.file`)
102
+ @output_to_file = false
107
103
  @spinner = nil
108
104
  end
109
105
 
@@ -185,9 +181,12 @@ module Aspera
185
181
  when :flat then @parser.set_option(:flat_hash, v, warn_deprecation: false)
186
182
  when :secrets then @parser.set_option(:show_secrets, v, warn_deprecation: false)
187
183
  when :img then @parser.set_option(:image, v, warn_deprecation: false)
188
- when :colors then Rainbow.enabled = v.nil? ? @colors_detected : v
184
+ when :query then self.query = v
189
185
  when :utf8 then Environment.unicode = v
190
- else Aspera.error_unexpected_value(k) { 'out sub-option (format, level, file, fields, select, table[.pivot], flat, secrets, colors, utf8, img)' }
186
+ when :colors
187
+ @colors = v
188
+ apply_colors
189
+ else Aspera.error_unexpected_value(k) { "out sub-option (#{OUT_SUB_OPTIONS})" }
191
190
  end
192
191
  end
193
192
  return
@@ -199,11 +198,13 @@ module Aspera
199
198
  # The default output level depends on the format, unless the level is set explicitly (e.g. `--out.level`)
200
199
  @options[:display] = value.eql?(:table) ? :info : :data if @parser.nil? || @parser.option_def(:display).source.eql?(:default)
201
200
  when :output
202
- $stdout = if value.eql?('-')
203
- STDOUT # rubocop:disable Style/GlobalStdStream
204
- else
201
+ @output_to_file = !value.eql?('-')
202
+ $stdout = if @output_to_file
205
203
  File.open(value, 'w')
204
+ else
205
+ STDOUT
206
206
  end
207
+ apply_colors
207
208
  when :image
208
209
  # get list if key arguments of method
209
210
  allowed_options = Preview::Terminal.method(:build).parameters.select { |i| i[0].eql?(:key) }.map { |i| i[1] }
@@ -214,6 +215,28 @@ module Aspera
214
215
  nil
215
216
  end
216
217
 
218
+ # Enable colors as set by `--out.colors`, else as detected, but not when results go to a file
219
+ # @return [nil]
220
+ def apply_colors
221
+ Rainbow.enabled = @colors.nil? ? @colors_detected && !@output_to_file : @colors
222
+ nil
223
+ end
224
+
225
+ # Set the query of `--out.query`, syntax of a JMESPath expression is checked before the command is executed
226
+ # @param expression [String, Proc, nil] JMESPath expression, or Ruby lambda, or `nil` for none
227
+ def query=(expression)
228
+ Aspera.assert_type(expression, String, Proc, NilClass, type: Cli::BadArgument) { 'out.query' }
229
+ if expression.is_a?(String)
230
+ require 'jmespath'
231
+ begin
232
+ JMESPath::Parser.new.parse(expression)
233
+ rescue JMESPath::Errors::SyntaxError => e
234
+ raise Cli::BadArgument, "JMESPath: #{e.message}"
235
+ end
236
+ end
237
+ @options[:query] = expression
238
+ end
239
+
217
240
  # Main output method for displaying messages to the user
218
241
  # This method is used by Result classes to output formatted data
219
242
  # @param message_level [Symbol] The level of the message - must be one of: :data, :info, :error
@@ -225,7 +248,7 @@ module Aspera
225
248
  # - `:info` messages are only displayed when display level is `:info`
226
249
  # - `:error` messages are always displayed on `stderr`
227
250
  def display_message(message_level, message, hide_secrets: true)
228
- message = SecretHider.instance.hide_secrets_in_string(message) if hide_secrets && message.is_a?(String) && hide_secrets?
251
+ message = Secret::Hider.instance.hide_secrets_in_string(message) if hide_secrets && message.is_a?(String) && hide_secrets?
229
252
  case message_level
230
253
  when :data then $stdout.puts(message) unless @options[:display].eql?(:error)
231
254
  when :info then $stdout.puts(message) if @options[:display].eql?(:info)
@@ -238,14 +261,15 @@ module Aspera
238
261
  display_message(:info, status, **kwargs)
239
262
  end
240
263
 
241
- # Check if secrets should be hidden
264
+ # Check if secrets should be hidden: always, unless explicitly requested (`--out.secrets=yes`)
242
265
  def hide_secrets?
243
- !@options[:show_secrets] && !@options[:display].eql?(:data)
266
+ !@options[:show_secrets]
244
267
  end
245
268
 
246
- # Hides secrets in Hash or Array
269
+ # @param data [Object] Data of a result
270
+ # @return [Object] Copy of data with secrets hidden (by key name, and declared values), or data itself if secrets are shown
247
271
  def hide_secrets(data)
248
- SecretHider.instance.deep_remove_secret(data) if hide_secrets?
272
+ hide_secrets? ? Secret::Hider.instance.hidden_copy(data) : data
249
273
  end
250
274
 
251
275
  # Display results using the Visitor pattern
@@ -253,15 +277,17 @@ module Aspera
253
277
  # formatter method, eliminating the need for type checking in the formatter.
254
278
  # @param result [Result] Result object to display
255
279
  def display_results(result)
256
- require 'aspera/cli/result'
280
+ require_relative 'result'
257
281
  Aspera.assert_type(result, Cli::Result) { 'result must be a Result object' }
258
282
 
259
283
  Log.log.debug { "display_results: result class=#{result.class.name}" }
260
284
  Log.dump(:data, result.data, level: :trace1)
261
285
  Log.dump(:fields, result.fields, level: :trace1)
262
286
 
263
- # Hide secrets in data
264
- hide_secrets(result.data)
287
+ # Hide secrets in a copy of data: it may be shared (e.g. a preset of the configuration)
288
+ result = result.with_data(hide_secrets(result.data)) if hide_secrets?
289
+ # Secrets are hidden before the expression may rename their keys
290
+ result = query_result(result) unless @options[:query].nil?
265
291
 
266
292
  # Use the Visitor pattern: delegate formatting to the result object
267
293
  result.format(self)
@@ -280,7 +306,7 @@ module Aspera
280
306
 
281
307
  # Hide secrets in a string
282
308
  def hide_secrets_in_string(string)
283
- SecretHider.instance.hide_secrets_in_string(string)
309
+ Secret::Hider.instance.hide_secrets_in_string(string)
284
310
  end
285
311
 
286
312
  # Get special format string
@@ -357,9 +383,9 @@ module Aspera
357
383
  # no filter for single element
358
384
  return data unless data.is_a?(Array)
359
385
  # by default, keep all data intact
360
- return data if @options[:fields].eql?(SpecialValues::DEF) && @options[:select].nil?
386
+ return data if @options[:fields].eql?(SpecialValues::DEF) && [nil, {}].include?(@options[:select])
361
387
  Aspera.assert_array_all(data, Hash) { 'filter or select' }
362
- filter_columns_on_select(data)
388
+ data = filter_columns_on_select(data)
363
389
  return data if @options[:fields].eql?(SpecialValues::DEF)
364
390
  selected_fields = compute_fields(data, @options[:fields])
365
391
  return data.map { |i| i[selected_fields.first] } if selected_fields.length == 1
@@ -367,20 +393,58 @@ module Aspera
367
393
  end
368
394
 
369
395
  # Filter the list of items on the select option
370
- # @param data [Array<Hash>] List of items
371
- # @return [nil]
396
+ # @param data [Array<Hash>] List of items (not modified)
397
+ # @return [Array<Hash>] Selected items
372
398
  def filter_columns_on_select(data)
373
399
  case @options[:select]
374
400
  when Proc
375
401
  begin
376
- data.select! { |i| @options[:select].call(i) }
402
+ data.select { |i| @options[:select].call(i) }
377
403
  rescue StandardError => e
378
404
  raise Cli::BadArgument, "Error in user-provided ruby lambda code during select: #{e.message}"
379
405
  end
380
406
  when Hash
381
- @options[:select].each { |k, v| data.select! { |i| i[k].eql?(v) } }
407
+ data.select do |item|
408
+ # Not flattened with a structured format: a dot-path key is looked up in the flattened item, as in a table
409
+ flat = nil
410
+ @options[:select].all? { |k, v| (item.key?(k) ? item[k] : (flat ||= DotContainer.new(item).to_dotted)[k]).eql?(v) }
411
+ end
412
+ else data
382
413
  end
383
- nil
414
+ end
415
+
416
+ # Apply the query of `--out.query` to the data of an object or list result
417
+ # @param result [Result] Result of command
418
+ # @return [Result] Result with the value of the expression, or `result` if no query, or not an object or list
419
+ def query_result(result)
420
+ return result if @options[:query].nil?
421
+ data =
422
+ case result
423
+ when Result::SingleObject, Result::ObjectList, Result::ValueList then result.data
424
+ when Result::Empty then []
425
+ else return result
426
+ end
427
+ value =
428
+ if @options[:query].is_a?(Proc)
429
+ begin
430
+ # JSON view of the value, as with JMESPath: keys, and objects like `Time`, become strings
431
+ JSON.parse(JSON.generate(@options[:query].call(data)))
432
+ rescue StandardError => e
433
+ raise Cli::BadArgument, "Error in user-provided ruby lambda code during query: #{e.message}"
434
+ end
435
+ else
436
+ begin
437
+ JMESPath.search(@options[:query], data)
438
+ rescue JMESPath::Errors::Error => e
439
+ raise Cli::BadArgument, "JMESPath: #{e.message}"
440
+ end
441
+ end
442
+ # Lists of objects, and lists of scalar values are displayed as tables
443
+ return Result.auto(value) unless value.is_a?(Array) && !value.all?(Hash) && !value.all? { |i| Result::SCALAR_TYPES.include?(i.class) }
444
+ Aspera.assert(Result::STRUCTURED_FORMATS.include?(@options[:format]), type: Cli::BadArgument) do
445
+ "Result of query is a list of lists or of mixed values: use a JMESPath multi-select hash (e.g. `[].{a: x, b: y}`), or a format among: #{Result::STRUCTURED_FORMATS.join(', ')}"
446
+ end
447
+ Result.new(data: value)
384
448
  end
385
449
 
386
450
  # Displays a list of objects
@@ -391,16 +455,17 @@ module Aspera
391
455
  def display_table(object_array, fields, single: false)
392
456
  Aspera.assert_array_all(object_array, Hash)
393
457
  Aspera.assert_array_all(fields, String)
458
+ # select before the empty check: no item may remain
459
+ object_array = filter_columns_on_select(object_array)
394
460
  if object_array.empty?
395
461
  # no display for csv
396
462
  display_message(:info, TerminalFormatter.special_format('empty')) if @options[:format].eql?(:table)
397
463
  return
398
464
  end
399
- filter_columns_on_select(object_array)
400
465
  format_style = @options[:table_style].symbolize_keys
401
466
  string_list_separator = format_style.delete(:str_lst_sep) || STR_LST_SEP_VERT
402
467
  # convert data to string, and keep only display fields
403
- object_array.each { |i| self.class.replace_specific_for_terminal(i, string_list_separator) }
468
+ object_array = object_array.map { |i| self.class.replace_specific_for_terminal(i, string_list_separator) }
404
469
  # if table has only one element, and only one field, display the value
405
470
  if object_array.length == 1 && fields.length == 1
406
471
  Log.log.debug("single element, field: #{fields.first}")
@@ -423,22 +488,26 @@ module Aspera
423
488
  case @options[:format]
424
489
  when :table
425
490
  format_style[:border] = :unicode_round if Environment.terminal_supports_unicode?
491
+ # fit in the requested maximum width, or in the terminal
492
+ width = format_style.delete(:width) || Environment.terminal_columns
426
493
  if single || @options[:multi_single].eql?(:yes) ||
427
494
  (@options[:multi_single].eql?(:single) && final_table_rows.length.eql?(1))
428
495
  # display multiple objects as multiple transposed tables
429
496
  final_table_rows.each do |row|
430
- display_message(:data, Terminal::Table.new(
497
+ display_message(:data, TableFitter.table(
431
498
  headings: SINGLE_OBJECT_COLUMN_NAMES,
432
499
  rows: fields.zip(row),
433
- style: format_style
500
+ style: format_style,
501
+ width: width
434
502
  ))
435
503
  end
436
504
  else
437
505
  # display the table as single table
438
- display_message(:data, Terminal::Table.new(
506
+ display_message(:data, TableFitter.table(
439
507
  headings: fields,
440
508
  rows: final_table_rows,
441
- style: format_style
509
+ style: format_style,
510
+ width: width
442
511
  ))
443
512
  end
444
513
  when :csv
@@ -1,11 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/cli/version'
4
- require 'aspera/cli/info'
5
- require 'aspera/environment'
6
- require 'aspera/persistency_action_once'
7
- require 'aspera/rest'
8
- require 'aspera/log'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'version'
7
+ require_relative 'info'
8
+ require_relative '../environment'
9
+ require_relative '../persistency_action_once'
10
+ require_relative '../rest'
11
+ require_relative '../log'
9
12
  require 'date'
10
13
 
11
14
  module Aspera