aspera-cli 4.28.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +217 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +122 -26
  10. data/SECURITY.md +84 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1895 -632
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +29 -22
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +95 -67
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +370 -0
  41. data/lib/aspera/ascp/log_locations.rb +163 -0
  42. data/lib/aspera/ascp/management.rb +16 -8
  43. data/lib/aspera/ascp/session_errors.rb +217 -0
  44. data/lib/aspera/assert.rb +4 -0
  45. data/lib/aspera/cli/ascp_actions.rb +12 -19
  46. data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
  47. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  48. data/lib/aspera/cli/bootstrapper.rb +38 -33
  49. data/lib/aspera/cli/command_line.rb +63 -8
  50. data/lib/aspera/cli/command_registry.rb +9 -1
  51. data/lib/aspera/cli/command_spec.rb +47 -7
  52. data/lib/aspera/cli/completion/ascli.bash +19 -5
  53. data/lib/aspera/cli/completion/ascli.fish +13 -2
  54. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  55. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  56. data/lib/aspera/cli/completion_actions.rb +303 -69
  57. data/lib/aspera/cli/context.rb +9 -4
  58. data/lib/aspera/cli/deprecation.rb +5 -2
  59. data/lib/aspera/cli/error.rb +6 -0
  60. data/lib/aspera/cli/extended_value.rb +66 -31
  61. data/lib/aspera/cli/formatter.rb +43 -30
  62. data/lib/aspera/cli/gem_checker.rb +9 -6
  63. data/lib/aspera/cli/hints.rb +22 -10
  64. data/lib/aspera/cli/http.rb +38 -10
  65. data/lib/aspera/cli/info.rb +12 -7
  66. data/lib/aspera/cli/mailer.rb +6 -3
  67. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  68. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  69. data/lib/aspera/cli/mcp/tool.rb +20 -17
  70. data/lib/aspera/cli/option_declarator.rb +10 -4
  71. data/lib/aspera/cli/option_registry.rb +6 -3
  72. data/lib/aspera/cli/option_types.rb +17 -2
  73. data/lib/aspera/cli/option_value.rb +77 -18
  74. data/lib/aspera/cli/options.schema.yaml +641 -80
  75. data/lib/aspera/cli/parser.rb +71 -35
  76. data/lib/aspera/cli/plugins/alee.rb +7 -4
  77. data/lib/aspera/cli/plugins/aoc.rb +248 -37
  78. data/lib/aspera/cli/plugins/ats.rb +10 -9
  79. data/lib/aspera/cli/plugins/base.rb +50 -18
  80. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  81. data/lib/aspera/cli/plugins/config.rb +210 -104
  82. data/lib/aspera/cli/plugins/console.rb +6 -3
  83. data/lib/aspera/cli/plugins/cos.rb +8 -4
  84. data/lib/aspera/cli/plugins/factory.rb +21 -7
  85. data/lib/aspera/cli/plugins/faspex5.rb +31 -26
  86. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  87. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  88. data/lib/aspera/cli/plugins/mcp.rb +42 -22
  89. data/lib/aspera/cli/plugins/node.rb +39 -38
  90. data/lib/aspera/cli/plugins/oauth.rb +23 -6
  91. data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
  92. data/lib/aspera/cli/plugins/preview.rb +72 -45
  93. data/lib/aspera/cli/plugins/server.rb +66 -14
  94. data/lib/aspera/cli/plugins/shares.rb +87 -18
  95. data/lib/aspera/cli/plugins/transferd.rb +228 -0
  96. data/lib/aspera/cli/poll.rb +236 -0
  97. data/lib/aspera/cli/preset_actions.rb +144 -30
  98. data/lib/aspera/cli/preset_manager.rb +113 -35
  99. data/lib/aspera/cli/prompt.rb +157 -9
  100. data/lib/aspera/cli/result.rb +12 -8
  101. data/lib/aspera/cli/runner.rb +89 -48
  102. data/lib/aspera/cli/secret_finder.rb +4 -1
  103. data/lib/aspera/cli/security.rb +89 -9
  104. data/lib/aspera/cli/special_values.rb +4 -1
  105. data/lib/aspera/cli/sync_actions.rb +15 -10
  106. data/lib/aspera/cli/table_fitter.rb +3 -0
  107. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  108. data/lib/aspera/cli/transfer_actions.rb +11 -8
  109. data/lib/aspera/cli/transfer_agent.rb +43 -20
  110. data/lib/aspera/cli/transfer_progress.rb +14 -45
  111. data/lib/aspera/cli/vault_actions.rb +45 -0
  112. data/lib/aspera/cli/vault_manager.rb +188 -42
  113. data/lib/aspera/cli/version.rb +4 -1
  114. data/lib/aspera/cli/web_secret.rb +12 -10
  115. data/lib/aspera/cli/wizard.rb +42 -39
  116. data/lib/aspera/command_line_builder.rb +9 -5
  117. data/lib/aspera/command_line_converter.rb +4 -1
  118. data/lib/aspera/coverage.rb +103 -33
  119. data/lib/aspera/data_repository.rb +4 -1
  120. data/lib/aspera/dot_container.rb +4 -1
  121. data/lib/aspera/environment.rb +23 -12
  122. data/lib/aspera/exec_spec.rb +3 -0
  123. data/lib/aspera/faspex_gw.rb +13 -9
  124. data/lib/aspera/faspex_postproc.rb +15 -12
  125. data/lib/aspera/formatter_interface.rb +4 -1
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +20 -5
  129. data/lib/aspera/inflate.rb +43 -0
  130. data/lib/aspera/json_rpc/client.rb +6 -5
  131. data/lib/aspera/json_rpc/version.rb +3 -0
  132. data/lib/aspera/line_logger.rb +4 -1
  133. data/lib/aspera/link_header.rb +4 -1
  134. data/lib/aspera/log.rb +18 -5
  135. data/lib/aspera/markdown.rb +5 -2
  136. data/lib/aspera/mime.rb +3 -0
  137. data/lib/aspera/nagios.rb +6 -4
  138. data/lib/aspera/node_emulator.rb +15 -13
  139. data/lib/aspera/oauth/client/base.rb +124 -73
  140. data/lib/aspera/oauth/client/boot.rb +7 -4
  141. data/lib/aspera/oauth/client/factory.rb +41 -18
  142. data/lib/aspera/oauth/client/generic.rb +5 -1
  143. data/lib/aspera/oauth/client/json_credentials.rb +7 -3
  144. data/lib/aspera/oauth/client/jwt.rb +6 -3
  145. data/lib/aspera/oauth/client/shares_web.rb +94 -0
  146. data/lib/aspera/oauth/client/url_json.rb +7 -3
  147. data/lib/aspera/oauth/client/web.rb +40 -13
  148. data/lib/aspera/oauth/server/pkce.rb +6 -5
  149. data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +16 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +17 -2
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +3 -0
  165. data/lib/aspera/rest/aspera_errors.rb +4 -1
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +45 -34
  168. data/lib/aspera/rest/error_analyzer.rb +5 -2
  169. data/lib/aspera/rest/list.rb +8 -4
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +8 -5
  172. data/lib/aspera/rest.rb +9 -6
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  175. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  176. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  177. data/lib/aspera/schema/async_tables.yaml +11 -10
  178. data/lib/aspera/schema/documentation.rb +8 -2
  179. data/lib/aspera/schema/reader.rb +7 -0
  180. data/lib/aspera/schema/registry.rb +44 -28
  181. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  182. data/lib/aspera/schema/validator.rb +4 -1
  183. data/lib/aspera/sealed_hash.rb +150 -0
  184. data/lib/aspera/secret/hider.rb +213 -0
  185. data/lib/aspera/secret/reference.rb +150 -0
  186. data/lib/aspera/secret/resolver.rb +212 -0
  187. data/lib/aspera/secret/store/base.rb +153 -0
  188. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  189. data/lib/aspera/secret/store/factory.rb +103 -0
  190. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  191. data/lib/aspera/secret/store/keeper.rb +141 -0
  192. data/lib/aspera/secret/store/macos_security.rb +284 -0
  193. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  194. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  195. data/lib/aspera/secret/store/one_password_cli.rb +112 -0
  196. data/lib/aspera/secret/store/secret_service.rb +125 -0
  197. data/lib/aspera/secret/strength.rb +119 -0
  198. data/lib/aspera/servlet_helpers.rb +5 -2
  199. data/lib/aspera/ssh.rb +10 -7
  200. data/lib/aspera/ssl.rb +5 -2
  201. data/lib/aspera/string_ext.rb +3 -0
  202. data/lib/aspera/sync/args.schema.yaml +3 -0
  203. data/lib/aspera/sync/conf.schema.yaml +9 -4
  204. data/lib/aspera/sync/database.rb +3 -0
  205. data/lib/aspera/sync/operations.rb +17 -15
  206. data/lib/aspera/temp_file_manager.rb +8 -4
  207. data/lib/aspera/timer_limiter.rb +3 -0
  208. data/lib/aspera/transfer/error.rb +4 -1
  209. data/lib/aspera/transfer/faux_file.rb +4 -1
  210. data/lib/aspera/transfer/parameters.rb +17 -14
  211. data/lib/aspera/transfer/result.rb +23 -0
  212. data/lib/aspera/transfer/resumer.rb +6 -4
  213. data/lib/aspera/transfer/spec.rb +9 -22
  214. data/lib/aspera/units/decimal.rb +67 -0
  215. data/lib/aspera/units/duration.rb +109 -0
  216. data/lib/aspera/uri_reader.rb +64 -9
  217. data/lib/aspera/web_auth/index.html.erb +6 -2
  218. data/lib/aspera/web_auth/paste.rb +109 -0
  219. data/lib/aspera/web_auth/receiver.rb +50 -0
  220. data/lib/aspera/web_auth/relay.rb +76 -0
  221. data/lib/aspera/web_auth/reply.html.erb +150 -0
  222. data/lib/aspera/web_auth/server.rb +47 -10
  223. data/lib/aspera/web_page.rb +7 -2
  224. data/lib/aspera/web_server_simple.rb +6 -3
  225. data/lib/aspera/yaml.rb +5 -1
  226. data/llms-install.md +88 -0
  227. data/llms.txt +87 -0
  228. data/skills/ascli/SKILL.md +94 -0
  229. data.tar.gz.sig +0 -0
  230. metadata +77 -20
  231. metadata.gz.sig +0 -0
  232. data/lib/aspera/duration.rb +0 -42
  233. data/lib/aspera/keychain/base.rb +0 -30
  234. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  235. data/lib/aspera/keychain/factory.rb +0 -58
  236. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  237. data/lib/aspera/keychain/macos_security.rb +0 -277
  238. data/lib/aspera/keychain/one_password_api.rb +0 -91
  239. data/lib/aspera/keychain/one_password_base.rb +0 -39
  240. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  241. data/lib/aspera/secret_hider.rb +0 -196
  242. data/lib/aspera/secret_strength.rb +0 -114
@@ -1,9 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
4
- require 'aspera/secret_hider'
5
- require 'aspera/cli/security'
6
- require 'aspera/cli/preset_manager'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+ require_relative '../dot_container'
8
+ require_relative '../hash_ext'
9
+ require_relative '../secret/hider'
10
+ require_relative 'security'
11
+ require_relative 'preset_manager'
12
+ require_relative 'deprecation'
13
+ require_relative 'info'
14
+ require 'json'
7
15
 
8
16
  module Aspera
9
17
  module Cli
@@ -13,6 +21,12 @@ module Aspera
13
21
  GLOBAL_DEFAULT_KEYWORD = 'GLOBAL'
14
22
  # Display columns for preset overview
15
23
  CONF_OVERVIEW_KEYS = %w[preset parameter value].freeze
24
+ # Keys of the criteria for preset lookup
25
+ LOOKUP_KEYS = %w[url username].freeze
26
+ # Last released version without deprecation of preset commands `update`, `initialize` and `unset`
27
+ PRESET_DEPRECATED_LAST = '4.28.0'
28
+ # Default of an argument not given (`omit_absent`), different from `nil` (`@none:`)
29
+ NOT_GIVEN = Object.new.freeze
16
30
 
17
31
  def action_preset_overview(**)
18
32
  # Secrets hidden by key name here: the key becomes the value of column `parameter`
@@ -26,11 +40,10 @@ module Aspera
26
40
  Result::ObjectList.new(result, fields: CONF_OVERVIEW_KEYS)
27
41
  end
28
42
 
29
- def action_preset_lookup(**)
30
- Plugins::BasicAuth.declare_options(options, parse: true)
31
- url = options.get_option(:url, mandatory: true)
32
- user = options.get_option(:username, mandatory: true)
33
- result = presets.lookup_preset(url: url, username: user)
43
+ # @param criteria [Hash] `url` and `username` of the preset to find
44
+ def action_preset_lookup(criteria:, **)
45
+ Aspera.assert((criteria.keys - LOOKUP_KEYS).empty? && (LOOKUP_KEYS - criteria.keys).empty?, type: BadArgument) { "Expected keys: #{LOOKUP_KEYS.join(', ')}" }
46
+ result = presets.lookup_preset(url: criteria['url'], username: criteria['username'])
34
47
  Aspera.assert(!result.nil?, type: Error) { 'no such config found' }
35
48
  Result::SingleObject.new(result)
36
49
  end
@@ -65,6 +78,21 @@ module Aspera
65
78
  # @param preset [Hash] the preset hash (modified in place)
66
79
  # @param preset_name [String] name used as base for the vault label
67
80
  # @param option_name [String] option key to inspect
81
+ # A preset is evaluated when used, maybe at a lower security level (e.g. on the command line): it cannot hold what the current level refuses.
82
+ # @param param_name [String, Symbol] Option name, or dot-path in an option (e.g. `ui.command`)
83
+ # @param param_value [Object] Value to store
84
+ # @raise [BadArgument] If the option, a sub-option (`x-security` in schema), or the value, is refused at current security level
85
+ def check_preset_entry(param_name, param_value)
86
+ path = param_name.to_s.tr('-', '_').split(DotContainer::SEPARATOR)
87
+ option_name = path.first.to_sym
88
+ if options.option_declared?(option_name)
89
+ # Value of the option, from the sub-option given by dot-path
90
+ option_value = path.drop(1).reverse.inject(param_value) { |value, key| {key => value} }
91
+ options.option_def(option_name).check_security(option_value)
92
+ end
93
+ Security.instance.check_stored_value(param_value, "preset option #{option_name}")
94
+ end
95
+
68
96
  def secure_preset_option(preset, preset_name, option_name)
69
97
  return unless SECRET_KEYWORDS.any? { |kw| option_name.end_with?(kw) }
70
98
  # Vault password is not saved in vault
@@ -98,15 +126,8 @@ module Aspera
98
126
 
99
127
  def action_preset_delete(name:, **)
100
128
  name = preset_name(name)
101
- cp = presets.config_presets
102
- raise Cli::BadArgument, "no such preset: #{name}" unless cp.key?(name)
103
- cp.delete(name)
104
- # Remove default declarations referring to the deleted preset
105
- defaults = cp[PresetManager::Key::DEFAULTS]
106
- defaults&.select { |_, preset_name| preset_name.eql?(name) }&.each_key do |plugin|
107
- defaults.delete(plugin)
108
- Log.log.info { "Removed: #{PresetManager::Key::DEFAULTS}: #{plugin}" }
109
- end
129
+ raise Cli::BadArgument, "no such preset: #{name}" unless presets.config_presets.key?(name)
130
+ delete_preset(name)
110
131
  Result::Status.new("Deleted: #{name}")
111
132
  end
112
133
 
@@ -120,13 +141,14 @@ module Aspera
120
141
  when Numeric, String
121
142
  value = ExtendedValue.instance.evaluate(value.to_s, context: 'preset')
122
143
  # A secret has no key name in a text result: declared, so that it is hidden
123
- SecretHider.instance.add_secret_values(value) if SecretHider.instance.secret?(param_name, value)
144
+ Secret::Hider.instance.add_secret_values(value) if Secret::Hider.instance.secret?(param_name, value)
124
145
  return Result::Text.new(value)
125
146
  end
126
147
  Result::SingleObject.new(value)
127
148
  end
128
149
 
129
150
  def action_preset_unset(name:, param_name:, **)
151
+ Log.log.warn { "Command preset unset is #{Deprecation.new(last: PRESET_DEPRECATED_LAST, message: "use: #{Info::CMD_NAME} config preset set #{shell_word(name)} #{shell_word(param_name)} @none:")}" }
130
152
  name = preset_name(name)
131
153
  cp = presets.config_presets
132
154
  raise Cli::BadArgument, "no such preset: #{name}" unless cp.key?(name)
@@ -134,35 +156,55 @@ module Aspera
134
156
  Result::Status.new("Removed: #{name}: #{param_name}")
135
157
  end
136
158
 
137
- def action_preset_set(name:, param_name:, param_value:, **)
159
+ # Merge into presets (JSON Merge Patch: a value `nil` removes the key), with one of:
160
+ # - `name` is a Hash: presets by name (a preset `nil` is deleted)
161
+ # - `name` and `param_name` is a Hash: parameters of the preset
162
+ # - `name`, `param_name` and `param_value`: one parameter (dot-path), asked if not given
163
+ # @param name [String, Hash] Preset name, or presets
164
+ # @param param_name [String, Hash] Parameter name (dot-path), or parameters, not given for presets
165
+ # @param param_value [Object] Value of the parameter, `nil` removes it
166
+ def action_preset_set(name:, param_name: NOT_GIVEN, param_value: NOT_GIVEN, **)
167
+ if name.is_a?(Hash)
168
+ raise Cli::BadArgument, 'set <presets>: no other argument expected' unless param_name.equal?(NOT_GIVEN)
169
+ name.each { |preset, values| merge_preset(preset, values) }
170
+ return Result::Status.new("Updated: #{name.keys.join(', ')}")
171
+ end
172
+ raise Cli::BadArgument, 'set <name>: expecting <Hash> of parameters, or <param> <value>' if param_name.equal?(NOT_GIVEN)
173
+ if param_name.is_a?(Hash)
174
+ raise Cli::BadArgument, 'set <name> <Hash>: no other argument expected' unless param_value.equal?(NOT_GIVEN)
175
+ merge_preset(name, param_name)
176
+ return Result::Status.new("Updated: #{preset_name(name)}")
177
+ end
178
+ param_value = options.get_next_argument('param_value', validation: nil, evaluate: :smart) if param_value.equal?(NOT_GIVEN)
138
179
  name = preset_name(name, create: true)
139
180
  # Key names are not secrets: converted, or split with dot-notation, a secret would not be hidden in results
140
181
  Security.instance.check_secret_processing('preset parameter name', hint: 'give a name that is not a secret') do
141
- SecretHider.instance.contains_secret?(param_name)
182
+ Secret::Hider.instance.contains_secret?(param_name)
142
183
  end
143
184
  param_name = Parser.option_line_to_name(param_name)
144
- presets.set_key(name, param_name, Parser.smart_convert(param_value))
185
+ check_preset_entry(param_name, param_value) unless param_value.nil?
186
+ presets.set_key(name, param_name, param_value)
145
187
  secure_preset_option(presets.config_presets[name], name, param_name)
146
188
  Result::Nothing.new
147
189
  end
148
190
 
149
191
  def action_preset_initialize(name:, preset:, **)
192
+ Log.log.warn { "Command preset initialize is #{Deprecation.new(last: PRESET_DEPRECATED_LAST, message: "use: #{Info::CMD_NAME} config preset delete #{shell_word(name)}, then: #{Info::CMD_NAME} config preset set #{shell_word(name)} <Hash>")}" }
150
193
  name = preset_name(name, create: true)
151
194
  cp = presets.config_presets
152
195
  Log.log.warn { "configuration already exists: #{name}, overwriting" } if cp.key?(name)
196
+ Aspera.assert_type(preset, Hash, type: Cli::BadArgument) { 'preset' }
197
+ preset.each { |k, v| check_preset_entry(k, v) }
153
198
  cp[name] = preset
154
199
  Result::Status.new("Initialized: #{name}")
155
200
  end
156
201
 
202
+ # Deprecated: options placed after `update` are merged into the preset.
157
203
  def action_preset_update(name:, **)
158
- name = preset_name(name, create: true)
159
- unprocessed_options = options.unprocessed_options_with_value
160
- Log.dump(:opts, unprocessed_options)
161
- cp = presets.config_presets
162
- cp[name] ||= {}
163
- cp[name].merge!(unprocessed_options)
164
- unprocessed_options.each_key { |k| secure_preset_option(cp[name], name, k) }
165
- Result::Status.new("Updated: #{name}")
204
+ values = options.unprocessed_options_with_value(after_command: true)
205
+ Log.log.warn { "Command preset update is #{Deprecation.new(last: PRESET_DEPRECATED_LAST, message: "use: #{set_command_hint(name, values)}")}" }
206
+ merge_preset(name, values)
207
+ Result::Status.new("Updated: #{preset_name(name)}")
166
208
  end
167
209
 
168
210
  def action_preset_ask(name:, option_names:, **)
@@ -179,6 +221,78 @@ module Aspera
179
221
  end
180
222
  Result::Status.new("Updated: #{name}")
181
223
  end
224
+
225
+ private
226
+
227
+ # Merge parameters into a preset (JSON Merge Patch), or delete it.
228
+ # @param name [String] Preset name
229
+ # @param values [Hash, nil] Parameters: option names, a value `nil` removes the parameter. `nil`: delete the preset.
230
+ def merge_preset(name, values)
231
+ raise Cli::BadArgument, "section #{PresetManager::Key::CONFIG} is managed by #{Info::CMD_NAME}" if name.to_s.eql?(PresetManager::Key::CONFIG)
232
+ name = preset_name(name.to_s, create: !values.nil?)
233
+ return delete_preset(name) if values.nil?
234
+ Aspera.assert_type(values, Hash, type: Cli::BadArgument) { "parameters of preset #{name}" }
235
+ values = values.transform_keys { |k| Parser.option_line_to_name(k.to_s) }
236
+ Log.dump(:values, values)
237
+ values.each { |k, v| check_preset_entry(k, v) unless v.nil? }
238
+ cp = presets.config_presets
239
+ cp[name] ||= {}
240
+ Aspera.assert_type(cp[name], Hash, type: Cli::BadArgument) { "preset #{name}" }
241
+ merge_patch!(cp[name], PresetManager.stringify_secrets(nil, values))
242
+ values.each { |k, v| secure_preset_option(cp[name], name, k) unless v.nil? }
243
+ nil
244
+ end
245
+
246
+ # Merge `patch` into `target` (RFC 7396 JSON Merge Patch): a value `nil` removes the key, a `Hash` is merged.
247
+ # @param target [Hash] Modified in place
248
+ # @param patch [Hash]
249
+ # @return [Hash] target
250
+ def merge_patch!(target, patch)
251
+ patch.each do |key, value|
252
+ if value.nil? then target.delete(key)
253
+ elsif value.is_a?(Hash) then target[key] = merge_patch!(target[key].is_a?(Hash) ? target[key] : {}, value)
254
+ else target[key] = value
255
+ end
256
+ end
257
+ target
258
+ end
259
+
260
+ # Delete a preset, and default declarations referring to it.
261
+ # @param name [String] Preset name
262
+ def delete_preset(name)
263
+ cp = presets.config_presets
264
+ cp.delete(name)
265
+ defaults = cp[PresetManager::Key::DEFAULTS]
266
+ defaults&.select { |_, preset_name| preset_name.eql?(name) }&.each_key do |plugin|
267
+ defaults.delete(plugin)
268
+ Log.log.info { "Removed: #{PresetManager::Key::DEFAULTS}: #{plugin}" }
269
+ end
270
+ nil
271
+ end
272
+
273
+ # Command to use instead of the deprecated `preset update`, without secret values.
274
+ # Secret parameters are named after the command: in logs, the value of a secret parameter is hidden.
275
+ # @param name [String] Preset name
276
+ # @param values [Hash] Values taken from options
277
+ # @return [String] e.g. `ascli config preset set p1 @: url=https://example.com username=bob, and secret parameters: password`
278
+ def set_command_hint(name, values)
279
+ secrets, others = DotContainer.new(values).to_dotted.partition do |path, value|
280
+ Secret::Hider.instance.secret?(path.to_s.split(DotContainer::SEPARATOR).last, value)
281
+ end
282
+ assignments = others.map do |path, value|
283
+ value = "@json:#{JSON.generate(value)}" if value.is_a?(Array) || value.is_a?(Hash)
284
+ shell_word("#{path}=#{value}")
285
+ end
286
+ hint = [Info::CMD_NAME, 'config', 'preset', 'set', shell_word(name), '@:', *assignments].join(' ')
287
+ hint = "#{hint}, and secret parameters (e.g. with @secret: or @val:@vault:): #{secrets.map(&:first).join(', ')}" unless secrets.empty?
288
+ hint
289
+ end
290
+
291
+ # @param word [String] Word of a command line
292
+ # @return [String] Word, quoted for a shell only if needed
293
+ def shell_word(word)
294
+ word.match?(%r{\A[\w@:/.,=+%-]+\z}) ? word : "'#{word.gsub("'") { %q('\\'') }}'"
295
+ end
182
296
  end
183
297
  end
184
298
  end
@@ -1,16 +1,22 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/cli/extended_value'
4
- require 'aspera/dot_container'
5
- require 'aspera/cli/version'
6
- require 'aspera/cli/info'
7
- require 'aspera/log'
8
- require 'aspera/assert'
9
- require 'aspera/secret_hider'
10
- require 'aspera/yaml'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'extended_value'
7
+ require_relative 'vault_manager'
8
+ require_relative '../dot_container'
9
+ require_relative 'version'
10
+ require_relative 'info'
11
+ require_relative '../environment'
12
+ require_relative '../log'
13
+ require_relative '../assert'
14
+ require_relative '../secret/hider'
15
+ require_relative '../yaml'
16
+ require_relative '../rainbow'
11
17
  require 'digest'
18
+ require 'fileutils'
12
19
  require 'yaml'
13
- require 'aspera/rainbow'
14
20
  using Rainbow
15
21
 
16
22
  module Aspera
@@ -21,23 +27,32 @@ module Aspera
21
27
  class PresetManager
22
28
  # Reserved keys in the YAML config file
23
29
  module Key
30
+ # Section of the config file itself (holds its version)
24
31
  CONFIG = 'config'
32
+ # Key of the version of the config file, in section CONFIG
25
33
  VERSION = 'version'
34
+ # Section that maps a plugin name to its default preset name
26
35
  DEFAULTS = 'default'
36
+ # Name of the global default preset, when none is declared
27
37
  GLOBAL = 'global_common_defaults'
28
38
  # Prefix for comment/meta keys ignored during preset parsing (e.g. _comment)
29
39
  META_PREFIX = '_'
40
+ # Key of the free text describing a preset (ignored when the preset is applied)
41
+ COMMENT = "#{META_PREFIX}comment".freeze
30
42
  end
31
43
 
44
+ # Preset name given on command line to designate the global default preset
32
45
  GLOBAL_DEFAULT_KEYWORD = 'GLOBAL'
46
+ # Plugin name under which the global default preset is declared in section DEFAULTS
33
47
  CONF_GLOBAL_SYM = :config
34
48
  # Separator for dot-notation digging into presets
35
49
  PRESET_DIG_SEPARATOR = '.'
36
50
 
37
51
  private_constant :PRESET_DIG_SEPARATOR
38
52
 
39
- # @param config_file [String] absolute path to the YAML config file
40
- # @param use_plugin_defaults [Boolean] if false, skip default preset lookup
53
+ # Load the config file (created in memory if it does not exist).
54
+ # @param config_file [String] Absolute path to the YAML config file
55
+ # @param use_plugin_defaults [Boolean] `false`: plugin default presets are not applied (see #plugin_default_name)
41
56
  def initialize(config_file:, use_plugin_defaults: true)
42
57
  @config_file = config_file
43
58
  @use_plugin_defaults = use_plugin_defaults
@@ -53,15 +68,19 @@ module Aspera
53
68
  # File I/O
54
69
  # ------------------------------------------------------------------
55
70
 
56
- # @return [String] SHA1 of current in-memory presets
71
+ # Checksum of the presets in memory, compared to the one of the file to save only when changed.
72
+ # @return [String] SHA256 hex digest of the in-memory presets
57
73
  def checksum
58
- Digest::SHA1.hexdigest(JSON.generate(@config_presets))
74
+ Digest::SHA256.hexdigest(JSON.generate(@config_presets))
59
75
  end
60
76
 
61
77
  # Read and validate the YAML config file.
62
- # Sets @config_presets and @checksum_on_disk.
78
+ # Sets @config_presets and @checksum_on_disk, and declares the secrets found in presets to the secret hider.
63
79
  # A file that cannot be read (YAML error, duplicate keys, access) is left untouched: it shall be fixed.
64
80
  # A file with an unexpected structure is renamed for manual conversion, and a new one is created next time.
81
+ # @return [void]
82
+ # @raise [Psych::SyntaxError] YAML syntax error in config file
83
+ # @raise [Cli::Error] Config file cannot be read, or has an unexpected structure
65
84
  def read_config_file
66
85
  Log.log.debug { "config file is: #{@config_file}".red }
67
86
  if File.exist?(@config_file)
@@ -94,7 +113,7 @@ module Aspera
94
113
  raise Cli::Error, e.to_s
95
114
  end
96
115
  # Hidden wherever they appear (logs, results), also when copied under another key name
97
- SecretHider.instance.add_secrets_in(@config_presets)
116
+ Secret::Hider.instance.add_secrets_in(@config_presets)
98
117
  end
99
118
 
100
119
  # Save to disk only if content changed since last load/save.
@@ -115,7 +134,10 @@ module Aspera
115
134
  # Preset read/write helpers
116
135
  # ------------------------------------------------------------------
117
136
 
118
- # @return [String, nil] name of the default preset for a plugin, or nil
137
+ # Name of the default preset declared for a plugin in the `default` section.
138
+ # @param plugin_name_sym [Symbol] Plugin name (e.g. `:aoc`), or CONF_GLOBAL_SYM for the global default
139
+ # @return [String, nil] Name of the default preset, or `nil` if none, or if plugin defaults are disabled
140
+ # @raise [Cli::Error] The default preset is declared but does not exist, or is not a Hash
119
141
  def plugin_default_name(plugin_name_sym)
120
142
  Aspera.assert(!@config_presets.nil?, 'config_presets shall be defined')
121
143
  return unless @use_plugin_defaults
@@ -127,8 +149,8 @@ module Aspera
127
149
  Log.log.error do
128
150
  "Default config name [#{default_name}] specified for plugin [#{plugin_name_sym}], but it does not exist in config file.\n" \
129
151
  "Please fix: either create preset:\n" \
130
- "#{Info::CMD_NAME} config preset initialize #{default_name} @json:'{}'\n" \
131
- "or remove default:\n#{Info::CMD_NAME} config preset unset #{Key::DEFAULTS} #{plugin_name_sym}"
152
+ "#{Info::CMD_NAME} config preset set #{default_name} @: url=...\n" \
153
+ "or remove default:\n#{Info::CMD_NAME} config preset set #{Key::DEFAULTS} #{plugin_name_sym} @none:"
132
154
  end
133
155
  raise Cli::Error, "No such preset: #{default_name}"
134
156
  end
@@ -138,8 +160,11 @@ module Aspera
138
160
 
139
161
  # Resolve and return a deep-cloned, extended copy of a named preset.
140
162
  # Supports dot-notation (e.g. "outer.inner").
141
- # @param config_name [String]
142
- # @param include_path [Array] guard against include loops
163
+ # Meta keys (prefix `_`) are removed, extended values are evaluated, and a secret read alone is declared to the secret hider.
164
+ # @param config_name [String] Preset name, optionally followed by dot-separated sub keys
165
+ # @param include_path [Array<String>] Names already being resolved, to detect include loops
166
+ # @return [Hash, String, Object] Copy of the preset, or of the selected sub value
167
+ # @raise [Cli::Error] Unknown preset or sub key, or include loop
143
168
  def by_name(config_name, include_path = [])
144
169
  Aspera.assert(!include_path.include?(config_name), type: Cli::Error) { 'loop in include' }
145
170
  include_path = include_path.clone
@@ -158,13 +183,18 @@ module Aspera
158
183
  current.delete_if { |k, _| k.to_s.start_with?(Key::META_PREFIX) } if current.is_a?(Hash)
159
184
  current = ExtendedValue.instance.evaluate(current, context: 'preset')
160
185
  # A secret read alone has no key name: declared, so that it is hidden wherever it appears
161
- SecretHider.instance.add_secret_values(current) if SecretHider.instance.secret?(names.last, current)
186
+ Secret::Hider.instance.add_secret_values(current) if Secret::Hider.instance.secret?(names.last, current)
162
187
  current
163
188
  end
164
189
 
165
190
  # Set a single key in a preset hash (creates the preset if absent).
166
191
  # If param_name contains dots, it is treated as a dot-path and deep-merged into the preset.
167
192
  # If param_value is nil, the key is deleted instead.
193
+ # Numeric secrets are converted to String (see .stringify_secrets).
194
+ # @param preset [String] Name of preset
195
+ # @param param_name [String, Symbol] Key name, or dot-path of the key in the preset
196
+ # @param param_value [Object, nil] Value to set, or `nil` to delete the key
197
+ # @return [nil]
168
198
  def set_key(preset, param_name, param_value)
169
199
  Aspera.assert_type(param_name, String, Symbol) { 'parameter' }
170
200
  param_name = param_name.to_s
@@ -208,6 +238,7 @@ module Aspera
208
238
  nil
209
239
  end
210
240
 
241
+ # Name of the global default preset: declared in `default` under `config`, else `global_common_defaults`.
211
242
  # @param create [Boolean] `true`: declare and create the global default preset if needed (to write in it)
212
243
  # @return [String] name of the global default preset
213
244
  def global_default_preset(create: true)
@@ -223,33 +254,46 @@ module Aspera
223
254
  result
224
255
  end
225
256
 
257
+ # @param preset_name [String] Name of preset
226
258
  # @return [Boolean] true if `preset_name` is declared as the global default preset (does not create it)
227
259
  def global_preset?(preset_name)
228
260
  defaults = @config_presets[Key::DEFAULTS]
229
261
  defaults.is_a?(Hash) && defaults[CONF_GLOBAL_SYM.to_s].eql?(preset_name)
230
262
  end
231
263
 
232
- # Set param in the global defaults preset
264
+ # Set a parameter in the global default preset (created if needed).
265
+ # @param key [String, Symbol] Key name, or dot-path of the key
266
+ # @param value [Object, nil] Value to set, or `nil` to delete the key
267
+ # @return [nil]
233
268
  def set_global_default(key, value)
234
269
  set_key(global_default_preset, key, value)
235
270
  end
236
271
 
237
- # Create / overwrite a preset and optionally set it as plugin default.
238
- def defaults_set(plugin_name, preset_name, preset_values, option_default, option_override)
272
+ # Create or overwrite a preset, and optionally set it as default for the plugin (used by the wizard).
273
+ # @param plugin_name_sym [Symbol] Plugin name (e.g. `:aoc`)
274
+ # @param preset_name [String] Name of preset to create
275
+ # @param preset_values [Hash] Content of preset
276
+ # @param option_default [Boolean] `true`: set the preset as default for the plugin
277
+ # @param option_override [Boolean] `true`: replace an existing preset or plugin default
278
+ # @return [Hash] preset_values
279
+ # @raise [Cli::Error] The preset, or the plugin default, already exists, and option_override is `false`
280
+ def defaults_set(plugin_name_sym, preset_name, preset_values, option_default, option_override)
239
281
  @config_presets[Key::DEFAULTS] ||= {}
240
- raise Cli::Error, "A default configuration already exists for plugin '#{plugin_name}' (use --override=yes or --default=no)" \
241
- if !option_override && option_default && @config_presets[Key::DEFAULTS].key?(plugin_name)
242
- raise Cli::Error, "Preset already exists: #{preset_name} (use --override=yes or provide alternate name on command line)" \
282
+ raise Cli::Error, "A default configuration already exists for plugin '#{plugin_name_sym}' (use wizard key override=yes or default=no)" \
283
+ if !option_override && option_default && @config_presets[Key::DEFAULTS].key?(plugin_name_sym.to_s)
284
+ raise Cli::Error, "Preset already exists: #{preset_name} (use wizard key override=yes or provide alternate preset name)" \
243
285
  if !option_override && @config_presets.key?(preset_name)
244
286
  if option_default
245
- Log.log.info("Setting config preset as default for #{plugin_name}")
246
- @config_presets[Key::DEFAULTS][plugin_name.to_s] = preset_name
287
+ Log.log.info("Setting config preset as default for #{plugin_name_sym}")
288
+ @config_presets[Key::DEFAULTS][plugin_name_sym.to_s] = preset_name
247
289
  end
248
290
  @config_presets[preset_name] = preset_values
249
291
  end
250
292
 
251
- # Find the first preset whose url+username match
252
- # @return [Hash, nil]
293
+ # Find the first preset whose url and username match (URLs compared without trailing `/` and default port 443).
294
+ # @param url [String] URL of server
295
+ # @param username [String] User name
296
+ # @return [Hash, nil] Copy of the preset, or `nil` if none matches
253
297
  def lookup_preset(url:, username:)
254
298
  url = canonical_url(url)
255
299
  Log.log.debug { "Lookup preset for #{username}@#{url}" }
@@ -266,11 +310,14 @@ module Aspera
266
310
  # ------------------------------------------------------------------
267
311
 
268
312
  class << self
313
+ # Deep copy of a value (containers included).
314
+ # @param val [Object] Value to copy
315
+ # @return [Object] Copy of val
269
316
  def deep_clone(val)
270
317
  Marshal.load(Marshal.dump(val))
271
318
  end
272
319
 
273
- # Recursively merge src into dst in place (dst wins on scalar conflicts).
320
+ # Recursively merge src into dst in place (src wins on scalar conflicts).
274
321
  # @param dst [Hash] destination (modified in place)
275
322
  # @param src [Hash] source
276
323
  # @return [Hash] dst
@@ -294,7 +341,7 @@ module Aspera
294
341
  case value
295
342
  when Hash then value.each { |k, v| value[k] = stringify_secrets(k, v) }
296
343
  when Array then value.map! { |v| stringify_secrets(nil, v) }
297
- when Numeric then value = value.to_s if SecretHider.instance.secret?(key, value.to_s)
344
+ when Numeric then value = value.to_s if Secret::Hider.instance.secret?(key, value.to_s)
298
345
  end
299
346
  value
300
347
  end
@@ -302,12 +349,16 @@ module Aspera
302
349
 
303
350
  private
304
351
 
305
- # Normalize a URL for comparison
352
+ # Normalize a URL for comparison: no trailing `/`, no default HTTPS port.
353
+ # @param url [String] URL
354
+ # @return [String] Normalized URL
306
355
  def canonical_url(url)
307
356
  url.chomp('/').sub(%r{^(https://[^/]+):443$}, '\1')
308
357
  end
309
358
 
310
- # Structural validation + compatibility fixes on @config_presets
359
+ # Structural validation and compatibility fixes on @config_presets, stamped with the current version.
360
+ # @return [void]
361
+ # @raise [Cli::Error, StandardError] Unexpected structure
311
362
  def validate_config_presets!
312
363
  Log.dump(:available_presets, @config_presets, level: :trace1)
313
364
  Aspera.assert_type(@config_presets, Hash) { 'config file YAML' }
@@ -317,9 +368,36 @@ module Aspera
317
368
  Log.log.debug { "conf version: #{version}" }
318
369
  # Fix bug in 4.4 (creating key "true" in "default" preset)
319
370
  @config_presets[Key::DEFAULTS].delete(true) if @config_presets[Key::DEFAULTS].is_a?(Hash)
371
+ # Migrate deprecated vault format in presets if present
372
+ migrate_vault_configs!
320
373
  # Stamp with current version
321
374
  @config_presets[Key::CONFIG][Key::VERSION] = Cli::VERSION
322
375
  end
376
+
377
+ # Migrates deprecated vault options in presets to the new format (stores and default)
378
+ # and saves a backup of the original config file.
379
+ # @return [void]
380
+ def migrate_vault_configs!
381
+ migrated = false
382
+ @config_presets.each_value do |preset|
383
+ next unless preset.is_a?(Hash)
384
+ vault_config = preset['vault'] || preset[:vault]
385
+ next unless vault_config.is_a?(Hash)
386
+ converted = VaultManager.convert_option(vault_config)
387
+ if converted != vault_config
388
+ preset['vault'] = converted
389
+ preset.delete(:vault) if preset.key?(:vault)
390
+ migrated = true
391
+ end
392
+ end
393
+ return unless migrated && File.exist?(@config_file)
394
+ backup_file = "#{@config_file}.pre#{Cli::VERSION}.bak"
395
+ FileUtils.cp(@config_file, backup_file)
396
+ Environment.restrict_file_access(backup_file)
397
+ Log.log.warn { "Migrated vault option in #{@config_file} (backup saved to #{backup_file})" }
398
+ Environment.write_file_restricted(@config_file, force: true) { @config_presets.to_yaml }
399
+ @checksum_on_disk = checksum
400
+ end
323
401
  end
324
402
  end
325
403
  end