aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -0,0 +1,201 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'logger'
7
+ require 'set'
8
+ require 'singleton'
9
+ require 'json'
10
+ require 'uri'
11
+
12
+ module Aspera
13
+ module Secret
14
+ # remove secret from logs and output
15
+ #
16
+ # Secrets are identified:
17
+ # - by key name, in structures (`secret?`, `hidden_copy`) and in strings with known patterns (`hide_secrets_in_string`)
18
+ # - by value, when declared where they come from (`add_secret_values`, `add_secrets_in`): option values, tokens, ...
19
+ #
20
+ # @!method self.instance
21
+ # Returns the singleton instance of Secret::Hider
22
+ # @return [Secret::Hider] the singleton instance
23
+ class Hider
24
+ include Singleton
25
+
26
+ # display string for hidden secrets (see `Environment::UNICODE_TO_ASCII`)
27
+ HIDDEN_PASSWORD = '🔑'
28
+ # env vars for ascp with secrets
29
+ ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
30
+ # keys in hash that contain secrets
31
+ KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
32
+ HTTP_SECRETS = %w[Authorization].freeze
33
+ ALL_SECRETS = (ASCP_ENV_SECRETS + KEY_SECRETS + HTTP_SECRETS).freeze
34
+ NON_ENV_SECRETS = (KEY_SECRETS + HTTP_SECRETS).freeze
35
+ # not secrets: `*_file` is the path of a file (e.g. `ca_file`)
36
+ KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/, /public_key$/, /^token_type$/, /_file$/].freeze
37
+ # min length of hidden secrets, use `+` or `{n,}`
38
+ SECRET_LENGTH = '{5,}'
39
+ # min length of a declared secret value to be hidden inside a string: shorter ones are hidden only when they are the whole string,
40
+ # else common words would be hidden (e.g. password `aspera` in URLs), or optionally as a word (`short_in_text`)
41
+ VALUE_IN_STRING_LENGTH = 8
42
+ # regex that define named captures :begin and :end
43
+ REGEX_LOG_REPLACES = [
44
+ # private key values (place first)
45
+ /(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)\n*/,
46
+ # CLI manager get/set options
47
+ /(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/i,
48
+ # env var ascp exec
49
+ /(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)[^ \n]+(?<end> |$)/,
50
+ # rendered JSON or Ruby (quoted key or legacy symbol key: `"k":"v"`, `"k" => "v"`, `:k=>"v"`)
51
+ /(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>? *(?:=>|:) *")(?:[^"\\]|\\.)+(?<end>")/i,
52
+ # rendered Ruby 3.4+ symbol key: `k: "v"`
53
+ /(?<begin>\b\w*(?:#{ALL_SECRETS.join('|')})\w*: *")(?:[^"\\]|\\.)+(?<end>")/i,
54
+ # rendered Ruby Struct or object: `k="v"`, `@k="v"`
55
+ /(?<begin>@?\b\w*(?:#{ALL_SECRETS.join('|')})\w*=")(?:[^"\\]|\\.)+(?<end>")/i,
56
+ # query parameter of URL: `?token=v` or `&client_secret=v`
57
+ /(?<begin>[?&][^=&\s"]*(?:#{KEY_SECRETS.join('|')})[^=&\s"]*=)[^&\s"#]+(?<end>)/i,
58
+ # logged data
59
+ /(?<begin>(?:#{NON_ENV_SECRETS.join('|')})[ =:]+)[^ "\n]#{SECRET_LENGTH}(?<end>$)/i,
60
+ # cred in http dump
61
+ /(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
62
+ ].freeze
63
+ private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :SECRET_LENGTH, :VALUE_IN_STRING_LENGTH, :REGEX_LOG_REPLACES
64
+ attr_accessor :log_secrets
65
+
66
+ # @return [Proc] new log formatter that hides secrets
67
+ def log_formatter(original_formatter)
68
+ original_formatter ||= Logger::Formatter.new
69
+ # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
70
+ # Hiding is applied on the formatted line, so that non-String messages (e.g. Exception) are covered too
71
+ return lambda do |severity, date_time, program_name, msg|
72
+ line = original_formatter.call(severity, date_time, program_name, msg)
73
+ line = hide_secrets_in_string(line, all: true) if line.is_a?(String) && !@log_secrets
74
+ line
75
+ end
76
+ end
77
+
78
+ # Replace secrets in a string with the hidden password placeholder
79
+ # @param value [String] Input string possibly containing secrets
80
+ # @param all [Boolean] `false`: only declared values and private keys, `true`: also all known secret patterns
81
+ # @return [String] String with secrets replaced by placeholder
82
+ def hide_secrets_in_string(value, all: false)
83
+ marker = hidden_marker
84
+ value = hide_values_in_string(value, marker)
85
+ (all ? REGEX_LOG_REPLACES : REGEX_LOG_REPLACES.first(1)).each do |reg_ex|
86
+ value = value.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{marker}#{Regexp.last_match(:end)}" }
87
+ end
88
+ return value
89
+ end
90
+
91
+ # @param keyword [String, Symbol] Key name to check
92
+ # @param value [String] Associated value (must be a String to be a secret)
93
+ # @return [Boolean] true if the key denotes a secret
94
+ def secret?(keyword, value)
95
+ keyword = keyword.to_s if keyword.is_a?(Symbol)
96
+ # only Strings can be secrets, not booleans, or hash, arrays
97
+ return false unless keyword.is_a?(String) && value.is_a?(String)
98
+ return true if @additional_keys.include?(keyword)
99
+ keyword = keyword.downcase
100
+ # those are not secrets
101
+ return false if KEY_FALSE_POSITIVES.any? { |f| f.match?(keyword) }
102
+ # check if keyword (name) contains an element that designate it as a secret
103
+ ALL_SECRETS.any? { |kw| keyword.include?(kw.downcase) }
104
+ end
105
+
106
+ # Hides secrets in a structure: by key name, and declared values, also in key names (e.g. a secret given as name of a parameter).
107
+ # Containers are copied, not modified: the structure may be shared (e.g. configuration, transfer spec).
108
+ # @param obj [Object] Hash, Array, Struct, String, or other value (returned as is)
109
+ # @param short_in_text [Boolean] `true`: a declared value shorter than 8 characters is also hidden inside a text, as a word
110
+ # (not next to a letter or digit), e.g. a short password in an error message of `mcp server`
111
+ # @return [Object] Copy of obj with secrets replaced by placeholder
112
+ def hidden_copy(obj, short_in_text: false)
113
+ case obj
114
+ when Hash
115
+ obj.to_h do |k, v|
116
+ [k.is_a?(String) ? hidden_copy(k, short_in_text: short_in_text) : k, secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text)]
117
+ end
118
+ when Array then obj.map { |i| hidden_copy(i, short_in_text: short_in_text) }
119
+ when Struct then obj.dup.tap { |s| s.each_pair { |k, v| s[k] = secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text) } }
120
+ when String then hide_values_in_string(obj, hidden_marker, short_in_text: short_in_text)
121
+ else obj
122
+ end
123
+ end
124
+
125
+ # @param obj [Object] Hash, Array, Struct, String, or other value
126
+ # @return [Boolean] `true` if obj contains a secret: by key name, or declared value
127
+ def contains_secret?(obj)
128
+ !hidden_copy(obj).eql?(obj)
129
+ end
130
+
131
+ # Declare secret values: hidden wherever they appear (logs, results), also when escaped in JSON, Ruby or URL
132
+ # @param values [Array<Object>] Secret values: only non-empty Strings are retained
133
+ def add_secret_values(*values)
134
+ forms = values.select { |v| v.is_a?(String) && !v.empty? }.flat_map do |v|
135
+ [v, (JSON.generate(v)[1..-2] rescue nil), v.inspect[1..-2], URI.encode_www_form_component(v)]
136
+ end.compact
137
+ return if forms.empty?
138
+ @mutex.synchronize do
139
+ values = @values | forms
140
+ next if values.size.eql?(@values.size)
141
+ long, short = values.partition { |v| v.length >= VALUE_IN_STRING_LENGTH }
142
+ # Longest first, so that a secret is not partly hidden by another one that it contains
143
+ @values_regex = long.empty? ? nil : Regexp.union(long.sort_by { |v| -v.length })
144
+ @short_values_regex = short.empty? ? nil : /(?<![[:alnum:]])#{Regexp.union(short.sort_by { |v| -v.length })}(?![[:alnum:]])/
145
+ # Replaced, not modified, for readers in other threads
146
+ @values = values.freeze
147
+ end
148
+ end
149
+
150
+ # Declare the secret values found by key name in a structure
151
+ # @param obj [Object] Hash or Array, e.g. a transfer spec
152
+ def add_secrets_in(obj)
153
+ case obj
154
+ when Array then obj.each { |i| add_secrets_in(i) }
155
+ when Hash then obj.each { |k, v| secret?(k, v) ? add_secret_values(v) : add_secrets_in(v) }
156
+ end
157
+ end
158
+
159
+ # Declare additional exact key names whose values are secrets
160
+ # @param keys [Array<String, Symbol>] Key names
161
+ def add_secret_keys(keys)
162
+ @additional_keys.merge(keys.map(&:to_s))
163
+ end
164
+
165
+ private
166
+
167
+ def initialize
168
+ @log_secrets = false
169
+ @additional_keys = Set.new
170
+ # Declared secret values, and their escaped forms
171
+ @values = Set.new.freeze
172
+ # Matches declared values long enough to be hidden inside a string
173
+ @values_regex = nil
174
+ # Matches shorter declared values as a word
175
+ @short_values_regex = nil
176
+ @mutex = Mutex.new
177
+ end
178
+
179
+ # @param value [String] Input string
180
+ # @param marker [String] Placeholder
181
+ # @param short_in_text [Boolean] `true`: also hide short values as a word
182
+ # @return [String] value with declared secret values replaced by placeholder
183
+ def hide_values_in_string(value, marker, short_in_text: false)
184
+ return marker if @values.include?(value)
185
+ long_regex = @values_regex
186
+ short_regex = @short_values_regex if short_in_text
187
+ # Long values first, as they may contain short ones
188
+ value = value.gsub(long_regex, marker) unless long_regex.nil?
189
+ value = value.gsub(short_regex, marker) unless short_regex.nil?
190
+ value
191
+ end
192
+
193
+ # @return [String] Placeholder for hidden secrets, in ASCII when Unicode is not supported
194
+ def hidden_marker
195
+ # Required here, as `aspera/environment` requires `aspera/log`, which requires this file
196
+ require_relative '../environment'
197
+ Environment.char(HIDDEN_PASSWORD)
198
+ end
199
+ end
200
+ end
201
+ end
@@ -0,0 +1,150 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+ require_relative 'hider'
8
+ require 'uri'
9
+
10
+ module Aspera
11
+ # Reference to a secret held in a store: `secret://<store>/<path>[?<options>][#<field>]`
12
+ #
13
+ # The reference is stored instead of the value, the value is read when used (see `Secret::Resolver`).
14
+ # A reference is a whole value, it is not searched in a text.
15
+ # Path and field are percent-decoded (`%20` or a space), options are a URL query (`version=2&mount=a/b`).
16
+ # In `ascli`: `@vault://<store>/<path>[?<options>][#<field>]`, stores of option `vault`.
17
+ module Secret
18
+ # Error on a reference: the message gives the reference, never a value
19
+ class Error < Aspera::Error
20
+ end
21
+
22
+ # Parsed reference
23
+ # @!attribute text [String] Reference as given
24
+ # @!attribute store [String] Name of store
25
+ # @!attribute path [String] Path of secret in store (decoded)
26
+ # @!attribute options [Hash<String,String>] Options
27
+ # @!attribute field [String, nil] Field of secret (decoded), `nil` for default field of store
28
+ Reference = Data.define(:text, :store, :path, :options, :field)
29
+
30
+ # Bootstrap secret of a store in configuration (e.g. `password` of type `file`):
31
+ # the value, or `@file:<path>` to read it from a file when used (e.g. a systemd credential), without final newline.
32
+ # Syntax of extended values of `ascli`, only `@file:` is supported: other prefixes may be added.
33
+ # In YAML, quote the value: `@` cannot start a plain scalar.
34
+ class Bootstrap
35
+ FILE_PREFIX = '@file:'
36
+
37
+ # @param name [String] Name of parameter, for errors
38
+ # @param text [String] Value, or `@file:<path>`
39
+ # @raise [Error] Not a String, or empty
40
+ def initialize(name, text)
41
+ raise Error, "#{name}: expecting the value, or #{FILE_PREFIX}<path>" unless text.is_a?(String) && !text.empty?
42
+ if text.start_with?(FILE_PREFIX)
43
+ @path = text.delete_prefix(FILE_PREFIX)
44
+ raise Error, "#{name}: expecting #{FILE_PREFIX}<path>" if @path.empty?
45
+ else
46
+ @value = text
47
+ Secret::Hider.instance.add_secret_values(text)
48
+ end
49
+ end
50
+
51
+ # @return [Boolean] `true` if read from a file
52
+ def file?
53
+ !@path.nil?
54
+ end
55
+
56
+ # @return [Time, nil] Modification time of file, `nil` for a value
57
+ # @raise [Error] Missing file
58
+ def mtime
59
+ File.mtime(@path) unless @path.nil?
60
+ rescue SystemCallError => e
61
+ raise Error, "secret file: #{e.message}"
62
+ end
63
+
64
+ # @return [String] Value, read from the file at each call
65
+ # @raise [Error] Missing, unreadable or empty file
66
+ def value
67
+ @path.nil? ? @value : Secret.read_file(@path)
68
+ end
69
+
70
+ # No value in logs and errors
71
+ def inspect
72
+ "#<#{self.class} #{@path.nil? ? 'value' : "#{FILE_PREFIX}#{@path}"}>"
73
+ end
74
+ end
75
+
76
+ SCHEME = 'secret://'
77
+ # Store name, path, options, field
78
+ SYNTAX = %r{\Asecret://([A-Za-z0-9_.-]+)/([^?#]+)(?:\?([^#]*))?(?:#(.+))?\z}
79
+ private_constant :SYNTAX
80
+
81
+ class << self
82
+ # @param value [Object] Any value
83
+ # @return [Boolean] `true` if value is a String starting with `secret://` (then `parse_reference` checks it)
84
+ def reference?(value)
85
+ value.is_a?(String) && value.start_with?(SCHEME)
86
+ end
87
+
88
+ # Build a reference string from its components (inverse of `parse_reference`).
89
+ # Path segments are percent-encoded individually so `/` separators are preserved.
90
+ # Field and option values are fully percent-encoded.
91
+ # @param store [String] Name of store
92
+ # @param path [String] Path of secret in store
93
+ # @param options [Hash<String,String>] Options (e.g. `{'mount' => 'kv/team'}`)
94
+ # @param field [String, nil] Field of secret, `nil` for none
95
+ # @return [String] Reference
96
+ def reference_text(store:, path:, options: {}, field: nil)
97
+ encoded_path = path.split('/', -1).map { |seg| encode(seg) }.join('/')
98
+ text = "#{SCHEME}#{store}/#{encoded_path}"
99
+ text += "?#{URI.encode_www_form(options)}" unless options.empty?
100
+ text += "##{encode(field)}" unless field.nil?
101
+ text
102
+ end
103
+
104
+ # @param text [String] Reference
105
+ # @return [Reference] Parsed reference
106
+ # @raise [Error] Invalid syntax
107
+ def parse_reference(text)
108
+ match = SYNTAX.match(text) if text.is_a?(String)
109
+ raise Error, "secret reference #{text}: expected #{SCHEME}<store>/<path>[?<options>][#<field>]" if match.nil?
110
+ Reference.new(
111
+ text: text,
112
+ store: match[1],
113
+ path: decode(text, match[2]),
114
+ options: match[3].nil? ? {} : URI.decode_www_form(match[3]).to_h.freeze,
115
+ field: match[4].nil? ? nil : decode(text, match[4])
116
+ )
117
+ rescue ArgumentError => e
118
+ raise Error, "secret reference #{text}: #{e.message}"
119
+ end
120
+
121
+ # Read a bootstrap secret from a file (`@file:` of `Bootstrap`, e.g. a systemd credential).
122
+ # The value is declared to `Secret::Hider`.
123
+ # @param path [String] Path of file
124
+ # @return [String] Content of file without trailing newline
125
+ # @raise [Error] Missing, unreadable or empty file
126
+ def read_file(path)
127
+ value = File.read(path).chomp
128
+ raise Error, "empty secret file: #{path}" if value.empty?
129
+ Secret::Hider.instance.add_secret_values(value)
130
+ value
131
+ rescue SystemCallError => e
132
+ raise Error, "secret file: #{e.message}"
133
+ end
134
+
135
+ private
136
+
137
+ # @return [String] Percent-decoded part of reference (`+` is not a space)
138
+ def decode(text, part)
139
+ URI.decode_uri_component(part)
140
+ rescue ArgumentError
141
+ raise Error, "secret reference #{text}: invalid percent-encoding: #{part}"
142
+ end
143
+
144
+ # @return [String] Percent-encoded part of reference (encodes all reserved URI characters including `#`, `?`, `&`)
145
+ def encode(part)
146
+ URI.encode_uri_component(part)
147
+ end
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,212 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'reference'
7
+ require_relative 'store/factory'
8
+ require_relative 'hider'
9
+
10
+ module Aspera
11
+ module Secret
12
+ # Reads the values of references from named stores, with a cache.
13
+ #
14
+ # Configuration (e.g. from a YAML file):
15
+ # - `cache_ttl`: time in seconds a value is kept in memory (default: 60, 0: no cache)
16
+ # - `stores`: name of store → parameters of store, with `type` (see `Store::Factory`)
17
+ # - `writable: true`: allow `write` to this store (default: false)
18
+ #
19
+ # The type and the names of parameters are checked by `new`, a store is created when first used:
20
+ # a store not used (e.g. its secret is not available) is not an error.
21
+ # Bootstrap secrets of stores (`password` of type `file`, `token` and `secret_id` of type `vault`): the value, or `@file:<path>` (`Bootstrap`).
22
+ # Error messages give the reference, never a value.
23
+ class Resolver
24
+ DEFAULT_CACHE_TTL = 60
25
+ CONFIG_KEYS = %w[cache_ttl stores].freeze
26
+ STORE_NAME = /\A[A-Za-z0-9_.-]+\z/
27
+ WRITABLE_KEY = 'writable'
28
+ private_constant :CONFIG_KEYS, :STORE_NAME, :WRITABLE_KEY
29
+
30
+ # @param config [Hash] Configuration, keys String or Symbol
31
+ # @raise [Error] Invalid configuration
32
+ def initialize(config)
33
+ raise Error, 'secret references configuration: expecting a Hash' unless config.is_a?(Hash)
34
+ config = config.transform_keys(&:to_s)
35
+ unsupported = config.keys - CONFIG_KEYS
36
+ raise Error, "secret references configuration: unsupported keys: #{unsupported.join(', ')}" unless unsupported.empty?
37
+ @cache_ttl = config.fetch('cache_ttl', DEFAULT_CACHE_TTL)
38
+ raise Error, 'secret references configuration: cache_ttl: expecting a number >= 0' unless @cache_ttl.is_a?(Numeric) && !@cache_ttl.negative?
39
+ stores = config.fetch('stores', nil) || {}
40
+ raise Error, 'secret references configuration: stores: expecting a Hash' unless stores.is_a?(Hash)
41
+ # Name of store → {params: ..., writable: bool}, checked
42
+ @store_configs = stores.to_h { |name, params| [name.to_s, check_store(name.to_s, params)] }.freeze
43
+ # Name of store → store instance, created when first used
44
+ @stores = {}
45
+ # Key of reference → [value, monotonic time of expiry]
46
+ @cache = {}
47
+ @lock = Mutex.new
48
+ end
49
+
50
+ # @return [Array<String>] Names of stores
51
+ def store_names
52
+ @store_configs.keys
53
+ end
54
+
55
+ # @param name [String] Name of store
56
+ # @return [Boolean] `true` if the store is declared writable in the configuration
57
+ def writable?(name)
58
+ @store_configs.fetch(name) { raise Error, "unknown store #{name}" }[:writable]
59
+ end
60
+
61
+ # @param name [String] Name of store
62
+ # @return [Store::Base] Store, created when first used, e.g. to manage its secrets by label
63
+ # @raise [Error] Unknown store, or invalid parameter
64
+ def store(name)
65
+ config = @store_configs.fetch(name) { raise Error, "unknown store #{name}" }
66
+ @lock.synchronize { @stores[name] ||= create_store(name, config[:params]) }
67
+ end
68
+
69
+ # Checks that a reference is valid and reachable, without reading its value
70
+ # @param text [String] Reference
71
+ # @raise [Error] Invalid reference, not allowed, or not readable
72
+ def check(text)
73
+ ref = Secret.parse_reference(text)
74
+ access(ref) { nil }
75
+ end
76
+
77
+ # @param text [String] Reference
78
+ # @return [String] Value of secret, from cache if not expired
79
+ # @raise [Error] Invalid reference, not allowed, or not readable
80
+ def resolve(text)
81
+ ref = Secret.parse_reference(text)
82
+ access(ref) do |store|
83
+ key = ref.to_h.except(:text)
84
+ cached(key) || cache(key, ref.store, store.read_reference(ref)[:value])
85
+ end
86
+ end
87
+
88
+ # Checks that a secret can be read, without its value
89
+ # @param text [String] Reference
90
+ # @return [Hash] `store`, `version` (`nil` if store has no version), `updated_time` (`Time` or `nil`)
91
+ # @raise [Error] Invalid reference, not allowed, or not readable
92
+ def metadata(text)
93
+ ref = Secret.parse_reference(text)
94
+ access(ref) { |store| {store: ref.store}.merge(store.read_reference(ref).except(:value)) }
95
+ end
96
+
97
+ # Field names of a secret, without values, e.g. to choose the field of a reference
98
+ # @param text [String] Reference, its field is not used
99
+ # @return [Hash] `store`, `fields` (`Array<String>`), `version` (`nil` if store has no version), `updated_time` (`Time` or `nil`)
100
+ # @raise [Error] Invalid reference, not allowed, or not readable
101
+ def fields(text)
102
+ ref = Secret.parse_reference(text)
103
+ access(ref) { |store| {store: ref.store}.merge(store.read_fields(ref)) }
104
+ end
105
+
106
+ # Secrets and folders of a store that references can read, without reading the secrets, e.g. to build references
107
+ # @param name [String] Name of store
108
+ # @param path [String] Folder, empty for a store without folders (labels)
109
+ # @param options [Hash<String,String>] Options of references that select the folder (`mount` of type `vault`)
110
+ # @return [Array<Hash>] `name` (last segment of path, or label), `folder` (`true` for a folder), by name
111
+ # @raise [Error] Unknown store, folder not allowed, listing not supported by the type, or store not readable
112
+ def list(name, path = '', options = {})
113
+ found = store(name)
114
+ found.list_folder(path, options)
115
+ rescue StandardError => e
116
+ raise Error, "secret store #{name}#{", folder #{path}" unless path.empty?}: #{reason(e, found)}"
117
+ end
118
+
119
+ # Write a new secret to a writable store
120
+ # @param text [String] Reference (without field: the field names are given in `fields`)
121
+ # @param fields [Hash<String,String>] Field name → value
122
+ # @return [Hash] `version` (`Integer` or `nil`), `updated_time` (`Time` or `nil`)
123
+ # @raise [Error] Store not writable, reference has a field, unknown store, not allowed, or write failed
124
+ def write(text, fields)
125
+ ref = Secret.parse_reference(text)
126
+ raise Error, "secret reference #{ref.text}: field not allowed when writing (use field names in fields)" unless ref.field.nil?
127
+ raise Error, "secret reference #{ref.text}: store #{ref.store} is not writable" unless writable?(ref.store)
128
+ result = access(ref) { |store| store.write_reference(ref, fields) }
129
+ # Invalidate cache for this reference so it is readable at once
130
+ key = ref.to_h.except(:text)
131
+ @lock.synchronize { @cache.delete(key) }
132
+ result
133
+ end
134
+
135
+ # Remove values from cache, e.g. after a rotation
136
+ # @param store_name [String, nil] Name of store, `nil`: all stores
137
+ def invalidate(store_name = nil)
138
+ @lock.synchronize do
139
+ store_name.nil? ? @cache.clear : @cache.delete_if { |_, entry| entry[:store].eql?(store_name) }
140
+ end
141
+ nil
142
+ end
143
+
144
+ private
145
+
146
+ # @return [Hash] `{params:, writable:}`, parameters checked
147
+ def check_store(name, params)
148
+ raise Error, "secret references configuration: store #{name}: invalid name" unless STORE_NAME.match?(name)
149
+ raise Error, "secret references configuration: store #{name}: parameters: expecting a Hash" unless params.is_a?(Hash)
150
+ params = params.transform_keys(&:to_s)
151
+ writable = params.delete(WRITABLE_KEY) || false
152
+ raise Error, "secret references configuration: store #{name}: writable: expecting true or false" unless [true, false].include?(writable)
153
+ Store::Factory.check(params)
154
+ {params: params, writable: writable}
155
+ rescue Aspera::Secret::Error => e
156
+ # Re-raise with store context if the message doesn't already include it
157
+ raise e if e.message.start_with?('secret references configuration:')
158
+ raise Error, "secret references configuration: store #{name}: #{e.message}"
159
+ rescue StandardError => e
160
+ raise Error, "secret references configuration: store #{name}: #{e.message}"
161
+ end
162
+
163
+ # @return [Store::Base] New store from its parameters
164
+ def create_store(name, params)
165
+ Store::Factory.create(params)
166
+ rescue StandardError => e
167
+ raise Error, "store #{name}: #{e.message}"
168
+ end
169
+
170
+ # Checks reference and calls block with its store
171
+ # @yieldparam store [Store::Base]
172
+ # @return [Object] Value of block
173
+ def access(ref)
174
+ found = store(ref.store)
175
+ found.check_reference(ref)
176
+ yield(found)
177
+ rescue StandardError => e
178
+ raise Error, "secret reference #{ref.text}: #{reason(e, found)}"
179
+ end
180
+
181
+ # @param error [StandardError] Error
182
+ # @param store [Store::Base, nil] Store that raised the error
183
+ # @return [String] Reason of failure, without secret
184
+ def reason(error, store)
185
+ return error.message if error.is_a?(Error)
186
+ store&.class&.error_reason(error) || Secret::Hider.instance.hide_secrets_in_string("#{error.class}: #{error.message}", all: true)
187
+ end
188
+
189
+ # @return [String, nil] Value from cache, `nil` if absent or expired
190
+ def cached(key)
191
+ @lock.synchronize do
192
+ entry = @cache[key]
193
+ next if entry.nil?
194
+ next entry[:value] if monotonic_time < entry[:expiry]
195
+ @cache.delete(key)
196
+ nil
197
+ end
198
+ end
199
+
200
+ # @return [String] Value, after cache and declaration to `Secret::Hider`
201
+ def cache(key, store_name, value)
202
+ Secret::Hider.instance.add_secret_values(value)
203
+ @lock.synchronize { @cache[key] = {value: value, store: store_name, expiry: monotonic_time + @cache_ttl} } if @cache_ttl.positive?
204
+ value
205
+ end
206
+
207
+ def monotonic_time
208
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
209
+ end
210
+ end
211
+ end
212
+ end