aspera-cli 4.28.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +217 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +122 -26
  10. data/SECURITY.md +84 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1895 -632
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +29 -22
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +95 -67
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +370 -0
  41. data/lib/aspera/ascp/log_locations.rb +163 -0
  42. data/lib/aspera/ascp/management.rb +16 -8
  43. data/lib/aspera/ascp/session_errors.rb +217 -0
  44. data/lib/aspera/assert.rb +4 -0
  45. data/lib/aspera/cli/ascp_actions.rb +12 -19
  46. data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
  47. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  48. data/lib/aspera/cli/bootstrapper.rb +38 -33
  49. data/lib/aspera/cli/command_line.rb +63 -8
  50. data/lib/aspera/cli/command_registry.rb +9 -1
  51. data/lib/aspera/cli/command_spec.rb +47 -7
  52. data/lib/aspera/cli/completion/ascli.bash +19 -5
  53. data/lib/aspera/cli/completion/ascli.fish +13 -2
  54. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  55. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  56. data/lib/aspera/cli/completion_actions.rb +303 -69
  57. data/lib/aspera/cli/context.rb +9 -4
  58. data/lib/aspera/cli/deprecation.rb +5 -2
  59. data/lib/aspera/cli/error.rb +6 -0
  60. data/lib/aspera/cli/extended_value.rb +66 -31
  61. data/lib/aspera/cli/formatter.rb +43 -30
  62. data/lib/aspera/cli/gem_checker.rb +9 -6
  63. data/lib/aspera/cli/hints.rb +22 -10
  64. data/lib/aspera/cli/http.rb +38 -10
  65. data/lib/aspera/cli/info.rb +12 -7
  66. data/lib/aspera/cli/mailer.rb +6 -3
  67. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  68. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  69. data/lib/aspera/cli/mcp/tool.rb +20 -17
  70. data/lib/aspera/cli/option_declarator.rb +10 -4
  71. data/lib/aspera/cli/option_registry.rb +6 -3
  72. data/lib/aspera/cli/option_types.rb +17 -2
  73. data/lib/aspera/cli/option_value.rb +77 -18
  74. data/lib/aspera/cli/options.schema.yaml +641 -80
  75. data/lib/aspera/cli/parser.rb +71 -35
  76. data/lib/aspera/cli/plugins/alee.rb +7 -4
  77. data/lib/aspera/cli/plugins/aoc.rb +248 -37
  78. data/lib/aspera/cli/plugins/ats.rb +10 -9
  79. data/lib/aspera/cli/plugins/base.rb +50 -18
  80. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  81. data/lib/aspera/cli/plugins/config.rb +210 -104
  82. data/lib/aspera/cli/plugins/console.rb +6 -3
  83. data/lib/aspera/cli/plugins/cos.rb +8 -4
  84. data/lib/aspera/cli/plugins/factory.rb +21 -7
  85. data/lib/aspera/cli/plugins/faspex5.rb +31 -26
  86. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  87. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  88. data/lib/aspera/cli/plugins/mcp.rb +42 -22
  89. data/lib/aspera/cli/plugins/node.rb +39 -38
  90. data/lib/aspera/cli/plugins/oauth.rb +23 -6
  91. data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
  92. data/lib/aspera/cli/plugins/preview.rb +72 -45
  93. data/lib/aspera/cli/plugins/server.rb +66 -14
  94. data/lib/aspera/cli/plugins/shares.rb +87 -18
  95. data/lib/aspera/cli/plugins/transferd.rb +228 -0
  96. data/lib/aspera/cli/poll.rb +236 -0
  97. data/lib/aspera/cli/preset_actions.rb +144 -30
  98. data/lib/aspera/cli/preset_manager.rb +113 -35
  99. data/lib/aspera/cli/prompt.rb +157 -9
  100. data/lib/aspera/cli/result.rb +12 -8
  101. data/lib/aspera/cli/runner.rb +89 -48
  102. data/lib/aspera/cli/secret_finder.rb +4 -1
  103. data/lib/aspera/cli/security.rb +89 -9
  104. data/lib/aspera/cli/special_values.rb +4 -1
  105. data/lib/aspera/cli/sync_actions.rb +15 -10
  106. data/lib/aspera/cli/table_fitter.rb +3 -0
  107. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  108. data/lib/aspera/cli/transfer_actions.rb +11 -8
  109. data/lib/aspera/cli/transfer_agent.rb +43 -20
  110. data/lib/aspera/cli/transfer_progress.rb +14 -45
  111. data/lib/aspera/cli/vault_actions.rb +45 -0
  112. data/lib/aspera/cli/vault_manager.rb +188 -42
  113. data/lib/aspera/cli/version.rb +4 -1
  114. data/lib/aspera/cli/web_secret.rb +12 -10
  115. data/lib/aspera/cli/wizard.rb +42 -39
  116. data/lib/aspera/command_line_builder.rb +9 -5
  117. data/lib/aspera/command_line_converter.rb +4 -1
  118. data/lib/aspera/coverage.rb +103 -33
  119. data/lib/aspera/data_repository.rb +4 -1
  120. data/lib/aspera/dot_container.rb +4 -1
  121. data/lib/aspera/environment.rb +23 -12
  122. data/lib/aspera/exec_spec.rb +3 -0
  123. data/lib/aspera/faspex_gw.rb +13 -9
  124. data/lib/aspera/faspex_postproc.rb +15 -12
  125. data/lib/aspera/formatter_interface.rb +4 -1
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +20 -5
  129. data/lib/aspera/inflate.rb +43 -0
  130. data/lib/aspera/json_rpc/client.rb +6 -5
  131. data/lib/aspera/json_rpc/version.rb +3 -0
  132. data/lib/aspera/line_logger.rb +4 -1
  133. data/lib/aspera/link_header.rb +4 -1
  134. data/lib/aspera/log.rb +18 -5
  135. data/lib/aspera/markdown.rb +5 -2
  136. data/lib/aspera/mime.rb +3 -0
  137. data/lib/aspera/nagios.rb +6 -4
  138. data/lib/aspera/node_emulator.rb +15 -13
  139. data/lib/aspera/oauth/client/base.rb +124 -73
  140. data/lib/aspera/oauth/client/boot.rb +7 -4
  141. data/lib/aspera/oauth/client/factory.rb +41 -18
  142. data/lib/aspera/oauth/client/generic.rb +5 -1
  143. data/lib/aspera/oauth/client/json_credentials.rb +7 -3
  144. data/lib/aspera/oauth/client/jwt.rb +6 -3
  145. data/lib/aspera/oauth/client/shares_web.rb +94 -0
  146. data/lib/aspera/oauth/client/url_json.rb +7 -3
  147. data/lib/aspera/oauth/client/web.rb +40 -13
  148. data/lib/aspera/oauth/server/pkce.rb +6 -5
  149. data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +16 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +17 -2
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +3 -0
  165. data/lib/aspera/rest/aspera_errors.rb +4 -1
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +45 -34
  168. data/lib/aspera/rest/error_analyzer.rb +5 -2
  169. data/lib/aspera/rest/list.rb +8 -4
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +8 -5
  172. data/lib/aspera/rest.rb +9 -6
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  175. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  176. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  177. data/lib/aspera/schema/async_tables.yaml +11 -10
  178. data/lib/aspera/schema/documentation.rb +8 -2
  179. data/lib/aspera/schema/reader.rb +7 -0
  180. data/lib/aspera/schema/registry.rb +44 -28
  181. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  182. data/lib/aspera/schema/validator.rb +4 -1
  183. data/lib/aspera/sealed_hash.rb +150 -0
  184. data/lib/aspera/secret/hider.rb +213 -0
  185. data/lib/aspera/secret/reference.rb +150 -0
  186. data/lib/aspera/secret/resolver.rb +212 -0
  187. data/lib/aspera/secret/store/base.rb +153 -0
  188. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  189. data/lib/aspera/secret/store/factory.rb +103 -0
  190. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  191. data/lib/aspera/secret/store/keeper.rb +141 -0
  192. data/lib/aspera/secret/store/macos_security.rb +284 -0
  193. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  194. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  195. data/lib/aspera/secret/store/one_password_cli.rb +112 -0
  196. data/lib/aspera/secret/store/secret_service.rb +125 -0
  197. data/lib/aspera/secret/strength.rb +119 -0
  198. data/lib/aspera/servlet_helpers.rb +5 -2
  199. data/lib/aspera/ssh.rb +10 -7
  200. data/lib/aspera/ssl.rb +5 -2
  201. data/lib/aspera/string_ext.rb +3 -0
  202. data/lib/aspera/sync/args.schema.yaml +3 -0
  203. data/lib/aspera/sync/conf.schema.yaml +9 -4
  204. data/lib/aspera/sync/database.rb +3 -0
  205. data/lib/aspera/sync/operations.rb +17 -15
  206. data/lib/aspera/temp_file_manager.rb +8 -4
  207. data/lib/aspera/timer_limiter.rb +3 -0
  208. data/lib/aspera/transfer/error.rb +4 -1
  209. data/lib/aspera/transfer/faux_file.rb +4 -1
  210. data/lib/aspera/transfer/parameters.rb +17 -14
  211. data/lib/aspera/transfer/result.rb +23 -0
  212. data/lib/aspera/transfer/resumer.rb +6 -4
  213. data/lib/aspera/transfer/spec.rb +9 -22
  214. data/lib/aspera/units/decimal.rb +67 -0
  215. data/lib/aspera/units/duration.rb +109 -0
  216. data/lib/aspera/uri_reader.rb +64 -9
  217. data/lib/aspera/web_auth/index.html.erb +6 -2
  218. data/lib/aspera/web_auth/paste.rb +109 -0
  219. data/lib/aspera/web_auth/receiver.rb +50 -0
  220. data/lib/aspera/web_auth/relay.rb +76 -0
  221. data/lib/aspera/web_auth/reply.html.erb +150 -0
  222. data/lib/aspera/web_auth/server.rb +47 -10
  223. data/lib/aspera/web_page.rb +7 -2
  224. data/lib/aspera/web_server_simple.rb +6 -3
  225. data/lib/aspera/yaml.rb +5 -1
  226. data/llms-install.md +88 -0
  227. data/llms.txt +87 -0
  228. data/skills/ascli/SKILL.md +94 -0
  229. data.tar.gz.sig +0 -0
  230. metadata +77 -20
  231. metadata.gz.sig +0 -0
  232. data/lib/aspera/duration.rb +0 -42
  233. data/lib/aspera/keychain/base.rb +0 -30
  234. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  235. data/lib/aspera/keychain/factory.rb +0 -58
  236. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  237. data/lib/aspera/keychain/macos_security.rb +0 -277
  238. data/lib/aspera/keychain/one_password_api.rb +0 -91
  239. data/lib/aspera/keychain/one_password_base.rb +0 -39
  240. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  241. data/lib/aspera/secret_hider.rb +0 -196
  242. data/lib/aspera/secret_strength.rb +0 -114
@@ -0,0 +1,150 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'inflate'
8
+ require 'json'
9
+ require 'zlib'
10
+ require 'openssl'
11
+
12
+ module Aspera
13
+ # Converts a Hash to a compact text, typed or pasted by a user, and back. Only Ruby standard libraries are used.
14
+ # Used for the codes exchanged by the user between `ascli` and the browser, or Shares (option `ui.web_auth=paste`).
15
+ #
16
+ # Steps: Hash -> JSON -> deflate -> AES-256-GCM -> Base64 URL-safe, without padding (characters: `A-Z a-z 0-9 - _`).
17
+ #
18
+ # Text layout, once decoded from Base64: `IV (12 bytes) + authentication tag (16 bytes) + encrypted data`.
19
+ # The key is `SHA256(<base key> + <secret> + <slot>)`, where slot is the Unix time divided by the period (15 minutes by default), rounded down.
20
+ # Base key and secret are optional.
21
+ # The authentication tag detects an incomplete, mistyped or modified text, or a wrong key.
22
+ # There is no prefix nor type in the text: the receiver checks the keys of the Hash.
23
+ #
24
+ # The protection depends on the key: if the base key is a constant (e.g. in the source code) and there is no secret, the key is not secret, and the slot is easy to guess:
25
+ # the text is then not readable at first sight and is not valid for long, but it is not confidential.
26
+ # To make it confidential, give a `secret` known only by both sides (e.g. a random `state`).
27
+ #
28
+ # The slot changes every period, and the clocks of both sides can differ: `unseal` also tries the previous slot and the next one.
29
+ # So a text is valid from 1 to 2 periods after its creation (15 to 30 minutes by default), and is also accepted if the clock of the receiver is up to 1 period behind.
30
+ # When unsealing, spaces and line breaks are ignored (a text split on several lines is valid), as well as quotes and punctuation around the text.
31
+ # Letter case matters.
32
+ #
33
+ # With `encrypt: false`, there is no encryption and no validity period: the text is only Hash -> JSON -> deflate -> Base64 URL-safe, readable by anyone, without detection of errors.
34
+ # It is used for links that must stay valid and are decoded by other programs (e.g. JavaScript in a browser).
35
+ # `sealed?` does not know this mode.
36
+ # With `reverse: true`, the final text is reversed (it must be the same on both sides), in any mode.
37
+ module SealedHash
38
+ CIPHER = 'aes-256-gcm'
39
+ IV_SIZE = 12
40
+ TAG_SIZE = 16
41
+ # Duration of a slot, in seconds
42
+ PERIOD = 900
43
+ # Shortest text: IV, tag and one byte of data, in Base64
44
+ MIN_TEXT_LENGTH = ((IV_SIZE + TAG_SIZE + 1) * 4 / 3.0).ceil
45
+ # Limits on what is accepted: protection against an abusive input
46
+ MAX_TEXT_LENGTH = 65_536
47
+ MAX_INFLATED_SIZE = 262_144
48
+ # Slots tried to decrypt, relative to the current slot
49
+ SLOT_OFFSETS = [0, -1, 1].freeze
50
+ TEXT_CHARS = /\A[A-Za-z0-9_-]*\z/
51
+ private_constant :CIPHER, :IV_SIZE, :TAG_SIZE, :MIN_TEXT_LENGTH, :MAX_TEXT_LENGTH, :MAX_INFLATED_SIZE, :SLOT_OFFSETS, :TEXT_CHARS
52
+
53
+ class << self
54
+ # @param data [Hash] Data to transform
55
+ # @param secret [String, nil] Value known by both sides, added to the key
56
+ # @param base_key [String, nil] Base of the key, e.g. the name of the application (optional)
57
+ # @param time [Time] Time used for the key
58
+ # @param period [Integer] Duration of a slot, in seconds (must be the same on both sides)
59
+ # @param encrypt [Boolean] `false`: no encryption, no validity period (secret, base_key, time and period are ignored)
60
+ # @param reverse [Boolean] `true`: the text is reversed
61
+ # @return [String] Base64 URL-safe text, without padding
62
+ def seal(data, secret: nil, base_key: nil, time: Time.now, period: PERIOD, encrypt: true, reverse: false)
63
+ Aspera.assert_type(data, Hash)
64
+ deflated = Zlib::Deflate.deflate(JSON.generate(data), Zlib::BEST_COMPRESSION)
65
+ raw =
66
+ if encrypt
67
+ cipher = OpenSSL::Cipher.new(CIPHER).encrypt
68
+ cipher.key = derive_key(base_key, secret, time.to_i / period)
69
+ iv = cipher.random_iv
70
+ body = cipher.update(deflated) + cipher.final
71
+ iv + cipher.auth_tag(TAG_SIZE) + body
72
+ else
73
+ deflated
74
+ end
75
+ text = [raw].pack('m0').tr('+/', '-_').delete('=')
76
+ reverse ? text.reverse : text
77
+ end
78
+
79
+ # @param text [String] Text typed or pasted by the user
80
+ # @return [Boolean] `true` if text looks like a sealed hash (characters and length), to be decoded with `unseal`.
81
+ # A URL or a query always has other characters (`:`, `/`, `?`, `=`, `&`).
82
+ def sealed?(text)
83
+ clean = normalize(text)
84
+ clean.length >= MIN_TEXT_LENGTH && clean.match?(TEXT_CHARS)
85
+ end
86
+
87
+ # @param text [String] Text produced by `seal`
88
+ # @param secret [String, nil] Value known by both sides, added to the key
89
+ # @param base_key [String, nil] Base of the key, e.g. the name of the application (optional)
90
+ # @param time [Time] Time used for the key
91
+ # @param period [Integer] Duration of a slot, in seconds (must be the same on both sides)
92
+ # @param encrypt [Boolean] Same value as for `seal`
93
+ # @param reverse [Boolean] Same value as for `seal`
94
+ # @return [Hash] The data
95
+ # @raise [ParameterError] if text is not valid: incomplete, mistyped, modified, or made with another key or too long ago
96
+ def unseal(text, secret: nil, base_key: nil, time: Time.now, period: PERIOD, encrypt: true, reverse: false)
97
+ clean = normalize(text)
98
+ clean = clean.reverse if reverse
99
+ raise ParameterError, 'Code is too long' if clean.length > MAX_TEXT_LENGTH
100
+ raise ParameterError, 'Code has invalid characters: only letters, digits, - and _ are expected' unless clean.match?(TEXT_CHARS)
101
+ raise ParameterError, 'Code is incomplete' if clean.length < (encrypt ? MIN_TEXT_LENGTH : 1)
102
+ raw = begin
103
+ "#{clean.tr('-_', '+/')}#{'=' * ((4 - (clean.length % 4)) % 4)}".unpack1('m0')
104
+ rescue ArgumentError
105
+ raise ParameterError, 'Code is incomplete or has typing errors'
106
+ end
107
+ if encrypt
108
+ deflated = nil
109
+ SLOT_OFFSETS.each do |offset|
110
+ deflated = decrypt(raw, derive_key(base_key, secret, (time.to_i / period) + offset))
111
+ break unless deflated.nil?
112
+ end
113
+ raise ParameterError, 'Invalid code: incomplete, mistyped, expired, or the clock is wrong (check the time of both computers)' if deflated.nil?
114
+ else
115
+ deflated = raw
116
+ end
117
+ data = JSON.parse(Inflate.limited(deflated, max_size: MAX_INFLATED_SIZE, what: 'Code'))
118
+ raise ParameterError, 'Invalid code: unexpected content' unless data.is_a?(Hash)
119
+ data
120
+ rescue JSON::ParserError, Zlib::Error
121
+ raise ParameterError, 'Invalid code: unexpected content'
122
+ end
123
+
124
+ private
125
+
126
+ # @return [String] Text without spaces and surrounding punctuation (quotes, ...)
127
+ def normalize(text)
128
+ # Only patterns anchored at the start (the end is handled on the reversed text): linear time on any input
129
+ clean = text.to_s.gsub(/\s/, '').sub(/\A[^A-Za-z0-9_-]+/, '')
130
+ clean.reverse.sub(/\A[^A-Za-z0-9_-]+/, '').reverse
131
+ end
132
+
133
+ # @return [String] 32 bytes
134
+ def derive_key(base_key, secret, slot)
135
+ OpenSSL::Digest.digest('SHA256', "#{base_key}#{secret}#{slot}")
136
+ end
137
+
138
+ # @return [String, nil] Deflated data, or `nil` if the key or the tag is not the expected one
139
+ def decrypt(raw, key)
140
+ cipher = OpenSSL::Cipher.new(CIPHER).decrypt
141
+ cipher.key = key
142
+ cipher.iv = raw.byteslice(0, IV_SIZE)
143
+ cipher.auth_tag = raw.byteslice(IV_SIZE, TAG_SIZE)
144
+ cipher.update(raw.byteslice((IV_SIZE + TAG_SIZE)..)) + cipher.final
145
+ rescue OpenSSL::Cipher::CipherError
146
+ nil
147
+ end
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,213 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'logger'
7
+ require 'set'
8
+ require 'singleton'
9
+ require 'json'
10
+ require 'uri'
11
+
12
+ module Aspera
13
+ module Secret
14
+ # remove secret from logs and output
15
+ #
16
+ # Secrets are identified:
17
+ # - by key name, in structures (`secret?`, `hidden_copy`) and in strings with known patterns (`hide_secrets_in_string`)
18
+ # - by value, when declared where they come from (`add_secret_values`, `add_secrets_in`): option values, tokens, ...
19
+ #
20
+ # @!method self.instance
21
+ # Returns the singleton instance of Secret::Hider
22
+ # @return [Secret::Hider] the singleton instance
23
+ class Hider
24
+ include Singleton
25
+
26
+ # display string for hidden secrets (see `Environment::UNICODE_TO_ASCII`)
27
+ HIDDEN_PASSWORD = '🔑'
28
+ # env vars for ascp with secrets
29
+ ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
30
+ # keys in hash that contain secrets
31
+ KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
32
+ HTTP_SECRETS = %w[Authorization].freeze
33
+ ALL_SECRETS = (ASCP_ENV_SECRETS + KEY_SECRETS + HTTP_SECRETS).freeze
34
+ NON_ENV_SECRETS = (KEY_SECRETS + HTTP_SECRETS).freeze
35
+ # not secrets: `*_file` is the path of a file (e.g. `ca_file`)
36
+ KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/, /public_key$/, /^token_type$/, /_file$/].freeze
37
+ # boolean values as text: not secrets, even for a name that denotes a secret (e.g. `show_secrets`)
38
+ BOOLEAN_TEXTS = %w[yes no true false].freeze
39
+ # min length of hidden secrets, use `+` or `{n,}`
40
+ SECRET_LENGTH = '{5,}'
41
+ # min length of a declared secret value to be hidden inside a string: shorter ones are hidden only when they are the whole string,
42
+ # else common words would be hidden (e.g. password `aspera` in URLs), or optionally as a word (`short_in_text`)
43
+ VALUE_IN_STRING_LENGTH = 8
44
+ # regex that define named captures :begin and :end
45
+ REGEX_LOG_REPLACES = [
46
+ # private key values (place first)
47
+ /(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)\n*/,
48
+ # CLI manager get/set options
49
+ /(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/i,
50
+ # env var ascp exec
51
+ /(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)[^ \n]+(?<end> |$)/,
52
+ # rendered JSON or Ruby (quoted key or legacy symbol key: `"k":"v"`, `"k" => "v"`, `:k=>"v"`)
53
+ /(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>? *(?:=>|:) *")(?:[^"\\]|\\.)+(?<end>")/i,
54
+ # rendered Ruby 3.4+ symbol key: `k: "v"`
55
+ /(?<begin>\b\w*(?:#{ALL_SECRETS.join('|')})\w*: *")(?:[^"\\]|\\.)+(?<end>")/i,
56
+ # rendered Ruby Struct or object: `k="v"`, `@k="v"`
57
+ /(?<begin>@?\b\w*(?:#{ALL_SECRETS.join('|')})\w*=")(?:[^"\\]|\\.)+(?<end>")/i,
58
+ # query parameter of URL: `?token=v` or `&client_secret=v`
59
+ /(?<begin>[?&][^=&\s"]*(?:#{KEY_SECRETS.join('|')})[^=&\s"]*=)[^&\s"#]+(?<end>)/i,
60
+ # logged data
61
+ /(?<begin>(?:#{NON_ENV_SECRETS.join('|')})[ =:]+)[^ "\n]#{SECRET_LENGTH}(?<end>$)/i,
62
+ # cred in http dump
63
+ /(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
64
+ ].freeze
65
+ private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :BOOLEAN_TEXTS, :SECRET_LENGTH, :VALUE_IN_STRING_LENGTH, :REGEX_LOG_REPLACES
66
+ attr_accessor :log_secrets
67
+
68
+ # @return [Proc] new log formatter that hides secrets
69
+ def log_formatter(original_formatter)
70
+ original_formatter ||= Logger::Formatter.new
71
+ # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
72
+ # Hiding is applied on the formatted line, so that non-String messages (e.g. Exception) are covered too
73
+ return lambda do |severity, date_time, program_name, msg|
74
+ line = original_formatter.call(severity, date_time, program_name, msg)
75
+ line = hide_secrets_in_string(line, all: true) if line.is_a?(String) && !@log_secrets
76
+ line
77
+ end
78
+ end
79
+
80
+ # Replace secrets in a string with the hidden password placeholder
81
+ # @param value [String] Input string possibly containing secrets
82
+ # @param all [Boolean] `false`: only declared values and private keys, `true`: also all known secret patterns
83
+ # @return [String] String with secrets replaced by placeholder
84
+ def hide_secrets_in_string(value, all: false)
85
+ marker = hidden_marker
86
+ value = hide_values_in_string(value, marker)
87
+ (all ? REGEX_LOG_REPLACES : REGEX_LOG_REPLACES.first(1)).each do |reg_ex|
88
+ value = value.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{marker}#{Regexp.last_match(:end)}" }
89
+ end
90
+ return value
91
+ end
92
+
93
+ # @param keyword [String, Symbol] Key name to check
94
+ # @param value [String] Associated value (must be a String to be a secret)
95
+ # @return [Boolean] true if the key denotes a secret
96
+ def secret?(keyword, value)
97
+ keyword = keyword.to_s if keyword.is_a?(Symbol)
98
+ # only Strings can be secrets, not booleans, or hash, arrays
99
+ return false unless keyword.is_a?(String) && value.is_a?(String)
100
+ return true if @additional_keys.include?(keyword)
101
+ keyword = keyword.downcase
102
+ # those are not secrets
103
+ return false if KEY_FALSE_POSITIVES.any? { |f| f.match?(keyword) }
104
+ # check if keyword (name) contains an element that designate it as a secret
105
+ ALL_SECRETS.any? { |kw| keyword.include?(kw.downcase) }
106
+ end
107
+
108
+ # Hides secrets in a structure: by key name, and declared values, also in key names (e.g. a secret given as name of a parameter).
109
+ # Containers are copied, not modified: the structure may be shared (e.g. configuration, transfer spec).
110
+ # @param obj [Object] Hash, Array, Struct, String, or other value (returned as is)
111
+ # @param short_in_text [Boolean] `true`: a declared value shorter than 8 characters is also hidden inside a text, as a word
112
+ # (not next to a letter or digit), e.g. a short password in an error message of `mcp server`
113
+ # @return [Object] Copy of obj with secrets replaced by placeholder
114
+ def hidden_copy(obj, short_in_text: false)
115
+ case obj
116
+ when Hash
117
+ obj.to_h do |k, v|
118
+ [k.is_a?(String) ? hidden_copy(k, short_in_text: short_in_text) : k, secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text)]
119
+ end
120
+ when Array then obj.map { |i| hidden_copy(i, short_in_text: short_in_text) }
121
+ when Struct then obj.dup.tap { |s| s.each_pair { |k, v| s[k] = secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text) } }
122
+ when String then hide_values_in_string(obj, hidden_marker, short_in_text: short_in_text)
123
+ else obj
124
+ end
125
+ end
126
+
127
+ # @param obj [Object] Hash, Array, Struct, String, or other value
128
+ # @return [Boolean] `true` if obj contains a secret: by key name, or declared value
129
+ def contains_secret?(obj)
130
+ !hidden_copy(obj).eql?(obj)
131
+ end
132
+
133
+ # Declare secret values: hidden wherever they appear (logs, results), also when escaped in JSON, Ruby or URL
134
+ # @param values [Array<Object>] Secret values: only non-empty Strings are retained
135
+ def add_secret_values(*values)
136
+ forms = values.select { |v| v.is_a?(String) && !v.empty? }.flat_map do |v|
137
+ [v, (JSON.generate(v)[1..-2] rescue nil), v.inspect[1..-2], URI.encode_www_form_component(v)]
138
+ end.compact
139
+ return if forms.empty?
140
+ @mutex.synchronize do
141
+ values = @values | forms
142
+ next if values.size.eql?(@values.size)
143
+ long, short = values.partition { |v| v.length >= VALUE_IN_STRING_LENGTH }
144
+ # Longest first, so that a secret is not partly hidden by another one that it contains
145
+ @values_regex = long.empty? ? nil : Regexp.union(long.sort_by { |v| -v.length })
146
+ @short_values_regex = short.empty? ? nil : /(?<![[:alnum:]])#{Regexp.union(short.sort_by { |v| -v.length })}(?![[:alnum:]])/
147
+ # Replaced, not modified, for readers in other threads
148
+ @values = values.freeze
149
+ end
150
+ end
151
+
152
+ # Declare the secret values found by key name in a structure
153
+ # @param obj [Object] Hash or Array, e.g. a transfer spec
154
+ def add_secrets_in(obj)
155
+ case obj
156
+ when Array then obj.each { |i| add_secrets_in(i) }
157
+ when Hash then obj.each { |k, v| secret?(k, v) ? add_secret_values(v) : add_secrets_in(v) }
158
+ end
159
+ end
160
+
161
+ # Declare the secret values of a named value (e.g. an option): the whole value if the name denotes a secret,
162
+ # else the secret values found by key name in it.
163
+ # A boolean value given as text is not a secret, e.g. `no` for option `cache_tokens`, else it would be hidden everywhere.
164
+ # @param name [String, Symbol] Name, e.g. option name
165
+ # @param value [Object] Value
166
+ def add_named_secrets(name, value)
167
+ return add_secrets_in(value) unless secret?(name, '')
168
+ add_secret_values(value) unless BOOLEAN_TEXTS.include?(value)
169
+ end
170
+
171
+ # Declare additional exact key names whose values are secrets
172
+ # @param keys [Array<String, Symbol>] Key names
173
+ def add_secret_keys(keys)
174
+ @additional_keys.merge(keys.map(&:to_s))
175
+ end
176
+
177
+ private
178
+
179
+ def initialize
180
+ @log_secrets = false
181
+ @additional_keys = Set.new
182
+ # Declared secret values, and their escaped forms
183
+ @values = Set.new.freeze
184
+ # Matches declared values long enough to be hidden inside a string
185
+ @values_regex = nil
186
+ # Matches shorter declared values as a word
187
+ @short_values_regex = nil
188
+ @mutex = Mutex.new
189
+ end
190
+
191
+ # @param value [String] Input string
192
+ # @param marker [String] Placeholder
193
+ # @param short_in_text [Boolean] `true`: also hide short values as a word
194
+ # @return [String] value with declared secret values replaced by placeholder
195
+ def hide_values_in_string(value, marker, short_in_text: false)
196
+ return marker if @values.include?(value)
197
+ long_regex = @values_regex
198
+ short_regex = @short_values_regex if short_in_text
199
+ # Long values first, as they may contain short ones
200
+ value = value.gsub(long_regex, marker) unless long_regex.nil?
201
+ value = value.gsub(short_regex, marker) unless short_regex.nil?
202
+ value
203
+ end
204
+
205
+ # @return [String] Placeholder for hidden secrets, in ASCII when Unicode is not supported
206
+ def hidden_marker
207
+ # Required here, as `aspera/environment` requires `aspera/log`, which requires this file
208
+ require_relative '../environment'
209
+ Environment.char(HIDDEN_PASSWORD)
210
+ end
211
+ end
212
+ end
213
+ end
@@ -0,0 +1,150 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+ require_relative 'hider'
8
+ require 'uri'
9
+
10
+ module Aspera
11
+ # Reference to a secret held in a store: `secret://<store>/<path>[?<options>][#<field>]`
12
+ #
13
+ # The reference is stored instead of the value, the value is read when used (see `Secret::Resolver`).
14
+ # A reference is a whole value, it is not searched in a text.
15
+ # Path and field are percent-decoded (`%20` or a space), options are a URL query (`version=2&mount=a/b`).
16
+ # In `ascli`: `@vault://<store>/<path>[?<options>][#<field>]`, stores of option `vault`.
17
+ module Secret
18
+ # Error on a reference: the message gives the reference, never a value
19
+ class Error < Aspera::Error
20
+ end
21
+
22
+ # Parsed reference
23
+ # @!attribute text [String] Reference as given
24
+ # @!attribute store [String] Name of store
25
+ # @!attribute path [String] Path of secret in store (decoded)
26
+ # @!attribute options [Hash<String,String>] Options
27
+ # @!attribute field [String, nil] Field of secret (decoded), `nil` for default field of store
28
+ Reference = Data.define(:text, :store, :path, :options, :field)
29
+
30
+ # Bootstrap secret of a store in configuration (e.g. `password` of type `file`):
31
+ # the value, or `@file:<path>` to read it from a file when used (e.g. a systemd credential), without final newline.
32
+ # Syntax of extended values of `ascli`, only `@file:` is supported: other prefixes may be added.
33
+ # In YAML, quote the value: `@` cannot start a plain scalar.
34
+ class Bootstrap
35
+ FILE_PREFIX = '@file:'
36
+
37
+ # @param name [String] Name of parameter, for errors
38
+ # @param text [String] Value, or `@file:<path>`
39
+ # @raise [Error] Not a String, or empty
40
+ def initialize(name, text)
41
+ raise Error, "#{name}: expecting the value, or #{FILE_PREFIX}<path>" unless text.is_a?(String) && !text.empty?
42
+ if text.start_with?(FILE_PREFIX)
43
+ @path = text.delete_prefix(FILE_PREFIX)
44
+ raise Error, "#{name}: expecting #{FILE_PREFIX}<path>" if @path.empty?
45
+ else
46
+ @value = text
47
+ Secret::Hider.instance.add_secret_values(text)
48
+ end
49
+ end
50
+
51
+ # @return [Boolean] `true` if read from a file
52
+ def file?
53
+ !@path.nil?
54
+ end
55
+
56
+ # @return [Time, nil] Modification time of file, `nil` for a value
57
+ # @raise [Error] Missing file
58
+ def mtime
59
+ File.mtime(@path) unless @path.nil?
60
+ rescue SystemCallError => e
61
+ raise Error, "secret file: #{e.message}"
62
+ end
63
+
64
+ # @return [String] Value, read from the file at each call
65
+ # @raise [Error] Missing, unreadable or empty file
66
+ def value
67
+ @path.nil? ? @value : Secret.read_file(@path)
68
+ end
69
+
70
+ # No value in logs and errors
71
+ def inspect
72
+ "#<#{self.class} #{@path.nil? ? 'value' : "#{FILE_PREFIX}#{@path}"}>"
73
+ end
74
+ end
75
+
76
+ SCHEME = 'secret://'
77
+ # Store name, path, options, field
78
+ SYNTAX = %r{\Asecret://([A-Za-z0-9_.-]+)/([^?#]+)(?:\?([^#]*))?(?:#(.+))?\z}
79
+ private_constant :SYNTAX
80
+
81
+ class << self
82
+ # @param value [Object] Any value
83
+ # @return [Boolean] `true` if value is a String starting with `secret://` (then `parse_reference` checks it)
84
+ def reference?(value)
85
+ value.is_a?(String) && value.start_with?(SCHEME)
86
+ end
87
+
88
+ # Build a reference string from its components (inverse of `parse_reference`).
89
+ # Path segments are percent-encoded individually so `/` separators are preserved.
90
+ # Field and option values are fully percent-encoded.
91
+ # @param store [String] Name of store
92
+ # @param path [String] Path of secret in store
93
+ # @param options [Hash<String,String>] Options (e.g. `{'mount' => 'kv/team'}`)
94
+ # @param field [String, nil] Field of secret, `nil` for none
95
+ # @return [String] Reference
96
+ def reference_text(store:, path:, options: {}, field: nil)
97
+ encoded_path = path.split('/', -1).map { |seg| encode(seg) }.join('/')
98
+ text = "#{SCHEME}#{store}/#{encoded_path}"
99
+ text += "?#{URI.encode_www_form(options)}" unless options.empty?
100
+ text += "##{encode(field)}" unless field.nil?
101
+ text
102
+ end
103
+
104
+ # @param text [String] Reference
105
+ # @return [Reference] Parsed reference
106
+ # @raise [Error] Invalid syntax
107
+ def parse_reference(text)
108
+ match = SYNTAX.match(text) if text.is_a?(String)
109
+ raise Error, "secret reference #{text}: expected #{SCHEME}<store>/<path>[?<options>][#<field>]" if match.nil?
110
+ Reference.new(
111
+ text: text,
112
+ store: match[1],
113
+ path: decode(text, match[2]),
114
+ options: match[3].nil? ? {} : URI.decode_www_form(match[3]).to_h.freeze,
115
+ field: match[4].nil? ? nil : decode(text, match[4])
116
+ )
117
+ rescue ArgumentError => e
118
+ raise Error, "secret reference #{text}: #{e.message}"
119
+ end
120
+
121
+ # Read a bootstrap secret from a file (`@file:` of `Bootstrap`, e.g. a systemd credential).
122
+ # The value is declared to `Secret::Hider`.
123
+ # @param path [String] Path of file
124
+ # @return [String] Content of file without trailing newline
125
+ # @raise [Error] Missing, unreadable or empty file
126
+ def read_file(path)
127
+ value = File.read(path).chomp
128
+ raise Error, "empty secret file: #{path}" if value.empty?
129
+ Secret::Hider.instance.add_secret_values(value)
130
+ value
131
+ rescue SystemCallError => e
132
+ raise Error, "secret file: #{e.message}"
133
+ end
134
+
135
+ private
136
+
137
+ # @return [String] Percent-decoded part of reference (`+` is not a space)
138
+ def decode(text, part)
139
+ URI.decode_uri_component(part)
140
+ rescue ArgumentError
141
+ raise Error, "secret reference #{text}: invalid percent-encoding: #{part}"
142
+ end
143
+
144
+ # @return [String] Percent-encoded part of reference (encodes all reserved URI characters including `#`, `?`, `&`)
145
+ def encode(part)
146
+ URI.encode_uri_component(part)
147
+ end
148
+ end
149
+ end
150
+ end