aspera-cli 4.28.0 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +143 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1633 -609
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +20 -20
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +71 -65
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +16 -8
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +12 -19
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -33
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +6 -1
  50. data/lib/aspera/cli/command_spec.rb +34 -7
  51. data/lib/aspera/cli/completion/ascli.bash +19 -5
  52. data/lib/aspera/cli/completion/ascli.fish +13 -2
  53. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  54. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  55. data/lib/aspera/cli/completion_actions.rb +303 -69
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +3 -0
  59. data/lib/aspera/cli/extended_value.rb +61 -27
  60. data/lib/aspera/cli/formatter.rb +29 -22
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -10
  63. data/lib/aspera/cli/http.rb +12 -9
  64. data/lib/aspera/cli/info.rb +12 -7
  65. data/lib/aspera/cli/mailer.rb +6 -3
  66. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  67. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  68. data/lib/aspera/cli/mcp/tool.rb +15 -14
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +65 -12
  73. data/lib/aspera/cli/options.schema.yaml +487 -63
  74. data/lib/aspera/cli/parser.rb +56 -32
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +244 -35
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +17 -12
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +198 -103
  81. data/lib/aspera/cli/plugins/console.rb +6 -3
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +21 -7
  84. data/lib/aspera/cli/plugins/faspex5.rb +28 -25
  85. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  86. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +28 -22
  88. data/lib/aspera/cli/plugins/node.rb +31 -30
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +63 -11
  93. data/lib/aspera/cli/plugins/shares.rb +86 -16
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +144 -30
  96. data/lib/aspera/cli/preset_manager.rb +52 -18
  97. data/lib/aspera/cli/prompt.rb +4 -1
  98. data/lib/aspera/cli/result.rb +12 -8
  99. data/lib/aspera/cli/runner.rb +69 -46
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +79 -9
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +3 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +29 -20
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +188 -42
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.rb +12 -10
  113. data/lib/aspera/cli/wizard.rb +42 -39
  114. data/lib/aspera/command_line_builder.rb +9 -5
  115. data/lib/aspera/command_line_converter.rb +4 -1
  116. data/lib/aspera/coverage.rb +103 -33
  117. data/lib/aspera/data_repository.rb +4 -1
  118. data/lib/aspera/dot_container.rb +4 -1
  119. data/lib/aspera/duration.rb +5 -2
  120. data/lib/aspera/environment.rb +21 -12
  121. data/lib/aspera/exec_spec.rb +3 -0
  122. data/lib/aspera/faspex_gw.rb +13 -9
  123. data/lib/aspera/faspex_postproc.rb +15 -12
  124. data/lib/aspera/formatter_interface.rb +4 -1
  125. data/lib/aspera/graphql.rb +5 -1
  126. data/lib/aspera/hash_ext.rb +3 -0
  127. data/lib/aspera/id_generator.rb +4 -2
  128. data/lib/aspera/json_rpc/client.rb +6 -5
  129. data/lib/aspera/json_rpc/version.rb +3 -0
  130. data/lib/aspera/line_logger.rb +4 -1
  131. data/lib/aspera/link_header.rb +4 -1
  132. data/lib/aspera/log.rb +18 -5
  133. data/lib/aspera/markdown.rb +5 -2
  134. data/lib/aspera/mime.rb +3 -0
  135. data/lib/aspera/nagios.rb +6 -4
  136. data/lib/aspera/node_emulator.rb +15 -13
  137. data/lib/aspera/oauth/client/base.rb +91 -70
  138. data/lib/aspera/oauth/client/boot.rb +7 -4
  139. data/lib/aspera/oauth/client/factory.rb +6 -3
  140. data/lib/aspera/oauth/client/generic.rb +5 -1
  141. data/lib/aspera/oauth/client/json_credentials.rb +4 -1
  142. data/lib/aspera/oauth/client/jwt.rb +6 -3
  143. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  144. data/lib/aspera/oauth/client/url_json.rb +4 -1
  145. data/lib/aspera/oauth/client/web.rb +39 -12
  146. data/lib/aspera/oauth/server/pkce.rb +6 -5
  147. data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
  148. data/lib/aspera/persistency_action_once.rb +5 -3
  149. data/lib/aspera/persistency_folder.rb +6 -3
  150. data/lib/aspera/preview/file_types.rb +6 -3
  151. data/lib/aspera/preview/generator.rb +9 -5
  152. data/lib/aspera/preview/options.schema.yaml +3 -3
  153. data/lib/aspera/preview/terminal.rb +8 -4
  154. data/lib/aspera/preview/utils.rb +6 -4
  155. data/lib/aspera/products/connect.rb +5 -2
  156. data/lib/aspera/products/desktop.rb +4 -1
  157. data/lib/aspera/products/other.rb +4 -2
  158. data/lib/aspera/products/transferd.rb +6 -2
  159. data/lib/aspera/products/transferd_service.rb +610 -0
  160. data/lib/aspera/proxy_auto_config.js +2 -2
  161. data/lib/aspera/proxy_auto_config.rb +5 -2
  162. data/lib/aspera/rainbow.rb +3 -0
  163. data/lib/aspera/rest/aspera_errors.rb +4 -1
  164. data/lib/aspera/rest/call_error.rb +3 -0
  165. data/lib/aspera/rest/client.rb +22 -29
  166. data/lib/aspera/rest/error_analyzer.rb +5 -2
  167. data/lib/aspera/rest/list.rb +8 -4
  168. data/lib/aspera/rest/parameters.rb +3 -0
  169. data/lib/aspera/rest/util.rb +8 -5
  170. data/lib/aspera/rest.rb +9 -6
  171. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  172. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  173. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  174. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  175. data/lib/aspera/schema/async_tables.yaml +11 -10
  176. data/lib/aspera/schema/documentation.rb +8 -2
  177. data/lib/aspera/schema/reader.rb +7 -0
  178. data/lib/aspera/schema/registry.rb +43 -28
  179. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  180. data/lib/aspera/schema/validator.rb +4 -1
  181. data/lib/aspera/sealed_hash.rb +166 -0
  182. data/lib/aspera/secret/hider.rb +201 -0
  183. data/lib/aspera/secret/reference.rb +150 -0
  184. data/lib/aspera/secret/resolver.rb +212 -0
  185. data/lib/aspera/secret/store/base.rb +153 -0
  186. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  187. data/lib/aspera/secret/store/factory.rb +103 -0
  188. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  189. data/lib/aspera/secret/store/keeper.rb +141 -0
  190. data/lib/aspera/secret/store/macos_security.rb +284 -0
  191. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  192. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  193. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  194. data/lib/aspera/secret/store/secret_service.rb +125 -0
  195. data/lib/aspera/secret/strength.rb +119 -0
  196. data/lib/aspera/servlet_helpers.rb +5 -2
  197. data/lib/aspera/ssh.rb +10 -7
  198. data/lib/aspera/ssl.rb +5 -2
  199. data/lib/aspera/string_ext.rb +3 -0
  200. data/lib/aspera/sync/args.schema.yaml +3 -0
  201. data/lib/aspera/sync/conf.schema.yaml +9 -4
  202. data/lib/aspera/sync/database.rb +3 -0
  203. data/lib/aspera/sync/operations.rb +16 -14
  204. data/lib/aspera/temp_file_manager.rb +8 -4
  205. data/lib/aspera/timer_limiter.rb +3 -0
  206. data/lib/aspera/transfer/error.rb +4 -1
  207. data/lib/aspera/transfer/faux_file.rb +4 -1
  208. data/lib/aspera/transfer/parameters.rb +17 -14
  209. data/lib/aspera/transfer/result.rb +3 -0
  210. data/lib/aspera/transfer/resumer.rb +6 -4
  211. data/lib/aspera/transfer/spec.rb +9 -22
  212. data/lib/aspera/transfer/units.rb +75 -0
  213. data/lib/aspera/uri_reader.rb +57 -7
  214. data/lib/aspera/web_auth/index.html.erb +6 -2
  215. data/lib/aspera/web_auth/paste.rb +109 -0
  216. data/lib/aspera/web_auth/receiver.rb +50 -0
  217. data/lib/aspera/web_auth/relay.rb +76 -0
  218. data/lib/aspera/web_auth/reply.html.erb +150 -0
  219. data/lib/aspera/web_auth/server.rb +47 -10
  220. data/lib/aspera/web_page.rb +7 -2
  221. data/lib/aspera/web_server_simple.rb +6 -3
  222. data/lib/aspera/yaml.rb +5 -1
  223. data/llms.txt +81 -0
  224. data/skills/ascli/SKILL.md +88 -0
  225. data.tar.gz.sig +0 -0
  226. metadata +72 -19
  227. metadata.gz.sig +0 -0
  228. data/lib/aspera/keychain/base.rb +0 -30
  229. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  230. data/lib/aspera/keychain/factory.rb +0 -58
  231. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  232. data/lib/aspera/keychain/macos_security.rb +0 -277
  233. data/lib/aspera/keychain/one_password_api.rb +0 -91
  234. data/lib/aspera/keychain/one_password_base.rb +0 -39
  235. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  236. data/lib/aspera/secret_hider.rb +0 -196
  237. data/lib/aspera/secret_strength.rb +0 -114
@@ -0,0 +1,166 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require 'json'
8
+ require 'zlib'
9
+ require 'openssl'
10
+
11
+ module Aspera
12
+ # Converts a Hash to a compact text, typed or pasted by a user, and back. Only Ruby standard libraries are used.
13
+ # Used for the codes exchanged by the user between `ascli` and the browser, or Shares (option `ui.web_auth=paste`).
14
+ #
15
+ # Steps: Hash -> JSON -> deflate -> AES-256-GCM -> Base64 URL-safe, without padding (characters: `A-Z a-z 0-9 - _`).
16
+ #
17
+ # Text layout, once decoded from Base64: `IV (12 bytes) + authentication tag (16 bytes) + encrypted data`.
18
+ # The key is `SHA256(<base key> + <secret> + <slot>)`, where slot is the Unix time divided by the period (15 minutes by default), rounded down.
19
+ # Base key and secret are optional.
20
+ # The authentication tag detects an incomplete, mistyped or modified text, or a wrong key.
21
+ # There is no prefix nor type in the text: the receiver checks the keys of the Hash.
22
+ #
23
+ # The protection depends on the key: if the base key is a constant (e.g. in the source code) and there is no secret, the key is not secret, and the slot is easy to guess:
24
+ # the text is then not readable at first sight and is not valid for long, but it is not confidential.
25
+ # To make it confidential, give a `secret` known only by both sides (e.g. a random `state`).
26
+ #
27
+ # The slot changes every period, and the clocks of both sides can differ: `unseal` also tries the previous slot and the next one.
28
+ # So a text is valid from 1 to 2 periods after its creation (15 to 30 minutes by default), and is also accepted if the clock of the receiver is up to 1 period behind.
29
+ # When unsealing, spaces and line breaks are ignored (a text split on several lines is valid), as well as quotes and punctuation around the text.
30
+ # Letter case matters.
31
+ #
32
+ # With `encrypt: false`, there is no encryption and no validity period: the text is only Hash -> JSON -> deflate -> Base64 URL-safe, readable by anyone, without detection of errors.
33
+ # It is used for links that must stay valid and are decoded by other programs (e.g. JavaScript in a browser).
34
+ # `sealed?` does not know this mode.
35
+ # With `reverse: true`, the final text is reversed (it must be the same on both sides), in any mode.
36
+ module SealedHash
37
+ CIPHER = 'aes-256-gcm'
38
+ IV_SIZE = 12
39
+ TAG_SIZE = 16
40
+ # Duration of a slot, in seconds
41
+ PERIOD = 900
42
+ # Shortest text: IV, tag and one byte of data, in Base64
43
+ MIN_TEXT_LENGTH = ((IV_SIZE + TAG_SIZE + 1) * 4 / 3.0).ceil
44
+ # Limits on what is accepted: protection against an abusive input
45
+ MAX_TEXT_LENGTH = 65_536
46
+ MAX_INFLATED_SIZE = 262_144
47
+ # Slots tried to decrypt, relative to the current slot
48
+ SLOT_OFFSETS = [0, -1, 1].freeze
49
+ TEXT_CHARS = /\A[A-Za-z0-9_-]*\z/
50
+ private_constant :CIPHER, :IV_SIZE, :TAG_SIZE, :MIN_TEXT_LENGTH, :MAX_TEXT_LENGTH, :MAX_INFLATED_SIZE, :SLOT_OFFSETS, :TEXT_CHARS
51
+
52
+ class << self
53
+ # @param data [Hash] Data to transform
54
+ # @param secret [String, nil] Value known by both sides, added to the key
55
+ # @param base_key [String, nil] Base of the key, e.g. the name of the application (optional)
56
+ # @param time [Time] Time used for the key
57
+ # @param period [Integer] Duration of a slot, in seconds (must be the same on both sides)
58
+ # @param encrypt [Boolean] `false`: no encryption, no validity period (secret, base_key, time and period are ignored)
59
+ # @param reverse [Boolean] `true`: the text is reversed
60
+ # @return [String] Base64 URL-safe text, without padding
61
+ def seal(data, secret: nil, base_key: nil, time: Time.now, period: PERIOD, encrypt: true, reverse: false)
62
+ Aspera.assert_type(data, Hash)
63
+ deflated = Zlib::Deflate.deflate(JSON.generate(data), Zlib::BEST_COMPRESSION)
64
+ raw =
65
+ if encrypt
66
+ cipher = OpenSSL::Cipher.new(CIPHER).encrypt
67
+ cipher.key = derive_key(base_key, secret, time.to_i / period)
68
+ iv = cipher.random_iv
69
+ body = cipher.update(deflated) + cipher.final
70
+ iv + cipher.auth_tag(TAG_SIZE) + body
71
+ else
72
+ deflated
73
+ end
74
+ text = [raw].pack('m0').tr('+/', '-_').delete('=')
75
+ reverse ? text.reverse : text
76
+ end
77
+
78
+ # @param text [String] Text typed or pasted by the user
79
+ # @return [Boolean] `true` if text looks like a sealed hash (characters and length), to be decoded with `unseal`.
80
+ # A URL or a query always has other characters (`:`, `/`, `?`, `=`, `&`).
81
+ def sealed?(text)
82
+ clean = normalize(text)
83
+ clean.length >= MIN_TEXT_LENGTH && clean.match?(TEXT_CHARS)
84
+ end
85
+
86
+ # @param text [String] Text produced by `seal`
87
+ # @param secret [String, nil] Value known by both sides, added to the key
88
+ # @param base_key [String, nil] Base of the key, e.g. the name of the application (optional)
89
+ # @param time [Time] Time used for the key
90
+ # @param period [Integer] Duration of a slot, in seconds (must be the same on both sides)
91
+ # @param encrypt [Boolean] Same value as for `seal`
92
+ # @param reverse [Boolean] Same value as for `seal`
93
+ # @return [Hash] The data
94
+ # @raise [ParameterError] if text is not valid: incomplete, mistyped, modified, or made with another key or too long ago
95
+ def unseal(text, secret: nil, base_key: nil, time: Time.now, period: PERIOD, encrypt: true, reverse: false)
96
+ clean = normalize(text)
97
+ clean = clean.reverse if reverse
98
+ raise ParameterError, 'Code is too long' if clean.length > MAX_TEXT_LENGTH
99
+ raise ParameterError, 'Code has invalid characters: only letters, digits, - and _ are expected' unless clean.match?(TEXT_CHARS)
100
+ raise ParameterError, 'Code is incomplete' if clean.length < (encrypt ? MIN_TEXT_LENGTH : 1)
101
+ raw = begin
102
+ "#{clean.tr('-_', '+/')}#{'=' * ((4 - (clean.length % 4)) % 4)}".unpack1('m0')
103
+ rescue ArgumentError
104
+ raise ParameterError, 'Code is incomplete or has typing errors'
105
+ end
106
+ if encrypt
107
+ deflated = nil
108
+ SLOT_OFFSETS.each do |offset|
109
+ deflated = decrypt(raw, derive_key(base_key, secret, (time.to_i / period) + offset))
110
+ break unless deflated.nil?
111
+ end
112
+ raise ParameterError, 'Invalid code: incomplete, mistyped, expired, or the clock is wrong (check the time of both computers)' if deflated.nil?
113
+ else
114
+ deflated = raw
115
+ end
116
+ data = JSON.parse(inflate(deflated))
117
+ raise ParameterError, 'Invalid code: unexpected content' unless data.is_a?(Hash)
118
+ data
119
+ rescue JSON::ParserError, Zlib::Error
120
+ raise ParameterError, 'Invalid code: unexpected content'
121
+ end
122
+
123
+ private
124
+
125
+ # @return [String] Text without spaces and surrounding punctuation (quotes, ...)
126
+ def normalize(text)
127
+ # Only patterns anchored at the start (the end is handled on the reversed text): linear time on any input
128
+ clean = text.to_s.gsub(/\s/, '').sub(/\A[^A-Za-z0-9_-]+/, '')
129
+ clean.reverse.sub(/\A[^A-Za-z0-9_-]+/, '').reverse
130
+ end
131
+
132
+ # @return [String] 32 bytes
133
+ def derive_key(base_key, secret, slot)
134
+ OpenSSL::Digest.digest('SHA256', "#{base_key}#{secret}#{slot}")
135
+ end
136
+
137
+ # @return [String, nil] Deflated data, or `nil` if the key or the tag is not the expected one
138
+ def decrypt(raw, key)
139
+ cipher = OpenSSL::Cipher.new(CIPHER).decrypt
140
+ cipher.key = key
141
+ cipher.iv = raw.byteslice(0, IV_SIZE)
142
+ cipher.auth_tag = raw.byteslice(IV_SIZE, TAG_SIZE)
143
+ cipher.update(raw.byteslice((IV_SIZE + TAG_SIZE)..)) + cipher.final
144
+ rescue OpenSSL::Cipher::CipherError
145
+ nil
146
+ end
147
+
148
+ # @param body [String] Deflated data
149
+ # @return [String] Inflated data, limited in size
150
+ def inflate(body)
151
+ output = +''
152
+ inflater = Zlib::Inflate.new
153
+ begin
154
+ inflater.inflate(body) do |chunk|
155
+ output << chunk
156
+ raise ParameterError, 'Invalid code: content is too large' if output.bytesize > MAX_INFLATED_SIZE
157
+ end
158
+ raise ParameterError, 'Code is incomplete' unless inflater.finished?
159
+ ensure
160
+ inflater.close
161
+ end
162
+ output
163
+ end
164
+ end
165
+ end
166
+ end
@@ -0,0 +1,201 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'logger'
7
+ require 'set'
8
+ require 'singleton'
9
+ require 'json'
10
+ require 'uri'
11
+
12
+ module Aspera
13
+ module Secret
14
+ # remove secret from logs and output
15
+ #
16
+ # Secrets are identified:
17
+ # - by key name, in structures (`secret?`, `hidden_copy`) and in strings with known patterns (`hide_secrets_in_string`)
18
+ # - by value, when declared where they come from (`add_secret_values`, `add_secrets_in`): option values, tokens, ...
19
+ #
20
+ # @!method self.instance
21
+ # Returns the singleton instance of Secret::Hider
22
+ # @return [Secret::Hider] the singleton instance
23
+ class Hider
24
+ include Singleton
25
+
26
+ # display string for hidden secrets (see `Environment::UNICODE_TO_ASCII`)
27
+ HIDDEN_PASSWORD = '🔑'
28
+ # env vars for ascp with secrets
29
+ ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
30
+ # keys in hash that contain secrets
31
+ KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
32
+ HTTP_SECRETS = %w[Authorization].freeze
33
+ ALL_SECRETS = (ASCP_ENV_SECRETS + KEY_SECRETS + HTTP_SECRETS).freeze
34
+ NON_ENV_SECRETS = (KEY_SECRETS + HTTP_SECRETS).freeze
35
+ # not secrets: `*_file` is the path of a file (e.g. `ca_file`)
36
+ KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/, /public_key$/, /^token_type$/, /_file$/].freeze
37
+ # min length of hidden secrets, use `+` or `{n,}`
38
+ SECRET_LENGTH = '{5,}'
39
+ # min length of a declared secret value to be hidden inside a string: shorter ones are hidden only when they are the whole string,
40
+ # else common words would be hidden (e.g. password `aspera` in URLs), or optionally as a word (`short_in_text`)
41
+ VALUE_IN_STRING_LENGTH = 8
42
+ # regex that define named captures :begin and :end
43
+ REGEX_LOG_REPLACES = [
44
+ # private key values (place first)
45
+ /(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)\n*/,
46
+ # CLI manager get/set options
47
+ /(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/i,
48
+ # env var ascp exec
49
+ /(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)[^ \n]+(?<end> |$)/,
50
+ # rendered JSON or Ruby (quoted key or legacy symbol key: `"k":"v"`, `"k" => "v"`, `:k=>"v"`)
51
+ /(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>? *(?:=>|:) *")(?:[^"\\]|\\.)+(?<end>")/i,
52
+ # rendered Ruby 3.4+ symbol key: `k: "v"`
53
+ /(?<begin>\b\w*(?:#{ALL_SECRETS.join('|')})\w*: *")(?:[^"\\]|\\.)+(?<end>")/i,
54
+ # rendered Ruby Struct or object: `k="v"`, `@k="v"`
55
+ /(?<begin>@?\b\w*(?:#{ALL_SECRETS.join('|')})\w*=")(?:[^"\\]|\\.)+(?<end>")/i,
56
+ # query parameter of URL: `?token=v` or `&client_secret=v`
57
+ /(?<begin>[?&][^=&\s"]*(?:#{KEY_SECRETS.join('|')})[^=&\s"]*=)[^&\s"#]+(?<end>)/i,
58
+ # logged data
59
+ /(?<begin>(?:#{NON_ENV_SECRETS.join('|')})[ =:]+)[^ "\n]#{SECRET_LENGTH}(?<end>$)/i,
60
+ # cred in http dump
61
+ /(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
62
+ ].freeze
63
+ private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :SECRET_LENGTH, :VALUE_IN_STRING_LENGTH, :REGEX_LOG_REPLACES
64
+ attr_accessor :log_secrets
65
+
66
+ # @return [Proc] new log formatter that hides secrets
67
+ def log_formatter(original_formatter)
68
+ original_formatter ||= Logger::Formatter.new
69
+ # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
70
+ # Hiding is applied on the formatted line, so that non-String messages (e.g. Exception) are covered too
71
+ return lambda do |severity, date_time, program_name, msg|
72
+ line = original_formatter.call(severity, date_time, program_name, msg)
73
+ line = hide_secrets_in_string(line, all: true) if line.is_a?(String) && !@log_secrets
74
+ line
75
+ end
76
+ end
77
+
78
+ # Replace secrets in a string with the hidden password placeholder
79
+ # @param value [String] Input string possibly containing secrets
80
+ # @param all [Boolean] `false`: only declared values and private keys, `true`: also all known secret patterns
81
+ # @return [String] String with secrets replaced by placeholder
82
+ def hide_secrets_in_string(value, all: false)
83
+ marker = hidden_marker
84
+ value = hide_values_in_string(value, marker)
85
+ (all ? REGEX_LOG_REPLACES : REGEX_LOG_REPLACES.first(1)).each do |reg_ex|
86
+ value = value.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{marker}#{Regexp.last_match(:end)}" }
87
+ end
88
+ return value
89
+ end
90
+
91
+ # @param keyword [String, Symbol] Key name to check
92
+ # @param value [String] Associated value (must be a String to be a secret)
93
+ # @return [Boolean] true if the key denotes a secret
94
+ def secret?(keyword, value)
95
+ keyword = keyword.to_s if keyword.is_a?(Symbol)
96
+ # only Strings can be secrets, not booleans, or hash, arrays
97
+ return false unless keyword.is_a?(String) && value.is_a?(String)
98
+ return true if @additional_keys.include?(keyword)
99
+ keyword = keyword.downcase
100
+ # those are not secrets
101
+ return false if KEY_FALSE_POSITIVES.any? { |f| f.match?(keyword) }
102
+ # check if keyword (name) contains an element that designate it as a secret
103
+ ALL_SECRETS.any? { |kw| keyword.include?(kw.downcase) }
104
+ end
105
+
106
+ # Hides secrets in a structure: by key name, and declared values, also in key names (e.g. a secret given as name of a parameter).
107
+ # Containers are copied, not modified: the structure may be shared (e.g. configuration, transfer spec).
108
+ # @param obj [Object] Hash, Array, Struct, String, or other value (returned as is)
109
+ # @param short_in_text [Boolean] `true`: a declared value shorter than 8 characters is also hidden inside a text, as a word
110
+ # (not next to a letter or digit), e.g. a short password in an error message of `mcp server`
111
+ # @return [Object] Copy of obj with secrets replaced by placeholder
112
+ def hidden_copy(obj, short_in_text: false)
113
+ case obj
114
+ when Hash
115
+ obj.to_h do |k, v|
116
+ [k.is_a?(String) ? hidden_copy(k, short_in_text: short_in_text) : k, secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text)]
117
+ end
118
+ when Array then obj.map { |i| hidden_copy(i, short_in_text: short_in_text) }
119
+ when Struct then obj.dup.tap { |s| s.each_pair { |k, v| s[k] = secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text) } }
120
+ when String then hide_values_in_string(obj, hidden_marker, short_in_text: short_in_text)
121
+ else obj
122
+ end
123
+ end
124
+
125
+ # @param obj [Object] Hash, Array, Struct, String, or other value
126
+ # @return [Boolean] `true` if obj contains a secret: by key name, or declared value
127
+ def contains_secret?(obj)
128
+ !hidden_copy(obj).eql?(obj)
129
+ end
130
+
131
+ # Declare secret values: hidden wherever they appear (logs, results), also when escaped in JSON, Ruby or URL
132
+ # @param values [Array<Object>] Secret values: only non-empty Strings are retained
133
+ def add_secret_values(*values)
134
+ forms = values.select { |v| v.is_a?(String) && !v.empty? }.flat_map do |v|
135
+ [v, (JSON.generate(v)[1..-2] rescue nil), v.inspect[1..-2], URI.encode_www_form_component(v)]
136
+ end.compact
137
+ return if forms.empty?
138
+ @mutex.synchronize do
139
+ values = @values | forms
140
+ next if values.size.eql?(@values.size)
141
+ long, short = values.partition { |v| v.length >= VALUE_IN_STRING_LENGTH }
142
+ # Longest first, so that a secret is not partly hidden by another one that it contains
143
+ @values_regex = long.empty? ? nil : Regexp.union(long.sort_by { |v| -v.length })
144
+ @short_values_regex = short.empty? ? nil : /(?<![[:alnum:]])#{Regexp.union(short.sort_by { |v| -v.length })}(?![[:alnum:]])/
145
+ # Replaced, not modified, for readers in other threads
146
+ @values = values.freeze
147
+ end
148
+ end
149
+
150
+ # Declare the secret values found by key name in a structure
151
+ # @param obj [Object] Hash or Array, e.g. a transfer spec
152
+ def add_secrets_in(obj)
153
+ case obj
154
+ when Array then obj.each { |i| add_secrets_in(i) }
155
+ when Hash then obj.each { |k, v| secret?(k, v) ? add_secret_values(v) : add_secrets_in(v) }
156
+ end
157
+ end
158
+
159
+ # Declare additional exact key names whose values are secrets
160
+ # @param keys [Array<String, Symbol>] Key names
161
+ def add_secret_keys(keys)
162
+ @additional_keys.merge(keys.map(&:to_s))
163
+ end
164
+
165
+ private
166
+
167
+ def initialize
168
+ @log_secrets = false
169
+ @additional_keys = Set.new
170
+ # Declared secret values, and their escaped forms
171
+ @values = Set.new.freeze
172
+ # Matches declared values long enough to be hidden inside a string
173
+ @values_regex = nil
174
+ # Matches shorter declared values as a word
175
+ @short_values_regex = nil
176
+ @mutex = Mutex.new
177
+ end
178
+
179
+ # @param value [String] Input string
180
+ # @param marker [String] Placeholder
181
+ # @param short_in_text [Boolean] `true`: also hide short values as a word
182
+ # @return [String] value with declared secret values replaced by placeholder
183
+ def hide_values_in_string(value, marker, short_in_text: false)
184
+ return marker if @values.include?(value)
185
+ long_regex = @values_regex
186
+ short_regex = @short_values_regex if short_in_text
187
+ # Long values first, as they may contain short ones
188
+ value = value.gsub(long_regex, marker) unless long_regex.nil?
189
+ value = value.gsub(short_regex, marker) unless short_regex.nil?
190
+ value
191
+ end
192
+
193
+ # @return [String] Placeholder for hidden secrets, in ASCII when Unicode is not supported
194
+ def hidden_marker
195
+ # Required here, as `aspera/environment` requires `aspera/log`, which requires this file
196
+ require_relative '../environment'
197
+ Environment.char(HIDDEN_PASSWORD)
198
+ end
199
+ end
200
+ end
201
+ end
@@ -0,0 +1,150 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+ require_relative 'hider'
8
+ require 'uri'
9
+
10
+ module Aspera
11
+ # Reference to a secret held in a store: `secret://<store>/<path>[?<options>][#<field>]`
12
+ #
13
+ # The reference is stored instead of the value, the value is read when used (see `Secret::Resolver`).
14
+ # A reference is a whole value, it is not searched in a text.
15
+ # Path and field are percent-decoded (`%20` or a space), options are a URL query (`version=2&mount=a/b`).
16
+ # In `ascli`: `@vault://<store>/<path>[?<options>][#<field>]`, stores of option `vault`.
17
+ module Secret
18
+ # Error on a reference: the message gives the reference, never a value
19
+ class Error < Aspera::Error
20
+ end
21
+
22
+ # Parsed reference
23
+ # @!attribute text [String] Reference as given
24
+ # @!attribute store [String] Name of store
25
+ # @!attribute path [String] Path of secret in store (decoded)
26
+ # @!attribute options [Hash<String,String>] Options
27
+ # @!attribute field [String, nil] Field of secret (decoded), `nil` for default field of store
28
+ Reference = Data.define(:text, :store, :path, :options, :field)
29
+
30
+ # Bootstrap secret of a store in configuration (e.g. `password` of type `file`):
31
+ # the value, or `@file:<path>` to read it from a file when used (e.g. a systemd credential), without final newline.
32
+ # Syntax of extended values of `ascli`, only `@file:` is supported: other prefixes may be added.
33
+ # In YAML, quote the value: `@` cannot start a plain scalar.
34
+ class Bootstrap
35
+ FILE_PREFIX = '@file:'
36
+
37
+ # @param name [String] Name of parameter, for errors
38
+ # @param text [String] Value, or `@file:<path>`
39
+ # @raise [Error] Not a String, or empty
40
+ def initialize(name, text)
41
+ raise Error, "#{name}: expecting the value, or #{FILE_PREFIX}<path>" unless text.is_a?(String) && !text.empty?
42
+ if text.start_with?(FILE_PREFIX)
43
+ @path = text.delete_prefix(FILE_PREFIX)
44
+ raise Error, "#{name}: expecting #{FILE_PREFIX}<path>" if @path.empty?
45
+ else
46
+ @value = text
47
+ Secret::Hider.instance.add_secret_values(text)
48
+ end
49
+ end
50
+
51
+ # @return [Boolean] `true` if read from a file
52
+ def file?
53
+ !@path.nil?
54
+ end
55
+
56
+ # @return [Time, nil] Modification time of file, `nil` for a value
57
+ # @raise [Error] Missing file
58
+ def mtime
59
+ File.mtime(@path) unless @path.nil?
60
+ rescue SystemCallError => e
61
+ raise Error, "secret file: #{e.message}"
62
+ end
63
+
64
+ # @return [String] Value, read from the file at each call
65
+ # @raise [Error] Missing, unreadable or empty file
66
+ def value
67
+ @path.nil? ? @value : Secret.read_file(@path)
68
+ end
69
+
70
+ # No value in logs and errors
71
+ def inspect
72
+ "#<#{self.class} #{@path.nil? ? 'value' : "#{FILE_PREFIX}#{@path}"}>"
73
+ end
74
+ end
75
+
76
+ SCHEME = 'secret://'
77
+ # Store name, path, options, field
78
+ SYNTAX = %r{\Asecret://([A-Za-z0-9_.-]+)/([^?#]+)(?:\?([^#]*))?(?:#(.+))?\z}
79
+ private_constant :SYNTAX
80
+
81
+ class << self
82
+ # @param value [Object] Any value
83
+ # @return [Boolean] `true` if value is a String starting with `secret://` (then `parse_reference` checks it)
84
+ def reference?(value)
85
+ value.is_a?(String) && value.start_with?(SCHEME)
86
+ end
87
+
88
+ # Build a reference string from its components (inverse of `parse_reference`).
89
+ # Path segments are percent-encoded individually so `/` separators are preserved.
90
+ # Field and option values are fully percent-encoded.
91
+ # @param store [String] Name of store
92
+ # @param path [String] Path of secret in store
93
+ # @param options [Hash<String,String>] Options (e.g. `{'mount' => 'kv/team'}`)
94
+ # @param field [String, nil] Field of secret, `nil` for none
95
+ # @return [String] Reference
96
+ def reference_text(store:, path:, options: {}, field: nil)
97
+ encoded_path = path.split('/', -1).map { |seg| encode(seg) }.join('/')
98
+ text = "#{SCHEME}#{store}/#{encoded_path}"
99
+ text += "?#{URI.encode_www_form(options)}" unless options.empty?
100
+ text += "##{encode(field)}" unless field.nil?
101
+ text
102
+ end
103
+
104
+ # @param text [String] Reference
105
+ # @return [Reference] Parsed reference
106
+ # @raise [Error] Invalid syntax
107
+ def parse_reference(text)
108
+ match = SYNTAX.match(text) if text.is_a?(String)
109
+ raise Error, "secret reference #{text}: expected #{SCHEME}<store>/<path>[?<options>][#<field>]" if match.nil?
110
+ Reference.new(
111
+ text: text,
112
+ store: match[1],
113
+ path: decode(text, match[2]),
114
+ options: match[3].nil? ? {} : URI.decode_www_form(match[3]).to_h.freeze,
115
+ field: match[4].nil? ? nil : decode(text, match[4])
116
+ )
117
+ rescue ArgumentError => e
118
+ raise Error, "secret reference #{text}: #{e.message}"
119
+ end
120
+
121
+ # Read a bootstrap secret from a file (`@file:` of `Bootstrap`, e.g. a systemd credential).
122
+ # The value is declared to `Secret::Hider`.
123
+ # @param path [String] Path of file
124
+ # @return [String] Content of file without trailing newline
125
+ # @raise [Error] Missing, unreadable or empty file
126
+ def read_file(path)
127
+ value = File.read(path).chomp
128
+ raise Error, "empty secret file: #{path}" if value.empty?
129
+ Secret::Hider.instance.add_secret_values(value)
130
+ value
131
+ rescue SystemCallError => e
132
+ raise Error, "secret file: #{e.message}"
133
+ end
134
+
135
+ private
136
+
137
+ # @return [String] Percent-decoded part of reference (`+` is not a space)
138
+ def decode(text, part)
139
+ URI.decode_uri_component(part)
140
+ rescue ArgumentError
141
+ raise Error, "secret reference #{text}: invalid percent-encoding: #{part}"
142
+ end
143
+
144
+ # @return [String] Percent-encoded part of reference (encodes all reserved URI characters including `#`, `?`, `&`)
145
+ def encode(part)
146
+ URI.encode_uri_component(part)
147
+ end
148
+ end
149
+ end
150
+ end