aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -0,0 +1,390 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../../environment'
7
+ require_relative '../../log'
8
+ require_relative '../../assert'
9
+ require_relative '../hider'
10
+ require_relative '../reference'
11
+ require_relative 'base'
12
+ require 'vault'
13
+ require 'time'
14
+
15
+ module Aspera
16
+ module Secret
17
+ module Store
18
+ # Manage secrets in a Hashicorp Vault
19
+ class HashicorpVault < Base
20
+ STORE_PATH = 'secret/data/'
21
+ APPROLE_KEYS = %i[mount role_id secret_id].freeze
22
+ APPROLE_MOUNT = 'approle'
23
+ # The AppRole token is renewed after this part of its lease
24
+ RENEW_RATIO = 2.0 / 3
25
+ HTTP_FORBIDDEN = 403
26
+ # Invalid segments of path of reference
27
+ INVALID_SEGMENTS = ['', '.', '..'].freeze
28
+
29
+ private_constant :STORE_PATH, :APPROLE_KEYS, :APPROLE_MOUNT, :RENEW_RATIO, :HTTP_FORBIDDEN, :INVALID_SEGMENTS
30
+
31
+ # Reference: `secret://<store>/<mount>/<path>#<field>` in a KV v2 secrets engine, options: `version=<n>`, `mount=<mount>` (when the mount contains `/`).
32
+ # Authentication by `token` or `approle`: secrets are the value, or `@file:<path>` (`Bootstrap`).
33
+ PARAMETERS = %i[url token approle namespace ca_file allowed_prefixes].freeze
34
+ REFERENCE_OPTIONS = %w[version mount].freeze
35
+ # Options of a listing of a folder (`list_folder`)
36
+ LIST_OPTIONS = %w[mount].freeze
37
+
38
+ class << self
39
+ # Errors of gem `vault`: code and errors of response, or address of server
40
+ def error_reason(error)
41
+ case error
42
+ when ::Vault::HTTPError then "Vault: #{error.code}#{" #{error.errors.join(', ')}" unless error.errors.empty?}"
43
+ when ::Vault::HTTPConnectionError then "Vault: cannot connect to #{error.address}: #{error.original_exception.message}"
44
+ end
45
+ end
46
+ end
47
+
48
+ # `Vault::Client` (gem `vault` 0.20.1) gives the options of `JSON.parse` as a positional Hash,
49
+ # refused by gem `json` 3 (`ArgumentError`), with option `create_additions` (removed, `false` by default in gem `json` 2):
50
+ # responses are parsed here with keyword arguments.
51
+ class Client < ::Vault::Client
52
+ PARSE_OPTIONS = {max_nesting: false, symbolize_names: true}.freeze
53
+
54
+ def success(response)
55
+ return super unless json?(response)
56
+ JSON.parse(response.body, **PARSE_OPTIONS)
57
+ end
58
+
59
+ # Same errors as `Vault::Client#error`
60
+ def error(response)
61
+ return super unless json?(response)
62
+ klass =
63
+ if response.body.include?('missing client token') then ::Vault::HTTPClientError
64
+ else
65
+ case response
66
+ when Net::HTTPPreconditionFailed then raise ::Vault::MissingRequiredStateError
67
+ when Net::HTTPClientError then ::Vault::HTTPClientError
68
+ when Net::HTTPServerError then ::Vault::HTTPServerError
69
+ else ::Vault::HTTPError
70
+ end
71
+ end
72
+ errors = JSON.parse(response.body, **PARSE_OPTIONS)[:errors] rescue nil
73
+ raise klass.new(address, response, errors.is_a?(Array) ? errors : [response.body])
74
+ end
75
+
76
+ private
77
+
78
+ def json?(response)
79
+ !response.body.nil? && (response.content_type || '').include?('json')
80
+ end
81
+ end
82
+ private_constant :Client
83
+
84
+ # One of `token` or `approle` authenticates the client
85
+ # @param url [String] URL of Vault server
86
+ # @param token [String, nil] Token: the value, or `@file:<path>` (e.g. written by Vault Agent), read again when it is modified
87
+ # @param approle [Hash, nil] AppRole: `role_id`, `secret_id` (the value, or `@file:<path>`), `mount` (default: `approle`)
88
+ # @param namespace [String, nil] Namespace (Vault Enterprise)
89
+ # @param ca_file [String, nil] File with CA certificates of server
90
+ # @param allowed_prefixes [Array<String>, nil] Allowed prefixes of `<mount>/<path>` of references, `nil`: all
91
+ def initialize(url:, token: nil, approle: nil, namespace: nil, ca_file: nil, allowed_prefixes: nil)
92
+ super()
93
+ Aspera.assert_type(url, String, type: ParameterError) { 'vault url' }
94
+ Aspera.assert([token, approle].compact.size.eql?(1), type: ParameterError) { 'vault: one of token or approle is required' }
95
+ Aspera.assert(allowed_prefixes.nil? || (allowed_prefixes.is_a?(Array) && allowed_prefixes.all?(String)), type: ParameterError) { 'vault allowed_prefixes: list of strings' }
96
+ @allowed_prefixes = allowed_prefixes
97
+ @token = Bootstrap.new('token', token) unless token.nil?
98
+ # Modification time of file of token when read, `nil` before
99
+ @token_mtime = nil
100
+ @approle = approle.nil? ? nil : approle_parameters(approle)
101
+ # Monotonic time of renewal of the AppRole token, `nil` before login
102
+ @renew_at = nil
103
+ @auth_lock = Mutex.new
104
+ options = {address: url}
105
+ options[:token] = @token.value unless @token.nil? || @token.file?
106
+ options[:namespace] = namespace unless namespace.nil?
107
+ options[:ssl_ca_cert] = ca_file unless ca_file.nil?
108
+ @client = Client.new(options)
109
+ end
110
+
111
+ def info
112
+ {
113
+ url: @client.address,
114
+ namespace: @client.namespace,
115
+ auth: auth_method
116
+ }.compact
117
+ end
118
+
119
+ def all
120
+ metadata_path = STORE_PATH.sub('/data/', '/metadata/')
121
+ labels = call { |client| client.logical.list(metadata_path) }
122
+ return labels.filter_map do |label|
123
+ # KV v2: deleted secrets are still listed in metadata
124
+ get(label: label, exception: false)&.merge(label: label)
125
+ end
126
+ end
127
+
128
+ # Set a secret
129
+ # @param options [Hash] with keys :label, :username, :password, :url, :description
130
+ def set(options)
131
+ validate_set(options)
132
+ label = options.fetch(:label)
133
+ assert_new_label(label)
134
+ data = {
135
+ username: options[:username],
136
+ password: options[:password],
137
+ url: options[:url],
138
+ description: options[:description]
139
+ }.compact
140
+ call { |client| client.logical.write(path(label), data: data) }
141
+ end
142
+
143
+ def get(label:, exception: true)
144
+ # KV v2: a deleted secret may have metadata but no data
145
+ data = call { |client| client.logical.read(path(label)) }&.data&.[](:data)
146
+ if data.nil?
147
+ raise "Secret '#{label}' not found" if exception
148
+ return
149
+ end
150
+ return data
151
+ end
152
+
153
+ def delete(label:)
154
+ path = path(label)
155
+ call { |client| client.logical.delete(path) }
156
+ end
157
+
158
+ # Read a secret of a KV v2 secrets engine
159
+ # @param path [String] Path of secret in the mount
160
+ # @param mount [String] Mount of the secrets engine
161
+ # @param version [Integer, nil] Version of secret, `nil` for current
162
+ # @return [Hash, nil] `data` (fields with Symbol keys), `version`, `created_time` (`Time`), or `nil` if not found (or deleted)
163
+ def read(path:, mount: 'secret', version: nil)
164
+ secret = call { |client| client.kv(mount).read(path, version) }
165
+ return if secret.nil? || secret.data.nil?
166
+ metadata = secret.metadata || {}
167
+ return {
168
+ data: secret.data,
169
+ version: metadata[:version],
170
+ created_time: metadata[:created_time].nil? ? nil : Time.iso8601(metadata[:created_time])
171
+ }
172
+ end
173
+
174
+ def check_reference(ref)
175
+ super
176
+ mount, path = mount_and_path(ref)
177
+ full_path = "#{mount}/#{path}"
178
+ raise Error, "invalid path: #{full_path}" if full_path.split('/', -1).any? { |s| INVALID_SEGMENTS.include?(s) }
179
+ raise Error, "path not allowed: #{full_path}" unless allowed?(full_path)
180
+ reference_version(ref)
181
+ nil
182
+ end
183
+
184
+ # The field is required
185
+ def read_reference(ref)
186
+ mount, path = mount_and_path(ref)
187
+ secret = read(path: path, mount: mount, version: reference_version(ref))
188
+ raise Error, 'not found' if secret.nil?
189
+ {value: field_value(secret[:data], ref), version: secret[:version], updated_time: secret[:created_time]}
190
+ end
191
+
192
+ # Secrets and folders of a folder of a KV v2 secrets engine that references can read (`allowed_prefixes`): list of its metadata,
193
+ # with capability `list` on `<mount>/metadata/<path>`
194
+ # @param path [String] Folder `<mount>[/<path>]`, `/` at the end optional
195
+ # @param options [Hash<String,String>] `mount`, when it contains `/`: the folder starts with it
196
+ # @return [Array<Hash>] `name`, `folder` (`true` for a folder), by name
197
+ # @raise [Error] Invalid folder, not allowed, or Vault error
198
+ def list_folder(path, options = {})
199
+ unsupported = options.keys - LIST_OPTIONS
200
+ raise Error, "unsupported options: #{unsupported.join(', ')}" unless unsupported.empty?
201
+ folder = path.delete_suffix('/')
202
+ raise Error, 'path shall be <mount>[/<path>]' if folder.empty?
203
+ raise Error, "invalid path: #{path}" if folder.split('/', -1).any? { |s| INVALID_SEGMENTS.include?(s) }
204
+ mount = options['mount'] || folder.split('/').first
205
+ raise Error, "path shall start with mount #{mount}/" unless folder.eql?(mount) || folder.start_with?("#{mount}/")
206
+ raise Error, "path not allowed: #{folder}/" unless reachable?("#{folder}/")
207
+ sub = folder.delete_prefix(mount).delete_prefix('/')
208
+ keys = call { |client| client.kv(mount).list(sub.empty? ? '' : "#{sub}/") }
209
+ entries = keys.filter_map do |key|
210
+ full = "#{folder}/#{key}"
211
+ is_folder = key.end_with?('/')
212
+ {name: key.delete_suffix('/'), folder: is_folder} if is_folder ? reachable?(full) : allowed?(full)
213
+ end
214
+ entries.sort_by { |entry| [entry[:name], entry[:folder] ? 0 : 1] }
215
+ end
216
+
217
+ # Field names of a secret, version and time of creation: the secret is read, its values are not returned
218
+ def read_fields(ref)
219
+ mount, path = mount_and_path(ref)
220
+ secret = read(path: path, mount: mount, version: reference_version(ref))
221
+ raise Error, 'not found' if secret.nil?
222
+ {fields: field_names(secret[:data]), version: secret[:version], updated_time: secret[:created_time]}
223
+ end
224
+
225
+ # Write a new secret (KV v2, check-and-set: refuses if the path already exists)
226
+ # @param ref [Reference] Reference without field
227
+ # @param fields [Hash<String,String>] Field name → value
228
+ # @return [Hash] `version` (`Integer`), `updated_time` (`Time`)
229
+ # @raise [Error] Path already exists ("secret exists: <mount>/<path>"), or Vault error
230
+ def write_reference(ref, fields)
231
+ mount, path = mount_and_path(ref)
232
+ symbol_fields = fields.transform_keys(&:to_sym)
233
+ response = call do |client|
234
+ client.logical.write(
235
+ "#{mount}/data/#{path}",
236
+ options: {cas: 0},
237
+ data: symbol_fields
238
+ )
239
+ end
240
+ metadata = response.data[:metadata] || response.data
241
+ version = metadata[:version]
242
+ created_time = metadata[:created_time]
243
+ {version: version, updated_time: created_time.nil? ? nil : Time.iso8601(created_time)}
244
+ rescue ::Vault::HTTPClientError => e
245
+ raise Error, "secret exists: #{mount}/#{path}" if e.code == 400
246
+ raise
247
+ end
248
+
249
+ private
250
+
251
+ def path(label)
252
+ "#{STORE_PATH}#{label}"
253
+ end
254
+
255
+ # @return [Array<String>] Mount and path in mount of reference
256
+ def mount_and_path(ref)
257
+ mount = ref.options['mount']
258
+ if mount.nil?
259
+ mount, path = ref.path.split('/', 2)
260
+ raise Error, "path shall be <mount>/<path>: #{ref.path}" if path.nil?
261
+ return [mount, path]
262
+ end
263
+ raise Error, "path shall start with mount #{mount}/" unless ref.path.start_with?("#{mount}/")
264
+ [mount, ref.path.delete_prefix("#{mount}/")]
265
+ end
266
+
267
+ # @return [Integer, nil] Version of secret of reference, `nil` for current
268
+ def reference_version(ref)
269
+ value = ref.options['version']
270
+ return if value.nil?
271
+ raise Error, "version shall be a positive integer: #{value}" unless value.match?(/\A[1-9][0-9]*\z/)
272
+ value.to_i
273
+ end
274
+
275
+ # A prefix ending with `/` is a folder, else it is a path or folder
276
+ # @param full_path [String] `<mount>/<path>`
277
+ def allowed?(full_path)
278
+ return true if @allowed_prefixes.nil?
279
+ @allowed_prefixes.any? do |prefix|
280
+ prefix.end_with?('/') ? full_path.start_with?(prefix) : full_path.eql?(prefix) || full_path.start_with?("#{prefix}/")
281
+ end
282
+ end
283
+
284
+ # @param folder [String] `<mount>/<path>/`
285
+ # @return [Boolean] `true` if the folder holds paths that references can read: in an allowed prefix, or above one
286
+ def reachable?(folder)
287
+ allowed?(folder) || @allowed_prefixes.any? { |prefix| prefix.start_with?(folder) }
288
+ end
289
+
290
+ # @return [String] Authentication method
291
+ def auth_method
292
+ return 'approle' unless @approle.nil?
293
+ @token.file? ? 'token file' : 'token'
294
+ end
295
+
296
+ # @param approle [Hash] AppRole parameters, keys String or Symbol
297
+ # @return [Hash] AppRole parameters with Symbol keys, and default mount
298
+ def approle_parameters(approle)
299
+ Aspera.assert_type(approle, Hash, type: ParameterError) { 'vault approle' }
300
+ approle = approle.transform_keys(&:to_sym)
301
+ unsupported = approle.keys - APPROLE_KEYS
302
+ Aspera.assert(unsupported.empty?, type: ParameterError) { "vault approle: unsupported keys: #{unsupported.join(', ')}, use #{APPROLE_KEYS.join(', ')}" }
303
+ Aspera.assert_type(approle[:role_id], String, type: ParameterError) { 'vault approle role_id' }
304
+ approle[:secret_id] = Bootstrap.new('approle secret_id', approle[:secret_id]) unless approle[:secret_id].nil?
305
+ {mount: APPROLE_MOUNT}.merge(approle)
306
+ end
307
+
308
+ # Call the API with an authenticated client.
309
+ # On `403`, the call is done again once if credentials could change: new token in file, new AppRole login.
310
+ # @yieldparam client [Vault::Client]
311
+ # @return [Object] Value of block
312
+ def call
313
+ retried = false
314
+ begin
315
+ @auth_lock.synchronize { authenticate }
316
+ yield(@client)
317
+ rescue ::Vault::HTTPClientError => e
318
+ raise if retried || !e.code.eql?(HTTP_FORBIDDEN)
319
+ retried = true
320
+ raise unless @auth_lock.synchronize { reauthenticate }
321
+ Log.log.debug { "vault: #{e.code}, call again with new credentials (#{auth_method})" }
322
+ retry
323
+ end
324
+ end
325
+
326
+ # Set the token of client, if needed
327
+ def authenticate
328
+ if @token&.file?
329
+ mtime = @token.mtime
330
+ return if mtime.eql?(@token_mtime)
331
+ @client.token = @token.value
332
+ @token_mtime = mtime
333
+ elsif !@approle.nil?
334
+ if @renew_at.nil?
335
+ login
336
+ elsif monotonic_time >= @renew_at
337
+ renew
338
+ end
339
+ end
340
+ end
341
+
342
+ # After a `403`
343
+ # @return [Boolean] `true` if credentials changed
344
+ def reauthenticate
345
+ if @token&.file?
346
+ previous = @client.token
347
+ @token_mtime = nil
348
+ authenticate
349
+ return !@client.token.eql?(previous)
350
+ elsif !@approle.nil?
351
+ login
352
+ return true
353
+ end
354
+ false
355
+ end
356
+
357
+ # AppRole login: sets the token of client
358
+ def login
359
+ secret_id = @approle[:secret_id]&.value
360
+ @renew_at = nil
361
+ # Sets the token of client
362
+ auth = @client.auth.approle(@approle[:role_id], secret_id, mount: @approle[:mount]).auth
363
+ Secret::Hider.instance.add_secret_values(auth.client_token)
364
+ schedule_renewal(auth)
365
+ Log.log.debug { "vault: AppRole login, lease: #{auth.lease_duration} s" }
366
+ end
367
+
368
+ # Renew the AppRole token, or new login if renewal fails
369
+ def renew
370
+ schedule_renewal(@client.auth_token.renew_self.auth)
371
+ Log.log.debug('vault: token renewed')
372
+ rescue ::Vault::HTTPError => e
373
+ Log.log.debug { "vault: token renewal failed (#{e.code}), new login" }
374
+ login
375
+ end
376
+
377
+ # @param auth [Vault::SecretAuth] Authentication information with lease
378
+ def schedule_renewal(auth)
379
+ # A lease duration of 0 never expires
380
+ lease = auth.lease_duration.to_i
381
+ @renew_at = lease.positive? ? monotonic_time + (lease * RENEW_RATIO) : Float::INFINITY
382
+ end
383
+
384
+ def monotonic_time
385
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
386
+ end
387
+ end
388
+ end
389
+ end
390
+ end
@@ -0,0 +1,141 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../../assert'
7
+ require_relative '../reference'
8
+ require_relative 'base'
9
+ require 'logger'
10
+ begin
11
+ require 'keeper_secrets_manager'
12
+ rescue LoadError
13
+ raise Aspera::Secret::Error, 'type keeper requires gem keeper_secrets_manager: gem install keeper_secrets_manager'
14
+ end
15
+
16
+ module Aspera
17
+ module Secret
18
+ module Store
19
+ # Manage secrets in Keeper Secrets Manager (Keeper Vault), using the Ruby SDK
20
+ # https://docs.keeper.io/keeperpam/secrets-manager/overview
21
+ #
22
+ # A secret is a record of the Keeper vault shared with the application: its label is its title (or its UID).
23
+ # Reference: `secret://<store>/<title or uid>#<field>`, field `password` by default.
24
+ # Fields are those of the record: standard fields by type (`login`, `password`, `url`...), custom fields by label, and `notes`.
25
+ # `username` is an alias of `login`.
26
+ class Keeper < Base
27
+ # `config`: configuration of the client device (base64 or JSON), the value, or `@file:<path>` (`Bootstrap`)
28
+ # `folder_uid`: shared folder where `set` creates records (not needed to read)
29
+ PARAMETERS = %i[config folder_uid].freeze
30
+
31
+ class << self
32
+ def arguments(params)
33
+ params.merge(config: Bootstrap.new('config', params[:config]))
34
+ end
35
+ end
36
+
37
+ # @param config [String, Bootstrap] Configuration of the client device, from the one-time access token (`ksm profile export`)
38
+ # @param folder_uid [String, nil] UID of the shared folder where `set` creates records
39
+ def initialize(config:, folder_uid: nil)
40
+ super()
41
+ Aspera.assert_type(folder_uid, String, type: ParameterError) { 'keeper folder_uid' } unless folder_uid.nil?
42
+ config = Bootstrap.new('config', config) unless config.is_a?(Bootstrap)
43
+ @folder_uid = folder_uid
44
+ # The SDK logs on STDOUT by default: do not mix with results
45
+ @client = ::KeeperSecretsManager.from_config(config.value, logger: ::Logger.new($stderr))
46
+ end
47
+
48
+ def info
49
+ {
50
+ hostname: @client.hostname,
51
+ folder_uid: @folder_uid
52
+ }.compact
53
+ end
54
+
55
+ def all
56
+ @client.get_secrets.map { |record| secret(record) }
57
+ end
58
+
59
+ def ids
60
+ @client.get_secrets.map { |record| {id: record.uid, label: record.title} }
61
+ end
62
+
63
+ # Create a record of type `login` in the shared folder `folder_uid`
64
+ # @param options [Hash] with keys :label, :username, :password, :url, :description
65
+ def set(options)
66
+ validate_set(options)
67
+ Aspera.assert(!@folder_uid.nil?, type: ParameterError) { 'keeper: parameter folder_uid is required to create a secret' }
68
+ assert_new_label(options[:label])
69
+ fields = {login: :username, password: :password, url: :url}.filter_map do |type, key|
70
+ {type: type.to_s, value: [options[key]]} if options[key]
71
+ end
72
+ record = {type: 'login', title: options[:label], fields: fields}
73
+ record[:notes] = options[:description] if options[:description]
74
+ @client.create_secret(record, ::KeeperSecretsManager::Dto::CreateOptions.new(folder_uid: @folder_uid))
75
+ nil
76
+ end
77
+
78
+ def get(label:, id: nil, exception: true)
79
+ record = find(id || label, exception: exception)
80
+ return if record.nil?
81
+ secret(record)
82
+ end
83
+
84
+ def delete(label:, id: nil)
85
+ @client.delete_secret(find(id || label).uid)
86
+ nil
87
+ end
88
+
89
+ # The record is found by title or UID, the field is `password` by default
90
+ def read_reference(ref)
91
+ record = find(ref.path)
92
+ {value: field_value(record_values(record), ref, default: DEFAULT_FIELD), version: record.revision, updated_time: nil}
93
+ end
94
+
95
+ def read_fields(ref)
96
+ record = find(ref.path)
97
+ {fields: field_names(record_values(record).except('title')), version: record.revision, updated_time: nil}
98
+ end
99
+
100
+ private
101
+
102
+ # @param identifier [String] UID or title of record
103
+ # @return [KeeperSecretsManager::Dto::KeeperRecord, nil] Record, `nil` if not found and not `exception`
104
+ def find(identifier, exception: true)
105
+ records = @client.get_secrets
106
+ matches = records.select { |record| record.uid.eql?(identifier) }
107
+ matches = records.select { |record| record.title.eql?(identifier) } if matches.empty?
108
+ raise "Multiple secrets found with label '#{identifier}': use id to disambiguate" if matches.size > 1
109
+ raise "Secret '#{identifier}' not found" if matches.empty? && exception
110
+ matches.first
111
+ end
112
+
113
+ # @return [Hash<String,Object>] Name of field → first value: `title`, `notes`, standard fields by type, custom fields by label, `username` for `login`
114
+ def record_values(record)
115
+ values = {'title' => record.title, 'notes' => record.notes}
116
+ (record.fields + record.custom).each do |field|
117
+ name = field['label'].to_s.empty? ? field['type'] : field['label']
118
+ value = field['value']
119
+ value = value.first if value.is_a?(Array)
120
+ values[name] = value unless value.nil? || values.key?(name)
121
+ end
122
+ values['username'] = values['login'] if values.key?('login')
123
+ values
124
+ end
125
+
126
+ # @return [Hash] Secret as in other stores
127
+ def secret(record)
128
+ values = record_values(record)
129
+ {
130
+ label: record.title,
131
+ id: record.uid,
132
+ username: values['login'],
133
+ password: values['password'],
134
+ url: values['url'],
135
+ description: values['notes']
136
+ }.select { |_, value| value.is_a?(String) && !value.empty? }
137
+ end
138
+ end
139
+ end
140
+ end
141
+ end