aspera-cli 4.28.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +217 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +122 -26
  10. data/SECURITY.md +84 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1895 -632
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +29 -22
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +95 -67
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +370 -0
  41. data/lib/aspera/ascp/log_locations.rb +163 -0
  42. data/lib/aspera/ascp/management.rb +16 -8
  43. data/lib/aspera/ascp/session_errors.rb +217 -0
  44. data/lib/aspera/assert.rb +4 -0
  45. data/lib/aspera/cli/ascp_actions.rb +12 -19
  46. data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
  47. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  48. data/lib/aspera/cli/bootstrapper.rb +38 -33
  49. data/lib/aspera/cli/command_line.rb +63 -8
  50. data/lib/aspera/cli/command_registry.rb +9 -1
  51. data/lib/aspera/cli/command_spec.rb +47 -7
  52. data/lib/aspera/cli/completion/ascli.bash +19 -5
  53. data/lib/aspera/cli/completion/ascli.fish +13 -2
  54. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  55. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  56. data/lib/aspera/cli/completion_actions.rb +303 -69
  57. data/lib/aspera/cli/context.rb +9 -4
  58. data/lib/aspera/cli/deprecation.rb +5 -2
  59. data/lib/aspera/cli/error.rb +6 -0
  60. data/lib/aspera/cli/extended_value.rb +66 -31
  61. data/lib/aspera/cli/formatter.rb +43 -30
  62. data/lib/aspera/cli/gem_checker.rb +9 -6
  63. data/lib/aspera/cli/hints.rb +22 -10
  64. data/lib/aspera/cli/http.rb +38 -10
  65. data/lib/aspera/cli/info.rb +12 -7
  66. data/lib/aspera/cli/mailer.rb +6 -3
  67. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  68. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  69. data/lib/aspera/cli/mcp/tool.rb +20 -17
  70. data/lib/aspera/cli/option_declarator.rb +10 -4
  71. data/lib/aspera/cli/option_registry.rb +6 -3
  72. data/lib/aspera/cli/option_types.rb +17 -2
  73. data/lib/aspera/cli/option_value.rb +77 -18
  74. data/lib/aspera/cli/options.schema.yaml +641 -80
  75. data/lib/aspera/cli/parser.rb +71 -35
  76. data/lib/aspera/cli/plugins/alee.rb +7 -4
  77. data/lib/aspera/cli/plugins/aoc.rb +248 -37
  78. data/lib/aspera/cli/plugins/ats.rb +10 -9
  79. data/lib/aspera/cli/plugins/base.rb +50 -18
  80. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  81. data/lib/aspera/cli/plugins/config.rb +210 -104
  82. data/lib/aspera/cli/plugins/console.rb +6 -3
  83. data/lib/aspera/cli/plugins/cos.rb +8 -4
  84. data/lib/aspera/cli/plugins/factory.rb +21 -7
  85. data/lib/aspera/cli/plugins/faspex5.rb +31 -26
  86. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  87. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  88. data/lib/aspera/cli/plugins/mcp.rb +42 -22
  89. data/lib/aspera/cli/plugins/node.rb +39 -38
  90. data/lib/aspera/cli/plugins/oauth.rb +23 -6
  91. data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
  92. data/lib/aspera/cli/plugins/preview.rb +72 -45
  93. data/lib/aspera/cli/plugins/server.rb +66 -14
  94. data/lib/aspera/cli/plugins/shares.rb +87 -18
  95. data/lib/aspera/cli/plugins/transferd.rb +228 -0
  96. data/lib/aspera/cli/poll.rb +236 -0
  97. data/lib/aspera/cli/preset_actions.rb +144 -30
  98. data/lib/aspera/cli/preset_manager.rb +113 -35
  99. data/lib/aspera/cli/prompt.rb +157 -9
  100. data/lib/aspera/cli/result.rb +12 -8
  101. data/lib/aspera/cli/runner.rb +89 -48
  102. data/lib/aspera/cli/secret_finder.rb +4 -1
  103. data/lib/aspera/cli/security.rb +89 -9
  104. data/lib/aspera/cli/special_values.rb +4 -1
  105. data/lib/aspera/cli/sync_actions.rb +15 -10
  106. data/lib/aspera/cli/table_fitter.rb +3 -0
  107. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  108. data/lib/aspera/cli/transfer_actions.rb +11 -8
  109. data/lib/aspera/cli/transfer_agent.rb +43 -20
  110. data/lib/aspera/cli/transfer_progress.rb +14 -45
  111. data/lib/aspera/cli/vault_actions.rb +45 -0
  112. data/lib/aspera/cli/vault_manager.rb +188 -42
  113. data/lib/aspera/cli/version.rb +4 -1
  114. data/lib/aspera/cli/web_secret.rb +12 -10
  115. data/lib/aspera/cli/wizard.rb +42 -39
  116. data/lib/aspera/command_line_builder.rb +9 -5
  117. data/lib/aspera/command_line_converter.rb +4 -1
  118. data/lib/aspera/coverage.rb +103 -33
  119. data/lib/aspera/data_repository.rb +4 -1
  120. data/lib/aspera/dot_container.rb +4 -1
  121. data/lib/aspera/environment.rb +23 -12
  122. data/lib/aspera/exec_spec.rb +3 -0
  123. data/lib/aspera/faspex_gw.rb +13 -9
  124. data/lib/aspera/faspex_postproc.rb +15 -12
  125. data/lib/aspera/formatter_interface.rb +4 -1
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +20 -5
  129. data/lib/aspera/inflate.rb +43 -0
  130. data/lib/aspera/json_rpc/client.rb +6 -5
  131. data/lib/aspera/json_rpc/version.rb +3 -0
  132. data/lib/aspera/line_logger.rb +4 -1
  133. data/lib/aspera/link_header.rb +4 -1
  134. data/lib/aspera/log.rb +18 -5
  135. data/lib/aspera/markdown.rb +5 -2
  136. data/lib/aspera/mime.rb +3 -0
  137. data/lib/aspera/nagios.rb +6 -4
  138. data/lib/aspera/node_emulator.rb +15 -13
  139. data/lib/aspera/oauth/client/base.rb +124 -73
  140. data/lib/aspera/oauth/client/boot.rb +7 -4
  141. data/lib/aspera/oauth/client/factory.rb +41 -18
  142. data/lib/aspera/oauth/client/generic.rb +5 -1
  143. data/lib/aspera/oauth/client/json_credentials.rb +7 -3
  144. data/lib/aspera/oauth/client/jwt.rb +6 -3
  145. data/lib/aspera/oauth/client/shares_web.rb +94 -0
  146. data/lib/aspera/oauth/client/url_json.rb +7 -3
  147. data/lib/aspera/oauth/client/web.rb +40 -13
  148. data/lib/aspera/oauth/server/pkce.rb +6 -5
  149. data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +16 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +17 -2
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +3 -0
  165. data/lib/aspera/rest/aspera_errors.rb +4 -1
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +45 -34
  168. data/lib/aspera/rest/error_analyzer.rb +5 -2
  169. data/lib/aspera/rest/list.rb +8 -4
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +8 -5
  172. data/lib/aspera/rest.rb +9 -6
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  175. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  176. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  177. data/lib/aspera/schema/async_tables.yaml +11 -10
  178. data/lib/aspera/schema/documentation.rb +8 -2
  179. data/lib/aspera/schema/reader.rb +7 -0
  180. data/lib/aspera/schema/registry.rb +44 -28
  181. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  182. data/lib/aspera/schema/validator.rb +4 -1
  183. data/lib/aspera/sealed_hash.rb +150 -0
  184. data/lib/aspera/secret/hider.rb +213 -0
  185. data/lib/aspera/secret/reference.rb +150 -0
  186. data/lib/aspera/secret/resolver.rb +212 -0
  187. data/lib/aspera/secret/store/base.rb +153 -0
  188. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  189. data/lib/aspera/secret/store/factory.rb +103 -0
  190. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  191. data/lib/aspera/secret/store/keeper.rb +141 -0
  192. data/lib/aspera/secret/store/macos_security.rb +284 -0
  193. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  194. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  195. data/lib/aspera/secret/store/one_password_cli.rb +112 -0
  196. data/lib/aspera/secret/store/secret_service.rb +125 -0
  197. data/lib/aspera/secret/strength.rb +119 -0
  198. data/lib/aspera/servlet_helpers.rb +5 -2
  199. data/lib/aspera/ssh.rb +10 -7
  200. data/lib/aspera/ssl.rb +5 -2
  201. data/lib/aspera/string_ext.rb +3 -0
  202. data/lib/aspera/sync/args.schema.yaml +3 -0
  203. data/lib/aspera/sync/conf.schema.yaml +9 -4
  204. data/lib/aspera/sync/database.rb +3 -0
  205. data/lib/aspera/sync/operations.rb +17 -15
  206. data/lib/aspera/temp_file_manager.rb +8 -4
  207. data/lib/aspera/timer_limiter.rb +3 -0
  208. data/lib/aspera/transfer/error.rb +4 -1
  209. data/lib/aspera/transfer/faux_file.rb +4 -1
  210. data/lib/aspera/transfer/parameters.rb +17 -14
  211. data/lib/aspera/transfer/result.rb +23 -0
  212. data/lib/aspera/transfer/resumer.rb +6 -4
  213. data/lib/aspera/transfer/spec.rb +9 -22
  214. data/lib/aspera/units/decimal.rb +67 -0
  215. data/lib/aspera/units/duration.rb +109 -0
  216. data/lib/aspera/uri_reader.rb +64 -9
  217. data/lib/aspera/web_auth/index.html.erb +6 -2
  218. data/lib/aspera/web_auth/paste.rb +109 -0
  219. data/lib/aspera/web_auth/receiver.rb +50 -0
  220. data/lib/aspera/web_auth/relay.rb +76 -0
  221. data/lib/aspera/web_auth/reply.html.erb +150 -0
  222. data/lib/aspera/web_auth/server.rb +47 -10
  223. data/lib/aspera/web_page.rb +7 -2
  224. data/lib/aspera/web_server_simple.rb +6 -3
  225. data/lib/aspera/yaml.rb +5 -1
  226. data/llms-install.md +88 -0
  227. data/llms.txt +87 -0
  228. data/skills/ascli/SKILL.md +94 -0
  229. data.tar.gz.sig +0 -0
  230. metadata +77 -20
  231. metadata.gz.sig +0 -0
  232. data/lib/aspera/duration.rb +0 -42
  233. data/lib/aspera/keychain/base.rb +0 -30
  234. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  235. data/lib/aspera/keychain/factory.rb +0 -58
  236. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  237. data/lib/aspera/keychain/macos_security.rb +0 -277
  238. data/lib/aspera/keychain/one_password_api.rb +0 -91
  239. data/lib/aspera/keychain/one_password_base.rb +0 -39
  240. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  241. data/lib/aspera/secret_hider.rb +0 -196
  242. data/lib/aspera/secret_strength.rb +0 -114
@@ -0,0 +1,212 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'reference'
7
+ require_relative 'store/factory'
8
+ require_relative 'hider'
9
+
10
+ module Aspera
11
+ module Secret
12
+ # Reads the values of references from named stores, with a cache.
13
+ #
14
+ # Configuration (e.g. from a YAML file):
15
+ # - `cache_ttl`: time in seconds a value is kept in memory (default: 60, 0: no cache)
16
+ # - `stores`: name of store → parameters of store, with `type` (see `Store::Factory`)
17
+ # - `writable: true`: allow `write` to this store (default: false)
18
+ #
19
+ # The type and the names of parameters are checked by `new`, a store is created when first used:
20
+ # a store not used (e.g. its secret is not available) is not an error.
21
+ # Bootstrap secrets of stores (`password` of type `file`, `token` and `secret_id` of type `vault`): the value, or `@file:<path>` (`Bootstrap`).
22
+ # Error messages give the reference, never a value.
23
+ class Resolver
24
+ DEFAULT_CACHE_TTL = 60
25
+ CONFIG_KEYS = %w[cache_ttl stores].freeze
26
+ STORE_NAME = /\A[A-Za-z0-9_.-]+\z/
27
+ WRITABLE_KEY = 'writable'
28
+ private_constant :CONFIG_KEYS, :STORE_NAME, :WRITABLE_KEY
29
+
30
+ # @param config [Hash] Configuration, keys String or Symbol
31
+ # @raise [Error] Invalid configuration
32
+ def initialize(config)
33
+ raise Error, 'secret references configuration: expecting a Hash' unless config.is_a?(Hash)
34
+ config = config.transform_keys(&:to_s)
35
+ unsupported = config.keys - CONFIG_KEYS
36
+ raise Error, "secret references configuration: unsupported keys: #{unsupported.join(', ')}" unless unsupported.empty?
37
+ @cache_ttl = config.fetch('cache_ttl', DEFAULT_CACHE_TTL)
38
+ raise Error, 'secret references configuration: cache_ttl: expecting a number >= 0' unless @cache_ttl.is_a?(Numeric) && !@cache_ttl.negative?
39
+ stores = config.fetch('stores', nil) || {}
40
+ raise Error, 'secret references configuration: stores: expecting a Hash' unless stores.is_a?(Hash)
41
+ # Name of store → {params: ..., writable: bool}, checked
42
+ @store_configs = stores.to_h { |name, params| [name.to_s, check_store(name.to_s, params)] }.freeze
43
+ # Name of store → store instance, created when first used
44
+ @stores = {}
45
+ # Key of reference → [value, monotonic time of expiry]
46
+ @cache = {}
47
+ @lock = Mutex.new
48
+ end
49
+
50
+ # @return [Array<String>] Names of stores
51
+ def store_names
52
+ @store_configs.keys
53
+ end
54
+
55
+ # @param name [String] Name of store
56
+ # @return [Boolean] `true` if the store is declared writable in the configuration
57
+ def writable?(name)
58
+ @store_configs.fetch(name) { raise Error, "unknown store #{name}" }[:writable]
59
+ end
60
+
61
+ # @param name [String] Name of store
62
+ # @return [Store::Base] Store, created when first used, e.g. to manage its secrets by label
63
+ # @raise [Error] Unknown store, or invalid parameter
64
+ def store(name)
65
+ config = @store_configs.fetch(name) { raise Error, "unknown store #{name}" }
66
+ @lock.synchronize { @stores[name] ||= create_store(name, config[:params]) }
67
+ end
68
+
69
+ # Checks that a reference is valid and reachable, without reading its value
70
+ # @param text [String] Reference
71
+ # @raise [Error] Invalid reference, not allowed, or not readable
72
+ def check(text)
73
+ ref = Secret.parse_reference(text)
74
+ access(ref) { nil }
75
+ end
76
+
77
+ # @param text [String] Reference
78
+ # @return [String] Value of secret, from cache if not expired
79
+ # @raise [Error] Invalid reference, not allowed, or not readable
80
+ def resolve(text)
81
+ ref = Secret.parse_reference(text)
82
+ access(ref) do |store|
83
+ key = ref.to_h.except(:text)
84
+ cached(key) || cache(key, ref.store, store.read_reference(ref)[:value])
85
+ end
86
+ end
87
+
88
+ # Checks that a secret can be read, without its value
89
+ # @param text [String] Reference
90
+ # @return [Hash] `store`, `version` (`nil` if store has no version), `updated_time` (`Time` or `nil`)
91
+ # @raise [Error] Invalid reference, not allowed, or not readable
92
+ def metadata(text)
93
+ ref = Secret.parse_reference(text)
94
+ access(ref) { |store| {store: ref.store}.merge(store.read_reference(ref).except(:value)) }
95
+ end
96
+
97
+ # Field names of a secret, without values, e.g. to choose the field of a reference
98
+ # @param text [String] Reference, its field is not used
99
+ # @return [Hash] `store`, `fields` (`Array<String>`), `version` (`nil` if store has no version), `updated_time` (`Time` or `nil`)
100
+ # @raise [Error] Invalid reference, not allowed, or not readable
101
+ def fields(text)
102
+ ref = Secret.parse_reference(text)
103
+ access(ref) { |store| {store: ref.store}.merge(store.read_fields(ref)) }
104
+ end
105
+
106
+ # Secrets and folders of a store that references can read, without reading the secrets, e.g. to build references
107
+ # @param name [String] Name of store
108
+ # @param path [String] Folder, empty for a store without folders (labels)
109
+ # @param options [Hash<String,String>] Options of references that select the folder (`mount` of type `vault`)
110
+ # @return [Array<Hash>] `name` (last segment of path, or label), `folder` (`true` for a folder), by name
111
+ # @raise [Error] Unknown store, folder not allowed, listing not supported by the type, or store not readable
112
+ def list(name, path = '', options = {})
113
+ found = store(name)
114
+ found.list_folder(path, options)
115
+ rescue StandardError => e
116
+ raise Error, "secret store #{name}#{", folder #{path}" unless path.empty?}: #{reason(e, found)}"
117
+ end
118
+
119
+ # Write a new secret to a writable store
120
+ # @param text [String] Reference (without field: the field names are given in `fields`)
121
+ # @param fields [Hash<String,String>] Field name → value
122
+ # @return [Hash] `version` (`Integer` or `nil`), `updated_time` (`Time` or `nil`)
123
+ # @raise [Error] Store not writable, reference has a field, unknown store, not allowed, or write failed
124
+ def write(text, fields)
125
+ ref = Secret.parse_reference(text)
126
+ raise Error, "secret reference #{ref.text}: field not allowed when writing (use field names in fields)" unless ref.field.nil?
127
+ raise Error, "secret reference #{ref.text}: store #{ref.store} is not writable" unless writable?(ref.store)
128
+ result = access(ref) { |store| store.write_reference(ref, fields) }
129
+ # Invalidate cache for this reference so it is readable at once
130
+ key = ref.to_h.except(:text)
131
+ @lock.synchronize { @cache.delete(key) }
132
+ result
133
+ end
134
+
135
+ # Remove values from cache, e.g. after a rotation
136
+ # @param store_name [String, nil] Name of store, `nil`: all stores
137
+ def invalidate(store_name = nil)
138
+ @lock.synchronize do
139
+ store_name.nil? ? @cache.clear : @cache.delete_if { |_, entry| entry[:store].eql?(store_name) }
140
+ end
141
+ nil
142
+ end
143
+
144
+ private
145
+
146
+ # @return [Hash] `{params:, writable:}`, parameters checked
147
+ def check_store(name, params)
148
+ raise Error, "secret references configuration: store #{name}: invalid name" unless STORE_NAME.match?(name)
149
+ raise Error, "secret references configuration: store #{name}: parameters: expecting a Hash" unless params.is_a?(Hash)
150
+ params = params.transform_keys(&:to_s)
151
+ writable = params.delete(WRITABLE_KEY) || false
152
+ raise Error, "secret references configuration: store #{name}: writable: expecting true or false" unless [true, false].include?(writable)
153
+ Store::Factory.check(params)
154
+ {params: params, writable: writable}
155
+ rescue Aspera::Secret::Error => e
156
+ # Re-raise with store context if the message doesn't already include it
157
+ raise e if e.message.start_with?('secret references configuration:')
158
+ raise Error, "secret references configuration: store #{name}: #{e.message}"
159
+ rescue StandardError => e
160
+ raise Error, "secret references configuration: store #{name}: #{e.message}"
161
+ end
162
+
163
+ # @return [Store::Base] New store from its parameters
164
+ def create_store(name, params)
165
+ Store::Factory.create(params)
166
+ rescue StandardError => e
167
+ raise Error, "store #{name}: #{e.message}"
168
+ end
169
+
170
+ # Checks reference and calls block with its store
171
+ # @yieldparam store [Store::Base]
172
+ # @return [Object] Value of block
173
+ def access(ref)
174
+ found = store(ref.store)
175
+ found.check_reference(ref)
176
+ yield(found)
177
+ rescue StandardError => e
178
+ raise Error, "secret reference #{ref.text}: #{reason(e, found)}"
179
+ end
180
+
181
+ # @param error [StandardError] Error
182
+ # @param store [Store::Base, nil] Store that raised the error
183
+ # @return [String] Reason of failure, without secret
184
+ def reason(error, store)
185
+ return error.message if error.is_a?(Error)
186
+ store&.class&.error_reason(error) || Secret::Hider.instance.hide_secrets_in_string("#{error.class}: #{error.message}", all: true)
187
+ end
188
+
189
+ # @return [String, nil] Value from cache, `nil` if absent or expired
190
+ def cached(key)
191
+ @lock.synchronize do
192
+ entry = @cache[key]
193
+ next if entry.nil?
194
+ next entry[:value] if monotonic_time < entry[:expiry]
195
+ @cache.delete(key)
196
+ nil
197
+ end
198
+ end
199
+
200
+ # @return [String] Value, after cache and declaration to `Secret::Hider`
201
+ def cache(key, store_name, value)
202
+ Secret::Hider.instance.add_secret_values(value)
203
+ @lock.synchronize { @cache[key] = {value: value, store: store_name, expiry: monotonic_time + @cache_ttl} } if @cache_ttl.positive?
204
+ value
205
+ end
206
+
207
+ def monotonic_time
208
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
209
+ end
210
+ end
211
+ end
212
+ end
@@ -0,0 +1,153 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../../assert'
7
+ require_relative '../reference'
8
+
9
+ module Aspera
10
+ module Secret
11
+ module Store
12
+ # Base class of stores of secrets, created by `Factory` from parameters: management by label, and read by reference (`Secret::Resolver`).
13
+ # For references, errors are raised with the reason only, `Resolver` adds the reference.
14
+ class Base
15
+ CONTENT_KEYS = %i[label username password url description].freeze
16
+ # Parameters of store: keyword arguments of `new`.
17
+ # Secrets to access the store: the value, or `@file:<path>` (`Secret::Bootstrap`), when the type supports it.
18
+ PARAMETERS = [].freeze
19
+ # Options accepted in references
20
+ REFERENCE_OPTIONS = [].freeze
21
+ # Field of secret when the reference has none
22
+ DEFAULT_FIELD = 'password'
23
+ private_constant :DEFAULT_FIELD
24
+
25
+ class << self
26
+ # Checks the parameters of store
27
+ # @param params [Hash] Parameters, Symbol keys
28
+ # @raise [Error] Unsupported parameter
29
+ def check_parameters(params)
30
+ unsupported = params.keys - self::PARAMETERS
31
+ raise Error, "unsupported keys: #{unsupported.join(', ')}" unless unsupported.empty?
32
+ end
33
+
34
+ # @param params [Hash] Parameters of store, checked, Symbol keys
35
+ # @return [Hash] Keyword arguments of `new`
36
+ # @raise [Error] Invalid parameter
37
+ def arguments(params)
38
+ params
39
+ end
40
+
41
+ # @param _error [StandardError] Error raised by the store
42
+ # @return [String, nil] Reason of failure without secret, `nil` if the error is not specific to the type
43
+ def error_reason(_error)
44
+ nil
45
+ end
46
+ end
47
+
48
+ # Returns a lightweight list: one Hash per secret with :label (and :id if the backend has one).
49
+ # Override in subclasses that can provide this cheaply without fetching full item details.
50
+ def ids
51
+ all.map { |s| s.slice(:id, :label).compact }
52
+ end
53
+
54
+ def validate_set(options)
55
+ Aspera.assert_type(options, Hash) { 'options' }
56
+ unsupported = options.keys - CONTENT_KEYS
57
+ Aspera.assert(unsupported.empty?) { "unsupported options: #{unsupported}, use #{CONTENT_KEYS.join(', ')}" }
58
+ options.each_pair do |k, v|
59
+ Aspera.assert_type(v, String) { k.to_s }
60
+ end
61
+ Aspera.assert(options.key?(:label), 'label is required')
62
+ end
63
+
64
+ # Labels are unique: `set` does not overwrite an existing secret
65
+ def assert_new_label(label)
66
+ Aspera.assert(get(label: label, exception: false).nil?) { "secret #{label} already exist, delete first" }
67
+ end
68
+
69
+ # Checks before access: options of reference, paths allowed by configuration
70
+ # @param ref [Reference]
71
+ # @raise [Error] Reference not allowed
72
+ def check_reference(ref)
73
+ unsupported = ref.options.keys - self.class::REFERENCE_OPTIONS
74
+ raise Error, "unsupported options: #{unsupported.join(', ')}" unless unsupported.empty?
75
+ end
76
+
77
+ # Read a secret by reference: the path is the label, the field is `password` by default
78
+ # @param ref [Reference]
79
+ # @return [Hash] `value` (`String`), `version` (`nil` if store has no version), `updated_time` (`Time` or `nil`)
80
+ def read_reference(ref)
81
+ entry = get(label: ref.path, exception: false)
82
+ raise Error, "no label: #{ref.path}" if entry.nil?
83
+ {value: field_value(entry, ref, default: DEFAULT_FIELD), version: nil, updated_time: nil}
84
+ end
85
+
86
+ # Secrets of the store that references can read, without their values: labels, no folder
87
+ # @param path [String] Folder: empty, labels have no folder
88
+ # @param options [Hash<String,String>] Options of references
89
+ # @return [Array<Hash>] `name` (label), `folder` (`false`), by name
90
+ # @raise [Error] Unsupported option, or a folder
91
+ def list_folder(path, options = {})
92
+ check_list_options(options)
93
+ raise Error, "labels have no folder: #{path}" unless path.empty?
94
+ ids.map { |item| {name: item[:label].to_s, folder: false} }.sort_by { |entry| entry[:name] }
95
+ end
96
+
97
+ # Field names of a secret by reference, without values: the path is the label
98
+ # @param ref [Reference] Reference, its field is not used
99
+ # @return [Hash] `fields` (`Array<String>`, fields that a reference can read), `version` (`nil`), `updated_time` (`nil`)
100
+ # @raise [Error] No secret
101
+ def read_fields(ref)
102
+ entry = get(label: ref.path, exception: false)
103
+ raise Error, "no label: #{ref.path}" if entry.nil?
104
+ {fields: field_names(entry.reject { |name, _| name.to_s.eql?('label') }), version: nil, updated_time: nil}
105
+ end
106
+
107
+ # Write a new secret by reference.
108
+ # Subclasses that support writing must override this method.
109
+ # @param _ref [Reference] Reference (field is nil)
110
+ # @param _fields [Hash<String,String>] Field name → value
111
+ # @return [Hash] `version` (`Integer` or `nil`), `updated_time` (`Time` or `nil`)
112
+ # @raise [Error] Always — writing is not supported by this store type
113
+ def write_reference(_ref, _fields)
114
+ raise Error, 'writing not supported by store type'
115
+ end
116
+
117
+ private
118
+
119
+ # @param options [Hash<String,String>] Options of a listing: those of references
120
+ # @raise [Error] Unsupported option
121
+ def check_list_options(options)
122
+ unsupported = options.keys - self.class::REFERENCE_OPTIONS
123
+ raise Error, "unsupported options: #{unsupported.join(', ')}" unless unsupported.empty?
124
+ end
125
+
126
+ # @param data [Hash] Fields of secret, keys String or Symbol
127
+ # @return [Array<String>] Names of the fields that `field_value` reads: String not empty, number or Boolean
128
+ def field_names(data)
129
+ data.filter_map do |name, value|
130
+ readable = value.is_a?(String) ? !value.empty? : [Numeric, TrueClass, FalseClass].any? { |type| value.is_a?(type) }
131
+ name.to_s if readable
132
+ end
133
+ end
134
+
135
+ # @param data [Hash] Fields of secret, keys String or Symbol
136
+ # @param ref [Reference] Reference with field
137
+ # @param default [String, nil] Field when reference has none, `nil`: field required
138
+ # @return [String] Value of field
139
+ def field_value(data, ref, default: nil)
140
+ field = ref.field || default
141
+ raise Error, 'field required: #<field>' if field.nil?
142
+ value = data.key?(field.to_sym) ? data[field.to_sym] : data[field]
143
+ case value
144
+ when nil then raise Error, "no field: #{field}"
145
+ when String then value
146
+ when Numeric, true, false then value.to_s
147
+ else raise Error, "field #{field} is not a string"
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,329 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../../hash_ext'
7
+ require_relative '../../environment'
8
+ require_relative '../../log'
9
+ require_relative '../../assert'
10
+ require_relative 'base'
11
+ require_relative '../../yaml'
12
+ require 'openssl'
13
+ require 'yaml'
14
+
15
+ module Aspera
16
+ module Secret
17
+ module Store
18
+ # Manage secrets in a simple Hash
19
+ class EncryptedHash < Base
20
+ LEGACY_CIPHER_NAME = 'aes-256-cbc'
21
+ DEFAULT_CIPHER_NAME = 'aes-256-cbc'
22
+ FILE_TYPE = 'encrypted_hash_vault'
23
+ FILE_KEYS = %w[version type cipher data].sort.freeze
24
+ FILE_KEYS_KDF = (%w[kdf] + FILE_KEYS).sort.freeze
25
+ # PBKDF2 parameters for new vaults
26
+ KDF_ITERATIONS = 200_000
27
+ KDF_DIGEST = 'SHA256'
28
+ KDF_SALT_BYTES = 16
29
+ private_constant :LEGACY_CIPHER_NAME, :DEFAULT_CIPHER_NAME, :FILE_TYPE, :FILE_KEYS, :FILE_KEYS_KDF,
30
+ :KDF_ITERATIONS, :KDF_DIGEST, :KDF_SALT_BYTES
31
+ PARAMETERS = %i[file password].freeze
32
+
33
+ class << self
34
+ # `password` is the value, or `@file:<path>`
35
+ def arguments(params)
36
+ params.merge(password: Bootstrap.new('password', params[:password]))
37
+ end
38
+ end
39
+
40
+ # The vault file is read when used, and again when it, or the file of the password, is modified.
41
+ # It is created by the first `set`.
42
+ # @param file [String] Vault file
43
+ # @param password [String, Bootstrap] Password of vault file: the value, or a `Bootstrap`
44
+ def initialize(file:, password:)
45
+ super()
46
+ Aspera.assert_type(file, String) { 'path to vault file' }
47
+ @path = file
48
+ @password = password.is_a?(Bootstrap) ? password : Bootstrap.new('password', password)
49
+ # Modification times of vault file (`nil` if absent) and file of password, when read, `nil` before
50
+ @mtimes = nil
51
+ @lock = Mutex.new
52
+ Log.dump(:vault_file, File.expand_path(@path))
53
+ end
54
+
55
+ def info
56
+ return {
57
+ file: @path
58
+ }
59
+ end
60
+
61
+ def all
62
+ result = []
63
+ secrets.each do |label, values|
64
+ normal = values.symbolize_keys
65
+ normal[:label] = label
66
+ CONTENT_KEYS.each { |k| normal[k] = '' unless normal.key?(k) }
67
+ result.push(normal)
68
+ end
69
+ return result
70
+ end
71
+
72
+ # set a secret
73
+ # @param options [Hash] with keys :label, :username, :password, :url, :description
74
+ def set(options)
75
+ validate_set(options)
76
+ label = options.delete(:label)
77
+ with_lock do
78
+ assert_new_label(label)
79
+ secrets_unlocked[label] = options.symbolize_keys
80
+ save
81
+ end
82
+ end
83
+
84
+ def get(label:, exception: true)
85
+ Aspera.assert(secrets.key?(label)) { "Label not found: #{label}" } if exception
86
+ result = secrets[label].clone
87
+ result[:label] = label if result.is_a?(Hash)
88
+ return result
89
+ end
90
+
91
+ def delete(label:)
92
+ with_lock do
93
+ Aspera.assert(secrets_unlocked.key?(label)) { "Label not found: #{label}" }
94
+ secrets_unlocked.delete(label)
95
+ save
96
+ end
97
+ end
98
+
99
+ def change_password(password)
100
+ secrets
101
+ # Generate new KDF params so a password change also re-salts the vault
102
+ # (and migrates a legacy vault to PBKDF2)
103
+ @legacy_key = false
104
+ @kdf_params = nil
105
+ @cipher = cipher(password)
106
+ @password = Bootstrap.new('password', password)
107
+ save
108
+ end
109
+
110
+ # Modification time of vault file
111
+ def read_reference(ref)
112
+ raise Error, "no vault file: #{@path}" unless File.exist?(@path)
113
+ super.merge(updated_time: @mtimes&.first)
114
+ end
115
+
116
+ # Modification time of vault file
117
+ def read_fields(ref)
118
+ raise Error, "no vault file: #{@path}" unless File.exist?(@path)
119
+ super.merge(updated_time: @mtimes&.first)
120
+ end
121
+
122
+ # Write a new secret by reference: label = path of reference, fields within CONTENT_KEYS
123
+ # @param ref [Reference] Reference without field
124
+ # @param fields [Hash<String,String>] Field name → value
125
+ # @return [Hash] `version: nil`, `updated_time: nil`
126
+ def write_reference(ref, fields)
127
+ symbol_fields = fields.transform_keys(&:to_sym)
128
+ unsupported = symbol_fields.keys - CONTENT_KEYS
129
+ raise Error, "unsupported fields: #{unsupported.join(', ')}, use #{CONTENT_KEYS.join(', ')}" unless unsupported.empty?
130
+ with_lock do
131
+ assert_new_label(ref.path)
132
+ secrets_unlocked[ref.path] = symbol_fields
133
+ save
134
+ end
135
+ {version: nil, updated_time: nil}
136
+ end
137
+
138
+ private
139
+
140
+ # @return [Hash] Secrets: label → fields (caller must hold file lock)
141
+ def secrets_unlocked
142
+ @all_secrets
143
+ end
144
+
145
+ # @return [Hash] Secrets: label → fields, reloaded if file changed
146
+ def secrets
147
+ @lock.synchronize do
148
+ mtimes = current_mtimes
149
+ unless mtimes.eql?(@mtimes)
150
+ load_file(@password.value)
151
+ @mtimes = mtimes
152
+ end
153
+ end
154
+ @all_secrets
155
+ rescue SystemCallError => e
156
+ raise Error, e.message
157
+ end
158
+
159
+ # Acquire a file lock and reload secrets, then yield, then save.
160
+ # Prevents two concurrent processes (web + ascli) from overwriting each other.
161
+ def with_lock
162
+ lock_path = "#{@path}.lock"
163
+ File.open(lock_path, File::CREAT | File::RDWR, 0o600) do |lf|
164
+ # Same owner as the vault file, so that it can be locked by whoever owns the vault, not only by who created the lock file
165
+ give_owner(lock_path, File.stat(@path)) if File.exist?(@path)
166
+ lf.flock(File::LOCK_EX)
167
+ # Reload inside the lock so we see any write done by another process
168
+ load_file(@password.value)
169
+ @mtimes = current_mtimes
170
+ yield
171
+ end
172
+ rescue SystemCallError => e
173
+ raise Error, e.message
174
+ end
175
+
176
+ # @return [Array<Time, nil>] Modification times of vault file (`nil` if absent) and file of password (`nil` for a value)
177
+ def current_mtimes
178
+ [File.exist?(@path) ? File.mtime(@path) : nil, @password.mtime]
179
+ end
180
+
181
+ # Read and decrypt vault file, if it exists
182
+ # @param password [String] Password of vault file
183
+ def load_file(password)
184
+ @all_secrets = {}
185
+ @cipher_name = DEFAULT_CIPHER_NAME
186
+ @kdf_params = nil
187
+ # true when the vault was created by v4.26.0 (password padded with zeros, no KDF)
188
+ @legacy_key = false
189
+ vault_encrypted_data = nil
190
+ if File.exist?(@path)
191
+ vault_file = File.read(@path)
192
+ if vault_file.start_with?('---')
193
+ vault_info = Yaml.safe_load(vault_file)
194
+ Aspera.assert_type(vault_info, Hash) { "Invalid vault file: #{@path}" }
195
+ sorted_keys = vault_info.keys.sort
196
+ Aspera.assert(sorted_keys == FILE_KEYS || sorted_keys == FILE_KEYS_KDF) { "Invalid vault file: #{@path}: #{sorted_keys}" }
197
+ @cipher_name = vault_info['cipher']
198
+ @kdf_params = vault_info['kdf']
199
+ # vault created before PBKDF2 was introduced (v4.26.0 format: no kdf field)
200
+ @legacy_key = @kdf_params.nil?
201
+ vault_encrypted_data = vault_info['data']
202
+ else
203
+ # legacy vault file (binary, pre-YAML format)
204
+ @cipher_name = LEGACY_CIPHER_NAME
205
+ @legacy_key = true
206
+ vault_encrypted_data = File.read(@path, mode: 'rb')
207
+ end
208
+ end
209
+ # setting password also creates the cipher
210
+ @cipher = cipher(password)
211
+ @all_secrets = decrypt(vault_encrypted_data) unless vault_encrypted_data.nil?
212
+ end
213
+
214
+ # @param data [String] Encrypted secrets
215
+ # @return [Hash] Secrets
216
+ # @raise [Error] Wrong password
217
+ def decrypt(data)
218
+ Yaml.safe_load(@cipher.decrypt(data)) || {}
219
+ rescue OpenSSL::Cipher::CipherError, Psych::Exception
220
+ # Psych: rarely, a wrong key gives valid padding, then garbage
221
+ raise Error, "cannot decrypt #{@path} (password?)"
222
+ end
223
+
224
+ # Minimal cipher wrapper using OpenSSL, compatible with the SymmetricEncryption
225
+ # binary format: a 6-byte header "@EnC\x00\x00" followed by AES ciphertext
226
+ # produced with a zero IV (encoding: :none, no random IV in header).
227
+ class OsslCipher
228
+ HEADER = "@EnC\x00\x00".b.freeze
229
+ HEADER_SIZE = HEADER.bytesize
230
+
231
+ def initialize(cipher_name, key)
232
+ @cipher_name = cipher_name
233
+ @key = key
234
+ end
235
+
236
+ def encrypt(plaintext)
237
+ c = OpenSSL::Cipher.new(@cipher_name)
238
+ c.encrypt
239
+ c.key = @key
240
+ c.iv = "\x00".b * c.iv_len
241
+ HEADER + c.update(plaintext.b) + c.final
242
+ end
243
+
244
+ def decrypt(ciphertext)
245
+ body = ciphertext.b[HEADER_SIZE..]
246
+ c = OpenSSL::Cipher.new(@cipher_name)
247
+ c.decrypt
248
+ c.key = @key
249
+ c.iv = "\x00".b * c.iv_len
250
+ (c.update(body) + c.final).force_encoding(Encoding::UTF_8)
251
+ end
252
+ end
253
+
254
+ # Derive an AES key from `new_password`.
255
+ # @legacy_key is true for vaults created by v4.26.0 or earlier (no kdf field):
256
+ # key = password truncated/zero-padded to key_bytes (same as symmetric-encryption did).
257
+ # Otherwise PBKDF2 is used. When @kdf_params is nil (new vault or password change)
258
+ # a fresh salt is generated; when already set (existing vault) it is reused.
259
+ def cipher(new_password)
260
+ key_bytes = @cipher_name.split('-')[1].to_i / Environment::BITS_PER_BYTE
261
+ key =
262
+ if @legacy_key
263
+ # Replicate the v4.26.0 key derivation: password + NUL padding, truncated to key_bytes
264
+ (new_password + ("\x00" * key_bytes))[0, key_bytes]
265
+ else
266
+ if @kdf_params.nil?
267
+ # New vault or password change: generate a fresh salt
268
+ salt = OpenSSL::Random.random_bytes(KDF_SALT_BYTES)
269
+ @kdf_params = {
270
+ 'algo' => 'PBKDF2',
271
+ 'digest' => KDF_DIGEST,
272
+ 'iterations' => KDF_ITERATIONS,
273
+ 'salt' => [salt].pack('m0') # base64, no newlines
274
+ }
275
+ end
276
+ salt = @kdf_params['salt'].unpack1('m0')
277
+ OpenSSL::KDF.pbkdf2_hmac(
278
+ new_password,
279
+ salt: salt,
280
+ iterations: @kdf_params['iterations'],
281
+ length: key_bytes,
282
+ hash: @kdf_params['digest']
283
+ )
284
+ end
285
+ OsslCipher.new(@cipher_name, key)
286
+ end
287
+
288
+ # Give the owner and group of a stat to a file, if allowed (only root can give a file to another user)
289
+ # @param path [String] File to change
290
+ # @param stat [File::Stat] Owner and group to set
291
+ def give_owner(path, stat)
292
+ return if File.stat(path).uid.eql?(stat.uid) && File.stat(path).gid.eql?(stat.gid)
293
+ File.chown(stat.uid, stat.gid, path)
294
+ rescue SystemCallError, NotImplementedError => e
295
+ Log.log.debug { "cannot set owner of #{path}: #{e.message}" }
296
+ end
297
+
298
+ # Save current data to file atomically (temp file + rename) so a concurrent reader
299
+ # never sees a partial file.
300
+ # The file that is replaced keeps its owner, group and mode.
301
+ def save
302
+ vault_info = {
303
+ 'version' => '1.1.0',
304
+ 'type' => FILE_TYPE,
305
+ 'cipher' => @cipher_name,
306
+ 'kdf' => @kdf_params,
307
+ 'data' => @cipher.encrypt(YAML.dump(@all_secrets))
308
+ }
309
+ content = YAML.dump(vault_info)
310
+ tmp = "#{@path}.tmp#{::Process.pid}"
311
+ begin
312
+ File.write(tmp, content, perm: 0o600)
313
+ if File.exist?(@path)
314
+ stat = File.stat(@path)
315
+ give_owner(tmp, stat)
316
+ File.chmod(stat.mode & 0o7777, tmp)
317
+ end
318
+ File.rename(tmp, @path)
319
+ rescue SystemCallError
320
+ File.unlink(tmp) rescue nil
321
+ raise
322
+ end
323
+ # Secrets in memory are those of the file
324
+ @mtimes = current_mtimes
325
+ end
326
+ end
327
+ end
328
+ end
329
+ end