aspera-cli 4.28.0 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +143 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1633 -609
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +20 -20
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +71 -65
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +16 -8
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +12 -19
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -33
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +6 -1
  50. data/lib/aspera/cli/command_spec.rb +34 -7
  51. data/lib/aspera/cli/completion/ascli.bash +19 -5
  52. data/lib/aspera/cli/completion/ascli.fish +13 -2
  53. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  54. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  55. data/lib/aspera/cli/completion_actions.rb +303 -69
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +3 -0
  59. data/lib/aspera/cli/extended_value.rb +61 -27
  60. data/lib/aspera/cli/formatter.rb +29 -22
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -10
  63. data/lib/aspera/cli/http.rb +12 -9
  64. data/lib/aspera/cli/info.rb +12 -7
  65. data/lib/aspera/cli/mailer.rb +6 -3
  66. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  67. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  68. data/lib/aspera/cli/mcp/tool.rb +15 -14
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +65 -12
  73. data/lib/aspera/cli/options.schema.yaml +487 -63
  74. data/lib/aspera/cli/parser.rb +56 -32
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +244 -35
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +17 -12
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +198 -103
  81. data/lib/aspera/cli/plugins/console.rb +6 -3
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +21 -7
  84. data/lib/aspera/cli/plugins/faspex5.rb +28 -25
  85. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  86. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +28 -22
  88. data/lib/aspera/cli/plugins/node.rb +31 -30
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +63 -11
  93. data/lib/aspera/cli/plugins/shares.rb +86 -16
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +144 -30
  96. data/lib/aspera/cli/preset_manager.rb +52 -18
  97. data/lib/aspera/cli/prompt.rb +4 -1
  98. data/lib/aspera/cli/result.rb +12 -8
  99. data/lib/aspera/cli/runner.rb +69 -46
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +79 -9
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +3 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +29 -20
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +188 -42
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.rb +12 -10
  113. data/lib/aspera/cli/wizard.rb +42 -39
  114. data/lib/aspera/command_line_builder.rb +9 -5
  115. data/lib/aspera/command_line_converter.rb +4 -1
  116. data/lib/aspera/coverage.rb +103 -33
  117. data/lib/aspera/data_repository.rb +4 -1
  118. data/lib/aspera/dot_container.rb +4 -1
  119. data/lib/aspera/duration.rb +5 -2
  120. data/lib/aspera/environment.rb +21 -12
  121. data/lib/aspera/exec_spec.rb +3 -0
  122. data/lib/aspera/faspex_gw.rb +13 -9
  123. data/lib/aspera/faspex_postproc.rb +15 -12
  124. data/lib/aspera/formatter_interface.rb +4 -1
  125. data/lib/aspera/graphql.rb +5 -1
  126. data/lib/aspera/hash_ext.rb +3 -0
  127. data/lib/aspera/id_generator.rb +4 -2
  128. data/lib/aspera/json_rpc/client.rb +6 -5
  129. data/lib/aspera/json_rpc/version.rb +3 -0
  130. data/lib/aspera/line_logger.rb +4 -1
  131. data/lib/aspera/link_header.rb +4 -1
  132. data/lib/aspera/log.rb +18 -5
  133. data/lib/aspera/markdown.rb +5 -2
  134. data/lib/aspera/mime.rb +3 -0
  135. data/lib/aspera/nagios.rb +6 -4
  136. data/lib/aspera/node_emulator.rb +15 -13
  137. data/lib/aspera/oauth/client/base.rb +91 -70
  138. data/lib/aspera/oauth/client/boot.rb +7 -4
  139. data/lib/aspera/oauth/client/factory.rb +6 -3
  140. data/lib/aspera/oauth/client/generic.rb +5 -1
  141. data/lib/aspera/oauth/client/json_credentials.rb +4 -1
  142. data/lib/aspera/oauth/client/jwt.rb +6 -3
  143. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  144. data/lib/aspera/oauth/client/url_json.rb +4 -1
  145. data/lib/aspera/oauth/client/web.rb +39 -12
  146. data/lib/aspera/oauth/server/pkce.rb +6 -5
  147. data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
  148. data/lib/aspera/persistency_action_once.rb +5 -3
  149. data/lib/aspera/persistency_folder.rb +6 -3
  150. data/lib/aspera/preview/file_types.rb +6 -3
  151. data/lib/aspera/preview/generator.rb +9 -5
  152. data/lib/aspera/preview/options.schema.yaml +3 -3
  153. data/lib/aspera/preview/terminal.rb +8 -4
  154. data/lib/aspera/preview/utils.rb +6 -4
  155. data/lib/aspera/products/connect.rb +5 -2
  156. data/lib/aspera/products/desktop.rb +4 -1
  157. data/lib/aspera/products/other.rb +4 -2
  158. data/lib/aspera/products/transferd.rb +6 -2
  159. data/lib/aspera/products/transferd_service.rb +610 -0
  160. data/lib/aspera/proxy_auto_config.js +2 -2
  161. data/lib/aspera/proxy_auto_config.rb +5 -2
  162. data/lib/aspera/rainbow.rb +3 -0
  163. data/lib/aspera/rest/aspera_errors.rb +4 -1
  164. data/lib/aspera/rest/call_error.rb +3 -0
  165. data/lib/aspera/rest/client.rb +22 -29
  166. data/lib/aspera/rest/error_analyzer.rb +5 -2
  167. data/lib/aspera/rest/list.rb +8 -4
  168. data/lib/aspera/rest/parameters.rb +3 -0
  169. data/lib/aspera/rest/util.rb +8 -5
  170. data/lib/aspera/rest.rb +9 -6
  171. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  172. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  173. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  174. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  175. data/lib/aspera/schema/async_tables.yaml +11 -10
  176. data/lib/aspera/schema/documentation.rb +8 -2
  177. data/lib/aspera/schema/reader.rb +7 -0
  178. data/lib/aspera/schema/registry.rb +43 -28
  179. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  180. data/lib/aspera/schema/validator.rb +4 -1
  181. data/lib/aspera/sealed_hash.rb +166 -0
  182. data/lib/aspera/secret/hider.rb +201 -0
  183. data/lib/aspera/secret/reference.rb +150 -0
  184. data/lib/aspera/secret/resolver.rb +212 -0
  185. data/lib/aspera/secret/store/base.rb +153 -0
  186. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  187. data/lib/aspera/secret/store/factory.rb +103 -0
  188. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  189. data/lib/aspera/secret/store/keeper.rb +141 -0
  190. data/lib/aspera/secret/store/macos_security.rb +284 -0
  191. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  192. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  193. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  194. data/lib/aspera/secret/store/secret_service.rb +125 -0
  195. data/lib/aspera/secret/strength.rb +119 -0
  196. data/lib/aspera/servlet_helpers.rb +5 -2
  197. data/lib/aspera/ssh.rb +10 -7
  198. data/lib/aspera/ssl.rb +5 -2
  199. data/lib/aspera/string_ext.rb +3 -0
  200. data/lib/aspera/sync/args.schema.yaml +3 -0
  201. data/lib/aspera/sync/conf.schema.yaml +9 -4
  202. data/lib/aspera/sync/database.rb +3 -0
  203. data/lib/aspera/sync/operations.rb +16 -14
  204. data/lib/aspera/temp_file_manager.rb +8 -4
  205. data/lib/aspera/timer_limiter.rb +3 -0
  206. data/lib/aspera/transfer/error.rb +4 -1
  207. data/lib/aspera/transfer/faux_file.rb +4 -1
  208. data/lib/aspera/transfer/parameters.rb +17 -14
  209. data/lib/aspera/transfer/result.rb +3 -0
  210. data/lib/aspera/transfer/resumer.rb +6 -4
  211. data/lib/aspera/transfer/spec.rb +9 -22
  212. data/lib/aspera/transfer/units.rb +75 -0
  213. data/lib/aspera/uri_reader.rb +57 -7
  214. data/lib/aspera/web_auth/index.html.erb +6 -2
  215. data/lib/aspera/web_auth/paste.rb +109 -0
  216. data/lib/aspera/web_auth/receiver.rb +50 -0
  217. data/lib/aspera/web_auth/relay.rb +76 -0
  218. data/lib/aspera/web_auth/reply.html.erb +150 -0
  219. data/lib/aspera/web_auth/server.rb +47 -10
  220. data/lib/aspera/web_page.rb +7 -2
  221. data/lib/aspera/web_server_simple.rb +6 -3
  222. data/lib/aspera/yaml.rb +5 -1
  223. data/llms.txt +81 -0
  224. data/skills/ascli/SKILL.md +88 -0
  225. data.tar.gz.sig +0 -0
  226. metadata +72 -19
  227. metadata.gz.sig +0 -0
  228. data/lib/aspera/keychain/base.rb +0 -30
  229. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  230. data/lib/aspera/keychain/factory.rb +0 -58
  231. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  232. data/lib/aspera/keychain/macos_security.rb +0 -277
  233. data/lib/aspera/keychain/one_password_api.rb +0 -91
  234. data/lib/aspera/keychain/one_password_base.rb +0 -39
  235. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  236. data/lib/aspera/secret_hider.rb +0 -196
  237. data/lib/aspera/secret_strength.rb +0 -114
@@ -1,196 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # cspell:ignore FILEPASS
4
- require 'logger'
5
- require 'set'
6
- require 'singleton'
7
- require 'json'
8
- require 'uri'
9
-
10
- module Aspera
11
- # remove secret from logs and output
12
- #
13
- # Secrets are identified:
14
- # - by key name, in structures (`secret?`, `hidden_copy`) and in strings with known patterns (`hide_secrets_in_string`)
15
- # - by value, when declared where they come from (`add_secret_values`, `add_secrets_in`): option values, tokens, ...
16
- #
17
- # @!method self.instance
18
- # Returns the singleton instance of SecretHider
19
- # @return [SecretHider] the singleton instance
20
- class SecretHider
21
- include Singleton
22
-
23
- # display string for hidden secrets (see `Environment::UNICODE_TO_ASCII`)
24
- HIDDEN_PASSWORD = '🔑'
25
- # env vars for ascp with secrets
26
- ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
27
- # keys in hash that contain secrets
28
- KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
29
- HTTP_SECRETS = %w[Authorization].freeze
30
- ALL_SECRETS = (ASCP_ENV_SECRETS + KEY_SECRETS + HTTP_SECRETS).freeze
31
- NON_ENV_SECRETS = (KEY_SECRETS + HTTP_SECRETS).freeze
32
- KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/, /public_key$/, /^token_type$/].freeze
33
- # min length of hidden secrets, use `+` or `{n,}`
34
- SECRET_LENGTH = '{5,}'
35
- # min length of a declared secret value to be hidden inside a string: shorter ones are hidden only when they are the whole string,
36
- # else common words would be hidden (e.g. password `aspera` in URLs), or optionally as a word (`short_in_text`)
37
- VALUE_IN_STRING_LENGTH = 8
38
- # regex that define named captures :begin and :end
39
- REGEX_LOG_REPLACES = [
40
- # private key values (place first)
41
- /(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)\n*/,
42
- # CLI manager get/set options
43
- /(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/i,
44
- # env var ascp exec
45
- /(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)[^ \n]+(?<end> |$)/,
46
- # rendered JSON or Ruby (quoted key or legacy symbol key: `"k":"v"`, `"k" => "v"`, `:k=>"v"`)
47
- /(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>? *(?:=>|:) *")(?:[^"\\]|\\.)+(?<end>")/i,
48
- # rendered Ruby 3.4+ symbol key: `k: "v"`
49
- /(?<begin>\b\w*(?:#{ALL_SECRETS.join('|')})\w*: *")(?:[^"\\]|\\.)+(?<end>")/i,
50
- # rendered Ruby Struct or object: `k="v"`, `@k="v"`
51
- /(?<begin>@?\b\w*(?:#{ALL_SECRETS.join('|')})\w*=")(?:[^"\\]|\\.)+(?<end>")/i,
52
- # query parameter of URL: `?token=v` or `&client_secret=v`
53
- /(?<begin>[?&][^=&\s"]*(?:#{KEY_SECRETS.join('|')})[^=&\s"]*=)[^&\s"#]+(?<end>)/i,
54
- # logged data
55
- /(?<begin>(?:#{NON_ENV_SECRETS.join('|')})[ =:]+)[^ "\n]#{SECRET_LENGTH}(?<end>$)/i,
56
- # cred in http dump
57
- /(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
58
- ].freeze
59
- private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :SECRET_LENGTH, :VALUE_IN_STRING_LENGTH, :REGEX_LOG_REPLACES
60
- attr_accessor :log_secrets
61
-
62
- # @return [Proc] new log formatter that hides secrets
63
- def log_formatter(original_formatter)
64
- original_formatter ||= Logger::Formatter.new
65
- # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
66
- # Hiding is applied on the formatted line, so that non-String messages (e.g. Exception) are covered too
67
- return lambda do |severity, date_time, program_name, msg|
68
- line = original_formatter.call(severity, date_time, program_name, msg)
69
- line = hide_secrets_in_string(line, all: true) if line.is_a?(String) && !@log_secrets
70
- line
71
- end
72
- end
73
-
74
- # Replace secrets in a string with the hidden password placeholder
75
- # @param value [String] Input string possibly containing secrets
76
- # @param all [Boolean] `false`: only declared values and private keys, `true`: also all known secret patterns
77
- # @return [String] String with secrets replaced by placeholder
78
- def hide_secrets_in_string(value, all: false)
79
- marker = hidden_marker
80
- value = hide_values_in_string(value, marker)
81
- (all ? REGEX_LOG_REPLACES : REGEX_LOG_REPLACES.first(1)).each do |reg_ex|
82
- value = value.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{marker}#{Regexp.last_match(:end)}" }
83
- end
84
- return value
85
- end
86
-
87
- # @param keyword [String, Symbol] Key name to check
88
- # @param value [String] Associated value (must be a String to be a secret)
89
- # @return [Boolean] true if the key denotes a secret
90
- def secret?(keyword, value)
91
- keyword = keyword.to_s if keyword.is_a?(Symbol)
92
- # only Strings can be secrets, not booleans, or hash, arrays
93
- return false unless keyword.is_a?(String) && value.is_a?(String)
94
- return true if @additional_keys.include?(keyword)
95
- keyword = keyword.downcase
96
- # those are not secrets
97
- return false if KEY_FALSE_POSITIVES.any? { |f| f.match?(keyword) }
98
- # check if keyword (name) contains an element that designate it as a secret
99
- ALL_SECRETS.any? { |kw| keyword.include?(kw.downcase) }
100
- end
101
-
102
- # Hides secrets in a structure: by key name, and declared values, also in key names (e.g. a secret given as name of a parameter).
103
- # Containers are copied, not modified: the structure may be shared (e.g. configuration, transfer spec).
104
- # @param obj [Object] Hash, Array, Struct, String, or other value (returned as is)
105
- # @param short_in_text [Boolean] `true`: a declared value shorter than 8 characters is also hidden inside a text, as a word
106
- # (not next to a letter or digit), e.g. a short password in an error message of `mcp server`
107
- # @return [Object] Copy of obj with secrets replaced by placeholder
108
- def hidden_copy(obj, short_in_text: false)
109
- case obj
110
- when Hash
111
- obj.to_h do |k, v|
112
- [k.is_a?(String) ? hidden_copy(k, short_in_text: short_in_text) : k, secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text)]
113
- end
114
- when Array then obj.map { |i| hidden_copy(i, short_in_text: short_in_text) }
115
- when Struct then obj.dup.tap { |s| s.each_pair { |k, v| s[k] = secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text) } }
116
- when String then hide_values_in_string(obj, hidden_marker, short_in_text: short_in_text)
117
- else obj
118
- end
119
- end
120
-
121
- # @param obj [Object] Hash, Array, Struct, String, or other value
122
- # @return [Boolean] `true` if obj contains a secret: by key name, or declared value
123
- def contains_secret?(obj)
124
- !hidden_copy(obj).eql?(obj)
125
- end
126
-
127
- # Declare secret values: hidden wherever they appear (logs, results), also when escaped in JSON, Ruby or URL
128
- # @param values [Array<Object>] Secret values: only non-empty Strings are retained
129
- def add_secret_values(*values)
130
- forms = values.select { |v| v.is_a?(String) && !v.empty? }.flat_map do |v|
131
- [v, (JSON.generate(v)[1..-2] rescue nil), v.inspect[1..-2], URI.encode_www_form_component(v)]
132
- end.compact
133
- return if forms.empty?
134
- @mutex.synchronize do
135
- values = @values | forms
136
- next if values.size.eql?(@values.size)
137
- long, short = values.partition { |v| v.length >= VALUE_IN_STRING_LENGTH }
138
- # Longest first, so that a secret is not partly hidden by another one that it contains
139
- @values_regex = long.empty? ? nil : Regexp.union(long.sort_by { |v| -v.length })
140
- @short_values_regex = short.empty? ? nil : /(?<![[:alnum:]])#{Regexp.union(short.sort_by { |v| -v.length })}(?![[:alnum:]])/
141
- # Replaced, not modified, for readers in other threads
142
- @values = values.freeze
143
- end
144
- end
145
-
146
- # Declare the secret values found by key name in a structure
147
- # @param obj [Object] Hash or Array, e.g. a transfer spec
148
- def add_secrets_in(obj)
149
- case obj
150
- when Array then obj.each { |i| add_secrets_in(i) }
151
- when Hash then obj.each { |k, v| secret?(k, v) ? add_secret_values(v) : add_secrets_in(v) }
152
- end
153
- end
154
-
155
- # Declare additional exact key names whose values are secrets
156
- # @param keys [Array<String, Symbol>] Key names
157
- def add_secret_keys(keys)
158
- @additional_keys.merge(keys.map(&:to_s))
159
- end
160
-
161
- private
162
-
163
- def initialize
164
- @log_secrets = false
165
- @additional_keys = Set.new
166
- # Declared secret values, and their escaped forms
167
- @values = Set.new.freeze
168
- # Matches declared values long enough to be hidden inside a string
169
- @values_regex = nil
170
- # Matches shorter declared values as a word
171
- @short_values_regex = nil
172
- @mutex = Mutex.new
173
- end
174
-
175
- # @param value [String] Input string
176
- # @param marker [String] Placeholder
177
- # @param short_in_text [Boolean] `true`: also hide short values as a word
178
- # @return [String] value with declared secret values replaced by placeholder
179
- def hide_values_in_string(value, marker, short_in_text: false)
180
- return marker if @values.include?(value)
181
- long_regex = @values_regex
182
- short_regex = @short_values_regex if short_in_text
183
- # Long values first, as they may contain short ones
184
- value = value.gsub(long_regex, marker) unless long_regex.nil?
185
- value = value.gsub(short_regex, marker) unless short_regex.nil?
186
- value
187
- end
188
-
189
- # @return [String] Placeholder for hidden secrets, in ASCII when Unicode is not supported
190
- def hidden_marker
191
- # Required here, as `aspera/environment` requires `aspera/log`, which requires this file
192
- require 'aspera/environment'
193
- Environment.char(HIDDEN_PASSWORD)
194
- end
195
- end
196
- end
@@ -1,114 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
-
6
- module Aspera
7
- # Estimate of the strength of a secret (e.g. a token), in bits: log2 of the number of guesses to find it
8
- #
9
- # Uses gem `zxcvbn` if installed (dictionaries, l33t, dates...), else a built-in estimate, the lowest of:
10
- # - alphabet: length x log2(size of character classes used)
11
- # - Shannon: empirical entropy of characters (catches repeats)
12
- # - patterns: repeats, sequences and keyboard rows cost 1 bit per character, words and years 2 bits per character
13
- # Both under-estimate random secrets: a random secret of 32 bytes (e.g. `openssl rand -hex 32`) is far above any usual minimum.
14
- module SecretStrength
15
- # Default minimum strength of tokens, in bits: random tokens of 32 bytes are far above
16
- DEFAULT_MIN_BITS = 80
17
- # Classes of characters, with their size
18
- CHARACTER_CLASSES = [[/[a-z]/, 26], [/[A-Z]/, 26], [/\d/, 10], [/[^a-zA-Z\d]/, 33]].freeze
19
- # Adjacent characters in those rows form a sequence
20
- KEYBOARD_ROWS = %w[1234567890 qwertyuiop asdfghjkl zxcvbnm azertyuiop qsdfghjklm wxcvbn].freeze
21
- # Words (3+ letters, optionally capitalized) and years
22
- LOW_ENTROPY_REGEXES = [/[A-Z]?[a-z]{3,}/, /(?:19|20)\d\d/].freeze
23
- # Entropy of English text is 1 to 1.5 bit per letter: keep a margin
24
- LOW_ENTROPY_BITS = 2.0
25
- # zxcvbn is slow on long secrets: the beginning is enough to assess strength
26
- ZXCVBN_MAX_LENGTH = 100
27
- private_constant :CHARACTER_CLASSES, :KEYBOARD_ROWS, :LOW_ENTROPY_REGEXES, :LOW_ENTROPY_BITS, :ZXCVBN_MAX_LENGTH
28
-
29
- class << self
30
- # @param secret [String] Secret to assess
31
- # @return [Float] Estimated strength in bits
32
- def bits(secret)
33
- secret = secret.to_s
34
- return 0.0 if secret.empty?
35
- zxcvbn_bits(secret) || estimate_bits(secret)
36
- end
37
-
38
- # Refuse an empty or weak secret: an empty one would often accept requests without secret
39
- # @param secret [String] Secret to check
40
- # @param name [String] Name of the parameter holding the secret, its minimum is in parameter `<name>_min_bits`
41
- # @param min_bits [Integer] Minimum estimated strength in bits, `0` checks only that the secret is not empty
42
- # @param type [Exception] Exception to raise
43
- # @raise [Exception] if the secret is empty or weaker than `min_bits`
44
- def assert_strong(secret, name:, min_bits: DEFAULT_MIN_BITS, type: ParameterError)
45
- Aspera.assert(!secret.to_s.empty?, type: type) { "#{name} must not be empty" }
46
- strength = bits(secret)
47
- Aspera.assert(strength >= min_bits, type: type) do
48
- "#{name} is too weak: about #{strength.round} bits, minimum #{min_bits} (#{name}_min_bits). Generate one with: openssl rand -hex 32"
49
- end
50
- end
51
-
52
- # Built-in estimate, without dictionary
53
- # @param secret [String] Secret to assess
54
- # @return [Float] Estimated strength in bits
55
- def estimate_bits(secret)
56
- return 0.0 if secret.empty?
57
- bits_per_char = Math.log2(CHARACTER_CLASSES.sum { |regex, size| secret.match?(regex) ? size : 0 })
58
- [secret.size * bits_per_char, shannon_bits(secret), pattern_bits(secret, bits_per_char)].min
59
- end
60
-
61
- private
62
-
63
- # @return [Float, nil] Strength estimated by gem `zxcvbn`, or `nil` if not installed
64
- def zxcvbn_bits(secret)
65
- return unless zxcvbn?
66
- Math.log2(Zxcvbn.test(secret[0, ZXCVBN_MAX_LENGTH]).guesses)
67
- rescue StandardError => e
68
- Log.log.debug { "zxcvbn failed: #{e.message}" }
69
- nil
70
- end
71
-
72
- # @return [Boolean] `true` if gem `zxcvbn` (or `zxcvbn-ruby`) is installed
73
- def zxcvbn?
74
- return @zxcvbn unless @zxcvbn.nil?
75
- @zxcvbn =
76
- begin
77
- require 'zxcvbn'
78
- true
79
- rescue LoadError
80
- false
81
- end
82
- end
83
-
84
- # Empirical Shannon entropy of characters of the secret
85
- def shannon_bits(secret)
86
- secret.chars.tally.values.sum do |count|
87
- probability = count.fdiv(secret.size)
88
- -probability * Math.log2(probability)
89
- end * secret.size
90
- end
91
-
92
- # Sum of bits of each character, depending on patterns
93
- def pattern_bits(secret, bits_per_char)
94
- # Bits of characters in words and years
95
- low = Array.new(secret.size)
96
- LOW_ENTROPY_REGEXES.each do |regex|
97
- secret.scan(regex) { low.fill(LOW_ENTROPY_BITS, Regexp.last_match.begin(0)...Regexp.last_match.end(0)) }
98
- end
99
- secret.chars.each_with_index.sum do |char, index|
100
- next 1.0 if index.positive? && sequence?(secret[index - 1], char)
101
- low[index] || bits_per_char
102
- end
103
- end
104
-
105
- # @return [Boolean] `true` if `char` repeats `previous`, or follows it in alphabet, digits or keyboard row
106
- def sequence?(previous, char)
107
- previous = previous.downcase
108
- char = char.downcase
109
- return true if (char.ord - previous.ord).abs <= 1
110
- KEYBOARD_ROWS.any? { |row| (one = row.index(previous)) && (two = row.index(char)) && (one - two).abs.eql?(1) }
111
- end
112
- end
113
- end
114
- end