aspera-cli 4.28.0 → 4.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +217 -2
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +135 -56
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +122 -26
- data/SECURITY.md +84 -0
- data/bin/ascli +8 -5
- data/bin/asession +9 -6
- data/docs/README.md +1895 -632
- data/docs/test-mcp-with-ai.md +8 -8
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +20 -19
- data/lib/aspera/agent/factory.rb +6 -3
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +7 -4
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +29 -22
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +95 -67
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +370 -0
- data/lib/aspera/ascp/log_locations.rb +163 -0
- data/lib/aspera/ascp/management.rb +16 -8
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +12 -19
- data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +38 -33
- data/lib/aspera/cli/command_line.rb +63 -8
- data/lib/aspera/cli/command_registry.rb +9 -1
- data/lib/aspera/cli/command_spec.rb +47 -7
- data/lib/aspera/cli/completion/ascli.bash +19 -5
- data/lib/aspera/cli/completion/ascli.fish +13 -2
- data/lib/aspera/cli/completion/ascli.ps1 +31 -9
- data/lib/aspera/cli/completion/ascli.zsh +22 -6
- data/lib/aspera/cli/completion_actions.rb +303 -69
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +6 -0
- data/lib/aspera/cli/extended_value.rb +66 -31
- data/lib/aspera/cli/formatter.rb +43 -30
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -10
- data/lib/aspera/cli/http.rb +38 -10
- data/lib/aspera/cli/info.rb +12 -7
- data/lib/aspera/cli/mailer.rb +6 -3
- data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
- data/lib/aspera/cli/mcp/secrets.rb +8 -5
- data/lib/aspera/cli/mcp/tool.rb +20 -17
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +77 -18
- data/lib/aspera/cli/options.schema.yaml +641 -80
- data/lib/aspera/cli/parser.rb +71 -35
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +248 -37
- data/lib/aspera/cli/plugins/ats.rb +10 -9
- data/lib/aspera/cli/plugins/base.rb +50 -18
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +210 -104
- data/lib/aspera/cli/plugins/console.rb +6 -3
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +21 -7
- data/lib/aspera/cli/plugins/faspex5.rb +31 -26
- data/lib/aspera/cli/plugins/faspio.rb +7 -5
- data/lib/aspera/cli/plugins/httpgw.rb +7 -4
- data/lib/aspera/cli/plugins/mcp.rb +42 -22
- data/lib/aspera/cli/plugins/node.rb +39 -38
- data/lib/aspera/cli/plugins/oauth.rb +23 -6
- data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
- data/lib/aspera/cli/plugins/preview.rb +72 -45
- data/lib/aspera/cli/plugins/server.rb +66 -14
- data/lib/aspera/cli/plugins/shares.rb +87 -18
- data/lib/aspera/cli/plugins/transferd.rb +228 -0
- data/lib/aspera/cli/poll.rb +236 -0
- data/lib/aspera/cli/preset_actions.rb +144 -30
- data/lib/aspera/cli/preset_manager.rb +113 -35
- data/lib/aspera/cli/prompt.rb +157 -9
- data/lib/aspera/cli/result.rb +12 -8
- data/lib/aspera/cli/runner.rb +89 -48
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +89 -9
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +15 -10
- data/lib/aspera/cli/table_fitter.rb +3 -0
- data/lib/aspera/cli/terminal_formatter.rb +9 -6
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +43 -20
- data/lib/aspera/cli/transfer_progress.rb +14 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +188 -42
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.rb +12 -10
- data/lib/aspera/cli/wizard.rb +42 -39
- data/lib/aspera/command_line_builder.rb +9 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +4 -1
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/environment.rb +23 -12
- data/lib/aspera/exec_spec.rb +3 -0
- data/lib/aspera/faspex_gw.rb +13 -9
- data/lib/aspera/faspex_postproc.rb +15 -12
- data/lib/aspera/formatter_interface.rb +4 -1
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +20 -5
- data/lib/aspera/inflate.rb +43 -0
- data/lib/aspera/json_rpc/client.rb +6 -5
- data/lib/aspera/json_rpc/version.rb +3 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +18 -5
- data/lib/aspera/markdown.rb +5 -2
- data/lib/aspera/mime.rb +3 -0
- data/lib/aspera/nagios.rb +6 -4
- data/lib/aspera/node_emulator.rb +15 -13
- data/lib/aspera/oauth/client/base.rb +124 -73
- data/lib/aspera/oauth/client/boot.rb +7 -4
- data/lib/aspera/oauth/client/factory.rb +41 -18
- data/lib/aspera/oauth/client/generic.rb +5 -1
- data/lib/aspera/oauth/client/json_credentials.rb +7 -3
- data/lib/aspera/oauth/client/jwt.rb +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +94 -0
- data/lib/aspera/oauth/client/url_json.rb +7 -3
- data/lib/aspera/oauth/client/web.rb +40 -13
- data/lib/aspera/oauth/server/pkce.rb +6 -5
- data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +16 -6
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +17 -2
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +3 -0
- data/lib/aspera/rest/aspera_errors.rb +4 -1
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +45 -34
- data/lib/aspera/rest/error_analyzer.rb +5 -2
- data/lib/aspera/rest/list.rb +8 -4
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +8 -5
- data/lib/aspera/rest.rb +9 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +8 -2
- data/lib/aspera/schema/reader.rb +7 -0
- data/lib/aspera/schema/registry.rb +44 -28
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +150 -0
- data/lib/aspera/secret/hider.rb +213 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +112 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +5 -2
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +3 -0
- data/lib/aspera/sync/conf.schema.yaml +9 -4
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +17 -15
- data/lib/aspera/temp_file_manager.rb +8 -4
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +17 -14
- data/lib/aspera/transfer/result.rb +23 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +9 -22
- data/lib/aspera/units/decimal.rb +67 -0
- data/lib/aspera/units/duration.rb +109 -0
- data/lib/aspera/uri_reader.rb +64 -9
- data/lib/aspera/web_auth/index.html.erb +6 -2
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +47 -10
- data/lib/aspera/web_page.rb +7 -2
- data/lib/aspera/web_server_simple.rb +6 -3
- data/lib/aspera/yaml.rb +5 -1
- data/llms-install.md +88 -0
- data/llms.txt +87 -0
- data/skills/ascli/SKILL.md +94 -0
- data.tar.gz.sig +0 -0
- metadata +77 -20
- metadata.gz.sig +0 -0
- data/lib/aspera/duration.rb +0 -42
- data/lib/aspera/keychain/base.rb +0 -30
- data/lib/aspera/keychain/encrypted_hash.rb +0 -192
- data/lib/aspera/keychain/factory.rb +0 -58
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
- data/lib/aspera/keychain/macos_security.rb +0 -277
- data/lib/aspera/keychain/one_password_api.rb +0 -91
- data/lib/aspera/keychain/one_password_base.rb +0 -39
- data/lib/aspera/keychain/one_password_cli.rb +0 -100
- data/lib/aspera/secret_hider.rb +0 -196
- data/lib/aspera/secret_strength.rb +0 -114
data/lib/aspera/secret_hider.rb
DELETED
|
@@ -1,196 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
# cspell:ignore FILEPASS
|
|
4
|
-
require 'logger'
|
|
5
|
-
require 'set'
|
|
6
|
-
require 'singleton'
|
|
7
|
-
require 'json'
|
|
8
|
-
require 'uri'
|
|
9
|
-
|
|
10
|
-
module Aspera
|
|
11
|
-
# remove secret from logs and output
|
|
12
|
-
#
|
|
13
|
-
# Secrets are identified:
|
|
14
|
-
# - by key name, in structures (`secret?`, `hidden_copy`) and in strings with known patterns (`hide_secrets_in_string`)
|
|
15
|
-
# - by value, when declared where they come from (`add_secret_values`, `add_secrets_in`): option values, tokens, ...
|
|
16
|
-
#
|
|
17
|
-
# @!method self.instance
|
|
18
|
-
# Returns the singleton instance of SecretHider
|
|
19
|
-
# @return [SecretHider] the singleton instance
|
|
20
|
-
class SecretHider
|
|
21
|
-
include Singleton
|
|
22
|
-
|
|
23
|
-
# display string for hidden secrets (see `Environment::UNICODE_TO_ASCII`)
|
|
24
|
-
HIDDEN_PASSWORD = '🔑'
|
|
25
|
-
# env vars for ascp with secrets
|
|
26
|
-
ASCP_ENV_SECRETS = %w[ASPERA_SCP_PASS ASPERA_SCP_KEY ASPERA_SCP_FILEPASS ASPERA_PROXY_PASS ASPERA_SCP_TOKEN].freeze
|
|
27
|
-
# keys in hash that contain secrets
|
|
28
|
-
KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
|
|
29
|
-
HTTP_SECRETS = %w[Authorization].freeze
|
|
30
|
-
ALL_SECRETS = (ASCP_ENV_SECRETS + KEY_SECRETS + HTTP_SECRETS).freeze
|
|
31
|
-
NON_ENV_SECRETS = (KEY_SECRETS + HTTP_SECRETS).freeze
|
|
32
|
-
KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/, /public_key$/, /^token_type$/].freeze
|
|
33
|
-
# min length of hidden secrets, use `+` or `{n,}`
|
|
34
|
-
SECRET_LENGTH = '{5,}'
|
|
35
|
-
# min length of a declared secret value to be hidden inside a string: shorter ones are hidden only when they are the whole string,
|
|
36
|
-
# else common words would be hidden (e.g. password `aspera` in URLs), or optionally as a word (`short_in_text`)
|
|
37
|
-
VALUE_IN_STRING_LENGTH = 8
|
|
38
|
-
# regex that define named captures :begin and :end
|
|
39
|
-
REGEX_LOG_REPLACES = [
|
|
40
|
-
# private key values (place first)
|
|
41
|
-
/(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)\n*/,
|
|
42
|
-
# CLI manager get/set options
|
|
43
|
-
/(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/i,
|
|
44
|
-
# env var ascp exec
|
|
45
|
-
/(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)[^ \n]+(?<end> |$)/,
|
|
46
|
-
# rendered JSON or Ruby (quoted key or legacy symbol key: `"k":"v"`, `"k" => "v"`, `:k=>"v"`)
|
|
47
|
-
/(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>? *(?:=>|:) *")(?:[^"\\]|\\.)+(?<end>")/i,
|
|
48
|
-
# rendered Ruby 3.4+ symbol key: `k: "v"`
|
|
49
|
-
/(?<begin>\b\w*(?:#{ALL_SECRETS.join('|')})\w*: *")(?:[^"\\]|\\.)+(?<end>")/i,
|
|
50
|
-
# rendered Ruby Struct or object: `k="v"`, `@k="v"`
|
|
51
|
-
/(?<begin>@?\b\w*(?:#{ALL_SECRETS.join('|')})\w*=")(?:[^"\\]|\\.)+(?<end>")/i,
|
|
52
|
-
# query parameter of URL: `?token=v` or `&client_secret=v`
|
|
53
|
-
/(?<begin>[?&][^=&\s"]*(?:#{KEY_SECRETS.join('|')})[^=&\s"]*=)[^&\s"#]+(?<end>)/i,
|
|
54
|
-
# logged data
|
|
55
|
-
/(?<begin>(?:#{NON_ENV_SECRETS.join('|')})[ =:]+)[^ "\n]#{SECRET_LENGTH}(?<end>$)/i,
|
|
56
|
-
# cred in http dump
|
|
57
|
-
/(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
|
|
58
|
-
].freeze
|
|
59
|
-
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :SECRET_LENGTH, :VALUE_IN_STRING_LENGTH, :REGEX_LOG_REPLACES
|
|
60
|
-
attr_accessor :log_secrets
|
|
61
|
-
|
|
62
|
-
# @return [Proc] new log formatter that hides secrets
|
|
63
|
-
def log_formatter(original_formatter)
|
|
64
|
-
original_formatter ||= Logger::Formatter.new
|
|
65
|
-
# NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
|
66
|
-
# Hiding is applied on the formatted line, so that non-String messages (e.g. Exception) are covered too
|
|
67
|
-
return lambda do |severity, date_time, program_name, msg|
|
|
68
|
-
line = original_formatter.call(severity, date_time, program_name, msg)
|
|
69
|
-
line = hide_secrets_in_string(line, all: true) if line.is_a?(String) && !@log_secrets
|
|
70
|
-
line
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
# Replace secrets in a string with the hidden password placeholder
|
|
75
|
-
# @param value [String] Input string possibly containing secrets
|
|
76
|
-
# @param all [Boolean] `false`: only declared values and private keys, `true`: also all known secret patterns
|
|
77
|
-
# @return [String] String with secrets replaced by placeholder
|
|
78
|
-
def hide_secrets_in_string(value, all: false)
|
|
79
|
-
marker = hidden_marker
|
|
80
|
-
value = hide_values_in_string(value, marker)
|
|
81
|
-
(all ? REGEX_LOG_REPLACES : REGEX_LOG_REPLACES.first(1)).each do |reg_ex|
|
|
82
|
-
value = value.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{marker}#{Regexp.last_match(:end)}" }
|
|
83
|
-
end
|
|
84
|
-
return value
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
# @param keyword [String, Symbol] Key name to check
|
|
88
|
-
# @param value [String] Associated value (must be a String to be a secret)
|
|
89
|
-
# @return [Boolean] true if the key denotes a secret
|
|
90
|
-
def secret?(keyword, value)
|
|
91
|
-
keyword = keyword.to_s if keyword.is_a?(Symbol)
|
|
92
|
-
# only Strings can be secrets, not booleans, or hash, arrays
|
|
93
|
-
return false unless keyword.is_a?(String) && value.is_a?(String)
|
|
94
|
-
return true if @additional_keys.include?(keyword)
|
|
95
|
-
keyword = keyword.downcase
|
|
96
|
-
# those are not secrets
|
|
97
|
-
return false if KEY_FALSE_POSITIVES.any? { |f| f.match?(keyword) }
|
|
98
|
-
# check if keyword (name) contains an element that designate it as a secret
|
|
99
|
-
ALL_SECRETS.any? { |kw| keyword.include?(kw.downcase) }
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
# Hides secrets in a structure: by key name, and declared values, also in key names (e.g. a secret given as name of a parameter).
|
|
103
|
-
# Containers are copied, not modified: the structure may be shared (e.g. configuration, transfer spec).
|
|
104
|
-
# @param obj [Object] Hash, Array, Struct, String, or other value (returned as is)
|
|
105
|
-
# @param short_in_text [Boolean] `true`: a declared value shorter than 8 characters is also hidden inside a text, as a word
|
|
106
|
-
# (not next to a letter or digit), e.g. a short password in an error message of `mcp server`
|
|
107
|
-
# @return [Object] Copy of obj with secrets replaced by placeholder
|
|
108
|
-
def hidden_copy(obj, short_in_text: false)
|
|
109
|
-
case obj
|
|
110
|
-
when Hash
|
|
111
|
-
obj.to_h do |k, v|
|
|
112
|
-
[k.is_a?(String) ? hidden_copy(k, short_in_text: short_in_text) : k, secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text)]
|
|
113
|
-
end
|
|
114
|
-
when Array then obj.map { |i| hidden_copy(i, short_in_text: short_in_text) }
|
|
115
|
-
when Struct then obj.dup.tap { |s| s.each_pair { |k, v| s[k] = secret?(k, v) ? hidden_marker : hidden_copy(v, short_in_text: short_in_text) } }
|
|
116
|
-
when String then hide_values_in_string(obj, hidden_marker, short_in_text: short_in_text)
|
|
117
|
-
else obj
|
|
118
|
-
end
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
# @param obj [Object] Hash, Array, Struct, String, or other value
|
|
122
|
-
# @return [Boolean] `true` if obj contains a secret: by key name, or declared value
|
|
123
|
-
def contains_secret?(obj)
|
|
124
|
-
!hidden_copy(obj).eql?(obj)
|
|
125
|
-
end
|
|
126
|
-
|
|
127
|
-
# Declare secret values: hidden wherever they appear (logs, results), also when escaped in JSON, Ruby or URL
|
|
128
|
-
# @param values [Array<Object>] Secret values: only non-empty Strings are retained
|
|
129
|
-
def add_secret_values(*values)
|
|
130
|
-
forms = values.select { |v| v.is_a?(String) && !v.empty? }.flat_map do |v|
|
|
131
|
-
[v, (JSON.generate(v)[1..-2] rescue nil), v.inspect[1..-2], URI.encode_www_form_component(v)]
|
|
132
|
-
end.compact
|
|
133
|
-
return if forms.empty?
|
|
134
|
-
@mutex.synchronize do
|
|
135
|
-
values = @values | forms
|
|
136
|
-
next if values.size.eql?(@values.size)
|
|
137
|
-
long, short = values.partition { |v| v.length >= VALUE_IN_STRING_LENGTH }
|
|
138
|
-
# Longest first, so that a secret is not partly hidden by another one that it contains
|
|
139
|
-
@values_regex = long.empty? ? nil : Regexp.union(long.sort_by { |v| -v.length })
|
|
140
|
-
@short_values_regex = short.empty? ? nil : /(?<![[:alnum:]])#{Regexp.union(short.sort_by { |v| -v.length })}(?![[:alnum:]])/
|
|
141
|
-
# Replaced, not modified, for readers in other threads
|
|
142
|
-
@values = values.freeze
|
|
143
|
-
end
|
|
144
|
-
end
|
|
145
|
-
|
|
146
|
-
# Declare the secret values found by key name in a structure
|
|
147
|
-
# @param obj [Object] Hash or Array, e.g. a transfer spec
|
|
148
|
-
def add_secrets_in(obj)
|
|
149
|
-
case obj
|
|
150
|
-
when Array then obj.each { |i| add_secrets_in(i) }
|
|
151
|
-
when Hash then obj.each { |k, v| secret?(k, v) ? add_secret_values(v) : add_secrets_in(v) }
|
|
152
|
-
end
|
|
153
|
-
end
|
|
154
|
-
|
|
155
|
-
# Declare additional exact key names whose values are secrets
|
|
156
|
-
# @param keys [Array<String, Symbol>] Key names
|
|
157
|
-
def add_secret_keys(keys)
|
|
158
|
-
@additional_keys.merge(keys.map(&:to_s))
|
|
159
|
-
end
|
|
160
|
-
|
|
161
|
-
private
|
|
162
|
-
|
|
163
|
-
def initialize
|
|
164
|
-
@log_secrets = false
|
|
165
|
-
@additional_keys = Set.new
|
|
166
|
-
# Declared secret values, and their escaped forms
|
|
167
|
-
@values = Set.new.freeze
|
|
168
|
-
# Matches declared values long enough to be hidden inside a string
|
|
169
|
-
@values_regex = nil
|
|
170
|
-
# Matches shorter declared values as a word
|
|
171
|
-
@short_values_regex = nil
|
|
172
|
-
@mutex = Mutex.new
|
|
173
|
-
end
|
|
174
|
-
|
|
175
|
-
# @param value [String] Input string
|
|
176
|
-
# @param marker [String] Placeholder
|
|
177
|
-
# @param short_in_text [Boolean] `true`: also hide short values as a word
|
|
178
|
-
# @return [String] value with declared secret values replaced by placeholder
|
|
179
|
-
def hide_values_in_string(value, marker, short_in_text: false)
|
|
180
|
-
return marker if @values.include?(value)
|
|
181
|
-
long_regex = @values_regex
|
|
182
|
-
short_regex = @short_values_regex if short_in_text
|
|
183
|
-
# Long values first, as they may contain short ones
|
|
184
|
-
value = value.gsub(long_regex, marker) unless long_regex.nil?
|
|
185
|
-
value = value.gsub(short_regex, marker) unless short_regex.nil?
|
|
186
|
-
value
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
# @return [String] Placeholder for hidden secrets, in ASCII when Unicode is not supported
|
|
190
|
-
def hidden_marker
|
|
191
|
-
# Required here, as `aspera/environment` requires `aspera/log`, which requires this file
|
|
192
|
-
require 'aspera/environment'
|
|
193
|
-
Environment.char(HIDDEN_PASSWORD)
|
|
194
|
-
end
|
|
195
|
-
end
|
|
196
|
-
end
|
|
@@ -1,114 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'aspera/log'
|
|
4
|
-
require 'aspera/assert'
|
|
5
|
-
|
|
6
|
-
module Aspera
|
|
7
|
-
# Estimate of the strength of a secret (e.g. a token), in bits: log2 of the number of guesses to find it
|
|
8
|
-
#
|
|
9
|
-
# Uses gem `zxcvbn` if installed (dictionaries, l33t, dates...), else a built-in estimate, the lowest of:
|
|
10
|
-
# - alphabet: length x log2(size of character classes used)
|
|
11
|
-
# - Shannon: empirical entropy of characters (catches repeats)
|
|
12
|
-
# - patterns: repeats, sequences and keyboard rows cost 1 bit per character, words and years 2 bits per character
|
|
13
|
-
# Both under-estimate random secrets: a random secret of 32 bytes (e.g. `openssl rand -hex 32`) is far above any usual minimum.
|
|
14
|
-
module SecretStrength
|
|
15
|
-
# Default minimum strength of tokens, in bits: random tokens of 32 bytes are far above
|
|
16
|
-
DEFAULT_MIN_BITS = 80
|
|
17
|
-
# Classes of characters, with their size
|
|
18
|
-
CHARACTER_CLASSES = [[/[a-z]/, 26], [/[A-Z]/, 26], [/\d/, 10], [/[^a-zA-Z\d]/, 33]].freeze
|
|
19
|
-
# Adjacent characters in those rows form a sequence
|
|
20
|
-
KEYBOARD_ROWS = %w[1234567890 qwertyuiop asdfghjkl zxcvbnm azertyuiop qsdfghjklm wxcvbn].freeze
|
|
21
|
-
# Words (3+ letters, optionally capitalized) and years
|
|
22
|
-
LOW_ENTROPY_REGEXES = [/[A-Z]?[a-z]{3,}/, /(?:19|20)\d\d/].freeze
|
|
23
|
-
# Entropy of English text is 1 to 1.5 bit per letter: keep a margin
|
|
24
|
-
LOW_ENTROPY_BITS = 2.0
|
|
25
|
-
# zxcvbn is slow on long secrets: the beginning is enough to assess strength
|
|
26
|
-
ZXCVBN_MAX_LENGTH = 100
|
|
27
|
-
private_constant :CHARACTER_CLASSES, :KEYBOARD_ROWS, :LOW_ENTROPY_REGEXES, :LOW_ENTROPY_BITS, :ZXCVBN_MAX_LENGTH
|
|
28
|
-
|
|
29
|
-
class << self
|
|
30
|
-
# @param secret [String] Secret to assess
|
|
31
|
-
# @return [Float] Estimated strength in bits
|
|
32
|
-
def bits(secret)
|
|
33
|
-
secret = secret.to_s
|
|
34
|
-
return 0.0 if secret.empty?
|
|
35
|
-
zxcvbn_bits(secret) || estimate_bits(secret)
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
# Refuse an empty or weak secret: an empty one would often accept requests without secret
|
|
39
|
-
# @param secret [String] Secret to check
|
|
40
|
-
# @param name [String] Name of the parameter holding the secret, its minimum is in parameter `<name>_min_bits`
|
|
41
|
-
# @param min_bits [Integer] Minimum estimated strength in bits, `0` checks only that the secret is not empty
|
|
42
|
-
# @param type [Exception] Exception to raise
|
|
43
|
-
# @raise [Exception] if the secret is empty or weaker than `min_bits`
|
|
44
|
-
def assert_strong(secret, name:, min_bits: DEFAULT_MIN_BITS, type: ParameterError)
|
|
45
|
-
Aspera.assert(!secret.to_s.empty?, type: type) { "#{name} must not be empty" }
|
|
46
|
-
strength = bits(secret)
|
|
47
|
-
Aspera.assert(strength >= min_bits, type: type) do
|
|
48
|
-
"#{name} is too weak: about #{strength.round} bits, minimum #{min_bits} (#{name}_min_bits). Generate one with: openssl rand -hex 32"
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
# Built-in estimate, without dictionary
|
|
53
|
-
# @param secret [String] Secret to assess
|
|
54
|
-
# @return [Float] Estimated strength in bits
|
|
55
|
-
def estimate_bits(secret)
|
|
56
|
-
return 0.0 if secret.empty?
|
|
57
|
-
bits_per_char = Math.log2(CHARACTER_CLASSES.sum { |regex, size| secret.match?(regex) ? size : 0 })
|
|
58
|
-
[secret.size * bits_per_char, shannon_bits(secret), pattern_bits(secret, bits_per_char)].min
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
private
|
|
62
|
-
|
|
63
|
-
# @return [Float, nil] Strength estimated by gem `zxcvbn`, or `nil` if not installed
|
|
64
|
-
def zxcvbn_bits(secret)
|
|
65
|
-
return unless zxcvbn?
|
|
66
|
-
Math.log2(Zxcvbn.test(secret[0, ZXCVBN_MAX_LENGTH]).guesses)
|
|
67
|
-
rescue StandardError => e
|
|
68
|
-
Log.log.debug { "zxcvbn failed: #{e.message}" }
|
|
69
|
-
nil
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
# @return [Boolean] `true` if gem `zxcvbn` (or `zxcvbn-ruby`) is installed
|
|
73
|
-
def zxcvbn?
|
|
74
|
-
return @zxcvbn unless @zxcvbn.nil?
|
|
75
|
-
@zxcvbn =
|
|
76
|
-
begin
|
|
77
|
-
require 'zxcvbn'
|
|
78
|
-
true
|
|
79
|
-
rescue LoadError
|
|
80
|
-
false
|
|
81
|
-
end
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
# Empirical Shannon entropy of characters of the secret
|
|
85
|
-
def shannon_bits(secret)
|
|
86
|
-
secret.chars.tally.values.sum do |count|
|
|
87
|
-
probability = count.fdiv(secret.size)
|
|
88
|
-
-probability * Math.log2(probability)
|
|
89
|
-
end * secret.size
|
|
90
|
-
end
|
|
91
|
-
|
|
92
|
-
# Sum of bits of each character, depending on patterns
|
|
93
|
-
def pattern_bits(secret, bits_per_char)
|
|
94
|
-
# Bits of characters in words and years
|
|
95
|
-
low = Array.new(secret.size)
|
|
96
|
-
LOW_ENTROPY_REGEXES.each do |regex|
|
|
97
|
-
secret.scan(regex) { low.fill(LOW_ENTROPY_BITS, Regexp.last_match.begin(0)...Regexp.last_match.end(0)) }
|
|
98
|
-
end
|
|
99
|
-
secret.chars.each_with_index.sum do |char, index|
|
|
100
|
-
next 1.0 if index.positive? && sequence?(secret[index - 1], char)
|
|
101
|
-
low[index] || bits_per_char
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
# @return [Boolean] `true` if `char` repeats `previous`, or follows it in alphabet, digits or keyboard row
|
|
106
|
-
def sequence?(previous, char)
|
|
107
|
-
previous = previous.downcase
|
|
108
|
-
char = char.downcase
|
|
109
|
-
return true if (char.ord - previous.ord).abs <= 1
|
|
110
|
-
KEYBOARD_ROWS.any? { |row| (one = row.index(previous)) && (two = row.index(char)) && (one - two).abs.eql?(1) }
|
|
111
|
-
end
|
|
112
|
-
end
|
|
113
|
-
end
|
|
114
|
-
end
|