aspera-cli 4.27.1 → 4.27.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +67 -1
- data/bin/ascli +2 -1
- data/docs/README.md +805 -747
- data/lib/aspera/agent/connect.rb +6 -4
- data/lib/aspera/agent/desktop.rb +2 -2
- data/lib/aspera/agent/direct.rb +3 -1
- data/lib/aspera/agent/node.rb +3 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -12
- data/lib/aspera/api/ats.rb +1 -1
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +9 -7
- data/lib/aspera/api/httpgw.rb +37 -33
- data/lib/aspera/api/node.rb +38 -33
- data/lib/aspera/ascmd.rb +3 -1
- data/lib/aspera/ascp/installation.rb +62 -27
- data/lib/aspera/ascp/management.rb +1 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +20 -41
- data/lib/aspera/cli/async_transfer_store.rb +2 -2
- data/lib/aspera/cli/bootstrapper.rb +11 -15
- data/lib/aspera/cli/command_line.rb +252 -0
- data/lib/aspera/cli/command_registry.rb +149 -33
- data/lib/aspera/cli/command_spec.rb +103 -14
- data/lib/aspera/cli/completion/ascli.bash +12 -0
- data/lib/aspera/cli/completion/ascli.fish +16 -0
- data/lib/aspera/cli/completion/ascli.zsh +19 -0
- data/lib/aspera/cli/context.rb +3 -0
- data/lib/aspera/cli/deprecation.rb +37 -0
- data/lib/aspera/cli/extended_value.rb +2 -0
- data/lib/aspera/cli/formatter.rb +87 -75
- data/lib/aspera/cli/gem_checker.rb +1 -1
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/http.rb +21 -21
- data/lib/aspera/cli/info.rb +3 -0
- data/lib/aspera/cli/mcp_tool.rb +47 -83
- data/lib/aspera/cli/option_declarator.rb +33 -42
- data/lib/aspera/cli/option_registry.rb +69 -0
- data/lib/aspera/cli/option_types.rb +103 -0
- data/lib/aspera/cli/option_value.rb +281 -0
- data/lib/aspera/cli/options.schema.yaml +38 -5
- data/lib/aspera/cli/parser.rb +307 -848
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +435 -380
- data/lib/aspera/cli/plugins/ats.rb +58 -73
- data/lib/aspera/cli/plugins/base.rb +190 -240
- data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
- data/lib/aspera/cli/plugins/config.rb +244 -178
- data/lib/aspera/cli/plugins/console.rb +102 -38
- data/lib/aspera/cli/plugins/cos.rb +6 -23
- data/lib/aspera/cli/plugins/factory.rb +3 -0
- data/lib/aspera/cli/plugins/faspex5.rb +176 -173
- data/lib/aspera/cli/plugins/faspio.rb +5 -10
- data/lib/aspera/cli/plugins/httpgw.rb +8 -11
- data/lib/aspera/cli/plugins/mcp.rb +20 -55
- data/lib/aspera/cli/plugins/node.rb +277 -311
- data/lib/aspera/cli/plugins/orchestrator.rb +90 -77
- data/lib/aspera/cli/plugins/preview.rb +79 -90
- data/lib/aspera/cli/plugins/server.rb +76 -50
- data/lib/aspera/cli/plugins/shares.rb +68 -116
- data/lib/aspera/cli/preset_actions.rb +17 -10
- data/lib/aspera/cli/preset_manager.rb +12 -2
- data/lib/aspera/cli/prompt.rb +35 -0
- data/lib/aspera/cli/result.rb +13 -18
- data/lib/aspera/cli/runner.rb +31 -54
- data/lib/aspera/cli/special_values.rb +5 -0
- data/lib/aspera/cli/sync_actions.rb +41 -37
- data/lib/aspera/cli/terminal_formatter.rb +9 -3
- data/lib/aspera/cli/transfer_actions.rb +0 -6
- data/lib/aspera/cli/transfer_agent.rb +29 -35
- data/lib/aspera/cli/vault_manager.rb +0 -17
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +4 -2
- data/lib/aspera/command_line_builder.rb +1 -0
- data/lib/aspera/coverage.rb +1 -0
- data/lib/aspera/environment.rb +7 -1
- data/lib/aspera/faspex_gw.rb +2 -1
- data/lib/aspera/faspex_postproc.rb +1 -0
- data/lib/aspera/graphql.rb +5 -5
- data/lib/aspera/json_rpc/client.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +1 -1
- data/lib/aspera/keychain/one_password_api.rb +1 -1
- data/lib/aspera/link_header.rb +2 -2
- data/lib/aspera/log.rb +22 -25
- data/lib/aspera/markdown.rb +2 -0
- data/lib/aspera/mime.rb +25 -0
- data/lib/aspera/node_simulator.rb +1 -0
- data/lib/aspera/oauth/base.rb +35 -25
- data/lib/aspera/oauth/factory.rb +1 -0
- data/lib/aspera/oauth/generic.rb +1 -1
- data/lib/aspera/oauth/jwt.rb +1 -1
- data/lib/aspera/oauth/web.rb +9 -8
- data/lib/aspera/preview/file_types.rb +4 -4
- data/lib/aspera/preview/generator.rb +7 -0
- data/lib/aspera/preview/options.rb +4 -4
- data/lib/aspera/preview/terminal.rb +4 -3
- data/lib/aspera/preview/utils.rb +9 -6
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/rainbow.rb +7 -0
- data/lib/aspera/rest/aspera_errors.rb +60 -0
- data/lib/aspera/rest/call_error.rb +27 -0
- data/lib/aspera/rest/client.rb +514 -0
- data/lib/aspera/rest/error_analyzer.rb +113 -0
- data/lib/aspera/rest/list.rb +143 -0
- data/lib/aspera/rest/parameters.rb +55 -0
- data/lib/aspera/rest/util.rb +176 -0
- data/lib/aspera/rest.rb +7 -621
- data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +39 -0
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
- data/lib/aspera/schema/documentation.rb +13 -3
- data/lib/aspera/schema/registry.rb +18 -1
- data/lib/aspera/schema/validator.rb +92 -0
- data/lib/aspera/secret_hider.rb +36 -25
- data/lib/aspera/string_ext.rb +15 -0
- data/lib/aspera/temp_file_manager.rb +6 -5
- data/lib/aspera/transfer/parameters.rb +2 -0
- data/lib/aspera/transfer/spec.rb +1 -0
- data/lib/aspera/transfer/spec.schema.yaml +1 -0
- data/lib/aspera/uri_reader.rb +11 -11
- data/lib/aspera/web_auth/index.html +147 -0
- data/lib/aspera/web_auth/server.rb +81 -0
- data.tar.gz.sig +0 -0
- metadata +39 -7
- metadata.gz.sig +0 -0
- data/lib/aspera/colors.rb +0 -79
- data/lib/aspera/rest_call_error.rb +0 -25
- data/lib/aspera/rest_error_analyzer.rb +0 -111
- data/lib/aspera/rest_errors_aspera.rb +0 -58
- data/lib/aspera/rest_list.rb +0 -136
- data/lib/aspera/web_auth.rb +0 -211
|
@@ -114,9 +114,9 @@ module Aspera
|
|
|
114
114
|
end
|
|
115
115
|
# Only keep lines that are usable in supported agents
|
|
116
116
|
next if agents.empty?
|
|
117
|
-
item['description'].push("Allowed values: #{node['enum'].map { |v|
|
|
118
|
-
item['description'].push("Default: #{
|
|
119
|
-
item['description'].push("Example: #{
|
|
117
|
+
item['description'].push("Allowed values: #{node['enum'].map { |v| value_text(v) }.join(', ')}.") if node.key?('enum')
|
|
118
|
+
item['description'].push("Default: #{value_text(node['default'])}.") if node.key?('default')
|
|
119
|
+
item['description'].push("Example: #{value_text(node['example'])}.") if node.key?('example')
|
|
120
120
|
if @include_option
|
|
121
121
|
envvar_prefix = ''
|
|
122
122
|
cli_option =
|
|
@@ -139,6 +139,16 @@ module Aspera
|
|
|
139
139
|
end
|
|
140
140
|
self
|
|
141
141
|
end
|
|
142
|
+
|
|
143
|
+
private
|
|
144
|
+
|
|
145
|
+
# Format a schema value (enum, default, example) as code
|
|
146
|
+
# @param value [Object] value from schema, `nil` for JSON `null`
|
|
147
|
+
# @return [String]
|
|
148
|
+
def value_text(value)
|
|
149
|
+
return @formatter.special_format('null') if value.nil?
|
|
150
|
+
@formatter.markdown_text("`#{value}`")
|
|
151
|
+
end
|
|
142
152
|
end
|
|
143
153
|
end
|
|
144
154
|
end
|
|
@@ -18,6 +18,12 @@ module Aspera
|
|
|
18
18
|
LOCATIONS.key?(sym)
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
+
# @param name_path [String] schema path, e.g. `opts:components.schemas.HttpOptions`
|
|
22
|
+
# @return [Boolean] `true` if schema is owned by ascli (not a vendor API)
|
|
23
|
+
def owned?(name_path)
|
|
24
|
+
OWNED.include?(name_path.split(':', 2).first.to_sym)
|
|
25
|
+
end
|
|
26
|
+
|
|
21
27
|
# Split a component string into registry key and optional path prefix.
|
|
22
28
|
# Syntax: 'key' or 'key+/prefix' (e.g. 'shares+/api/v1')
|
|
23
29
|
# @param component [String] registry key with optional '+/prefix'
|
|
@@ -53,20 +59,27 @@ module Aspera
|
|
|
53
59
|
conf: 'aspera/sync/conf.schema.yaml',
|
|
54
60
|
opts: 'aspera/cli/options.schema.yaml',
|
|
55
61
|
aoc: 'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
|
|
62
|
+
automation: 'aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml',
|
|
56
63
|
faspex: 'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml',
|
|
57
64
|
faspio: 'aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml',
|
|
65
|
+
console: 'aspera/schema/IBM Aspera Console-enhanced.yaml',
|
|
58
66
|
node: 'aspera/schema/IBM Aspera Node API-4.4.6.yaml',
|
|
59
67
|
shares: 'aspera/schema/IBM_Aspera_Shares.yaml',
|
|
60
68
|
async_tables: 'aspera/schema/async_tables.yaml'
|
|
61
69
|
}
|
|
62
70
|
|
|
71
|
+
# Schemas owned by ascli: values are validated against them (vendor API schemas are validated by the API)
|
|
72
|
+
OWNED = %i[spec args conf opts async_tables].freeze
|
|
73
|
+
|
|
63
74
|
OPTIONS = 'opts'
|
|
64
75
|
TRANSFER_SPEC = 'spec'
|
|
65
76
|
SYNC_CONF = 'conf'
|
|
66
77
|
SYNC_ARGS = 'args'
|
|
67
78
|
AOC = 'aoc'
|
|
79
|
+
AUTOMATION = 'automation'
|
|
68
80
|
FASPEX = 'faspex'
|
|
69
81
|
FASPIO = 'faspio'
|
|
82
|
+
CONSOLE = 'console'
|
|
70
83
|
NODE = 'node'
|
|
71
84
|
SHARES = 'shares+/api/v1'
|
|
72
85
|
ASYNC_TABLES = 'async_tables'
|
|
@@ -111,7 +124,11 @@ module Aspera
|
|
|
111
124
|
if path&.end_with?(QUERY_PARAMS_SUFFIX)
|
|
112
125
|
parent_path = path.delete_suffix(QUERY_PARAMS_SUFFIX)
|
|
113
126
|
node = @cache[sym].dig(*parent_path.split('.'))
|
|
114
|
-
|
|
127
|
+
# Parameters may be references to components.parameters
|
|
128
|
+
params = (node&.fetch('parameters', []) || []).map do |param|
|
|
129
|
+
param.key?('$ref') ? @cache[sym].dig(*param['$ref'].delete_prefix('#/').split('/')) : param
|
|
130
|
+
end
|
|
131
|
+
return Reader.from_query_params(params.compact)
|
|
115
132
|
end
|
|
116
133
|
reader = Reader.new(@cache[sym])
|
|
117
134
|
return reader unless path
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'singleton'
|
|
4
|
+
require 'json'
|
|
5
|
+
require 'aspera/schema/registry'
|
|
6
|
+
|
|
7
|
+
module Aspera
|
|
8
|
+
module Schema
|
|
9
|
+
# Validate values against the JSON schemas of the registry.
|
|
10
|
+
# Only schemas owned by ascli are validated: vendor API schemas are checked by the API itself.
|
|
11
|
+
# @!method self.instance
|
|
12
|
+
# Returns the singleton instance of Validator
|
|
13
|
+
# @return [Validator] the singleton instance
|
|
14
|
+
class Validator
|
|
15
|
+
include Singleton
|
|
16
|
+
|
|
17
|
+
def initialize
|
|
18
|
+
# compiled documents, key: [registry key, partial]
|
|
19
|
+
@documents = {}
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Validate a value against a schema.
|
|
23
|
+
# @param value [Object] value to validate
|
|
24
|
+
# @param path [String] schema path, e.g. `opts:components.schemas.HttpOptions`
|
|
25
|
+
# @param partial [Boolean] `true`: value may be incomplete, `required` is not enforced, `null` values (removal) are ignored
|
|
26
|
+
# @return [Array<String>] error messages, empty if valid or schema not validated
|
|
27
|
+
def errors(value, path, partial: false)
|
|
28
|
+
return [] unless path.is_a?(String) && Registry.owned?(path)
|
|
29
|
+
key, dotted = path.split(':', 2)
|
|
30
|
+
schema = document(key, partial)
|
|
31
|
+
schema = schema.ref("#/#{dotted.split('.').map { |s| s.gsub('~', '~0').gsub('/', '~1') }.join('/')}") if dotted
|
|
32
|
+
# JSON view of value: symbol keys and values become strings
|
|
33
|
+
data = JSON.parse(JSON.generate(value))
|
|
34
|
+
data = without_null(data) if partial
|
|
35
|
+
schema.validate(data).filter_map { |error| message(error, partial) }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
# @param key [String] registry key
|
|
41
|
+
# @param partial [Boolean] remove `required` from schema
|
|
42
|
+
# @return [JSONSchemer::Schema] compiled schema document
|
|
43
|
+
def document(key, partial)
|
|
44
|
+
@documents[[key, partial]] ||=
|
|
45
|
+
begin
|
|
46
|
+
require 'json_schemer'
|
|
47
|
+
root = Registry.instance.reader(key).current
|
|
48
|
+
root = without_required(root) if partial
|
|
49
|
+
meta =
|
|
50
|
+
case root['openapi']
|
|
51
|
+
when /^3\.0\./ then JSONSchemer.openapi30
|
|
52
|
+
when /^3\.1\./ then JSONSchemer.openapi31
|
|
53
|
+
end
|
|
54
|
+
meta ? JSONSchemer.schema(root, meta_schema: meta) : JSONSchemer.schema(root)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# @param node [Object] schema node
|
|
59
|
+
# @return [Object] copy of node without `required` lists
|
|
60
|
+
def without_required(node)
|
|
61
|
+
case node
|
|
62
|
+
when Hash then node.each_with_object({}) { |(k, v), h| h[k] = without_required(v) unless k.eql?('required') && v.is_a?(Array) }
|
|
63
|
+
when Array then node.map { |e| without_required(e) }
|
|
64
|
+
else node
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# @param node [Object] JSON value
|
|
69
|
+
# @return [Object] copy of node without Hash entries whose value is `null` (entry removal requested by user)
|
|
70
|
+
def without_null(node)
|
|
71
|
+
case node
|
|
72
|
+
when Hash then node.each_with_object({}) { |(k, v), h| h[k] = without_null(v) unless v.nil? }
|
|
73
|
+
when Array then node.map { |e| without_null(e) }
|
|
74
|
+
else node
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# @param error [Hash] json_schemer error
|
|
79
|
+
# @param partial [Boolean] partial validation
|
|
80
|
+
# @return [String, nil] message, or nil if error is ignored
|
|
81
|
+
def message(error, partial)
|
|
82
|
+
return error['error'] unless error['type'].eql?('discriminator')
|
|
83
|
+
property = error['schema'].dig('discriminator', 'propertyName')
|
|
84
|
+
object = error['data']
|
|
85
|
+
# Partial value: selector may come from another source or a default
|
|
86
|
+
return if partial && object.is_a?(Hash) && !object.key?(property)
|
|
87
|
+
allowed = error['schema'].dig('discriminator', 'mapping')&.keys
|
|
88
|
+
"value at `#{error['data_pointer']}/#{property}` is not one of: #{allowed}"
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
data/lib/aspera/secret_hider.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# cspell:ignore FILEPASS
|
|
4
4
|
require 'logger'
|
|
5
|
+
require 'set'
|
|
5
6
|
require 'singleton'
|
|
6
7
|
|
|
7
8
|
module Aspera
|
|
@@ -13,8 +14,6 @@ module Aspera
|
|
|
13
14
|
class SecretHider
|
|
14
15
|
include Singleton
|
|
15
16
|
|
|
16
|
-
# configurable:
|
|
17
|
-
ADDITIONAL_KEYS_TO_HIDE = []
|
|
18
17
|
# display string for hidden secrets
|
|
19
18
|
HIDDEN_PASSWORD = '🔑'
|
|
20
19
|
# env vars for ascp with secrets
|
|
@@ -23,8 +22,8 @@ module Aspera
|
|
|
23
22
|
KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
|
|
24
23
|
HTTP_SECRETS = %w[Authorization].freeze
|
|
25
24
|
ALL_SECRETS = (ASCP_ENV_SECRETS + KEY_SECRETS + HTTP_SECRETS).freeze
|
|
26
|
-
|
|
27
|
-
KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/].freeze
|
|
25
|
+
NON_ENV_SECRETS = (KEY_SECRETS + HTTP_SECRETS).freeze
|
|
26
|
+
KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/, /public_key$/, /^token_type$/].freeze
|
|
28
27
|
# min length of hidden secrets, use `+` or `{n,}`
|
|
29
28
|
SECRET_LENGTH = '{5,}'
|
|
30
29
|
# regex that define named captures :begin and :end
|
|
@@ -32,38 +31,42 @@ module Aspera
|
|
|
32
31
|
# private key values (place first)
|
|
33
32
|
/(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)\n*/,
|
|
34
33
|
# CLI manager get/set options
|
|
35
|
-
/(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)
|
|
34
|
+
/(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/i,
|
|
36
35
|
# env var ascp exec
|
|
37
|
-
/(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)[^ ]+(?<end> )/,
|
|
38
|
-
# rendered JSON or Ruby
|
|
39
|
-
/(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>?(?:=>|:) *")[^"]+(?<end>")
|
|
36
|
+
/(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)[^ \n]+(?<end> |$)/,
|
|
37
|
+
# rendered JSON or Ruby (quoted key or legacy symbol key: `"k":"v"`, `"k" => "v"`, `:k=>"v"`)
|
|
38
|
+
/(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>? *(?:=>|:) *")(?:[^"\\]|\\.)+(?<end>")/i,
|
|
39
|
+
# rendered Ruby 3.4+ symbol key: `k: "v"`
|
|
40
|
+
/(?<begin>\b\w*(?:#{ALL_SECRETS.join('|')})\w*: *")(?:[^"\\]|\\.)+(?<end>")/i,
|
|
40
41
|
# logged data
|
|
41
|
-
/(?<begin>(?:#{
|
|
42
|
+
/(?<begin>(?:#{NON_ENV_SECRETS.join('|')})[ =:]+)[^ "\n]#{SECRET_LENGTH}(?<end>$)/i,
|
|
42
43
|
# cred in http dump
|
|
43
44
|
/(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
|
|
44
45
|
].freeze
|
|
45
|
-
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :KEY_FALSE_POSITIVES, :REGEX_LOG_REPLACES
|
|
46
|
+
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :SECRET_LENGTH, :REGEX_LOG_REPLACES
|
|
46
47
|
attr_accessor :log_secrets
|
|
47
48
|
|
|
48
49
|
# @return [Proc] new log formatter that hides secrets
|
|
49
50
|
def log_formatter(original_formatter)
|
|
50
51
|
original_formatter ||= Logger::Formatter.new
|
|
51
52
|
# NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
|
53
|
+
# Hiding is applied on the formatted line, so that non-String messages (e.g. Exception) are covered too
|
|
52
54
|
return lambda do |severity, date_time, program_name, msg|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
original_formatter.call(severity, date_time, program_name, msg)
|
|
55
|
+
line = original_formatter.call(severity, date_time, program_name, msg)
|
|
56
|
+
line = hide_secrets_in_string(line, all: true) if line.is_a?(String) && !@log_secrets
|
|
57
|
+
line
|
|
59
58
|
end
|
|
60
59
|
end
|
|
61
60
|
|
|
62
|
-
# Replace
|
|
63
|
-
# @param value [String] Input string possibly containing
|
|
64
|
-
# @
|
|
65
|
-
|
|
66
|
-
|
|
61
|
+
# Replace secrets in a string with the hidden password placeholder
|
|
62
|
+
# @param value [String] Input string possibly containing secrets
|
|
63
|
+
# @param all [Boolean] `false`: only private keys, `true`: all known secret patterns
|
|
64
|
+
# @return [String] String with secrets replaced by placeholder
|
|
65
|
+
def hide_secrets_in_string(value, all: false)
|
|
66
|
+
(all ? REGEX_LOG_REPLACES : REGEX_LOG_REPLACES.first(1)).each do |reg_ex|
|
|
67
|
+
value = value.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}" }
|
|
68
|
+
end
|
|
69
|
+
return value
|
|
67
70
|
end
|
|
68
71
|
|
|
69
72
|
# @param keyword [String, Symbol] Key name to check
|
|
@@ -73,11 +76,12 @@ module Aspera
|
|
|
73
76
|
keyword = keyword.to_s if keyword.is_a?(Symbol)
|
|
74
77
|
# only Strings can be secrets, not booleans, or hash, arrays
|
|
75
78
|
return false unless keyword.is_a?(String) && value.is_a?(String)
|
|
79
|
+
return true if @additional_keys.include?(keyword)
|
|
80
|
+
keyword = keyword.downcase
|
|
76
81
|
# those are not secrets
|
|
77
82
|
return false if KEY_FALSE_POSITIVES.any? { |f| f.match?(keyword) }
|
|
78
|
-
return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
|
|
79
83
|
# check if keyword (name) contains an element that designate it as a secret
|
|
80
|
-
ALL_SECRETS.any? { |kw| keyword.include?(kw) }
|
|
84
|
+
ALL_SECRETS.any? { |kw| keyword.include?(kw.downcase) }
|
|
81
85
|
end
|
|
82
86
|
|
|
83
87
|
# Hides recursively secrets in Hash or Array of Hash
|
|
@@ -89,18 +93,25 @@ module Aspera
|
|
|
89
93
|
obj.each do |k, v|
|
|
90
94
|
if secret?(k, v)
|
|
91
95
|
obj[k] = HIDDEN_PASSWORD
|
|
92
|
-
elsif
|
|
93
|
-
deep_remove_secret(
|
|
96
|
+
elsif v.is_a?(Hash) || v.is_a?(Array)
|
|
97
|
+
deep_remove_secret(v)
|
|
94
98
|
end
|
|
95
99
|
end
|
|
96
100
|
end
|
|
97
101
|
return obj
|
|
98
102
|
end
|
|
99
103
|
|
|
104
|
+
# Declare additional exact key names whose values are secrets
|
|
105
|
+
# @param keys [Array<String, Symbol>] Key names
|
|
106
|
+
def add_secret_keys(keys)
|
|
107
|
+
@additional_keys.merge(keys.map(&:to_s))
|
|
108
|
+
end
|
|
109
|
+
|
|
100
110
|
private
|
|
101
111
|
|
|
102
112
|
def initialize
|
|
103
113
|
@log_secrets = false
|
|
114
|
+
@additional_keys = Set.new
|
|
104
115
|
end
|
|
105
116
|
end
|
|
106
117
|
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class ::String
|
|
4
|
+
# Transform capitalized to snake case
|
|
5
|
+
def capital_to_snake
|
|
6
|
+
return gsub(/([a-z\d])([A-Z])/, '\1_\2')
|
|
7
|
+
.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
|
|
8
|
+
.downcase
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# Transform snake case to capitalized
|
|
12
|
+
def snake_to_capital
|
|
13
|
+
split('_').map(&:capitalize).join
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'singleton'
|
|
4
4
|
require 'fileutils'
|
|
5
5
|
require 'etc'
|
|
6
|
-
require '
|
|
6
|
+
require 'tmpdir'
|
|
7
7
|
|
|
8
8
|
module Aspera
|
|
9
9
|
# create a temp file name for a given folder
|
|
@@ -58,10 +58,11 @@ module Aspera
|
|
|
58
58
|
def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
|
|
59
59
|
FileUtils.mkdir_p(temp_folder)
|
|
60
60
|
basename = [prefix || 'aspera', SecureRandom.uuid].join('-')
|
|
61
|
-
#
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
# Create the file atomically with O_EXCL (retry with another name if it exists)
|
|
62
|
+
# Note: Tempfile is not used, as its file is deleted when the object is garbage collected
|
|
63
|
+
new_file = Dir::Tmpname.create([basename, suffix ? "-#{suffix}" : ''], temp_folder) do |path|
|
|
64
|
+
File.open(path, File::WRONLY | File::CREAT | File::EXCL, 0o600) { nil }
|
|
65
|
+
end
|
|
65
66
|
@created_files.push(new_file)
|
|
66
67
|
new_file
|
|
67
68
|
end
|
data/lib/aspera/transfer/spec.rb
CHANGED
|
@@ -25,6 +25,7 @@ module Aspera
|
|
|
25
25
|
TRANSPORT_FIELDS = (%w[remote_host] + AK_TSPEC_BASE.keys + WSS_FIELDS).freeze
|
|
26
26
|
# reserved tag for Aspera
|
|
27
27
|
TAG_RESERVED = 'aspera'
|
|
28
|
+
# Specific fields to not display
|
|
28
29
|
SPECIFIC = %w{token paths direction source_root destination_root}.freeze
|
|
29
30
|
class << self
|
|
30
31
|
# wrong def in transferd
|
data/lib/aspera/uri_reader.rb
CHANGED
|
@@ -22,7 +22,7 @@ module Aspera
|
|
|
22
22
|
#
|
|
23
23
|
# The short form is consistent with RFC 8089 (file:///<path> = absolute,
|
|
24
24
|
# file:<path> = relative). Both forms are handled identically by this module.
|
|
25
|
-
# The canonical form built by {file_url} uses the
|
|
25
|
+
# The canonical form built by {file_url} uses the `file:///` prefix.
|
|
26
26
|
module UriReader
|
|
27
27
|
SCHEME_FILE = 'file'
|
|
28
28
|
SCHEME_FILE_PFX1 = "#{SCHEME_FILE}:"
|
|
@@ -31,13 +31,13 @@ module Aspera
|
|
|
31
31
|
SCHEME_FILE_PFX2 = "#{SCHEME_FILE_PFX1}///"
|
|
32
32
|
private_constant :SCHEME_FILE, :SCHEME_FILE_PFX1, :SCHEME_FILE_PFX2
|
|
33
33
|
class << self
|
|
34
|
-
# @return [Boolean] true if
|
|
34
|
+
# @return [Boolean] true if `url` uses the file: scheme recognised by this module
|
|
35
35
|
def file?(url)
|
|
36
36
|
url.start_with?(SCHEME_FILE_PFX1)
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
-
# Build a file: URL from
|
|
40
|
-
# A relative path yields
|
|
39
|
+
# Build a file: URL from `path`.
|
|
40
|
+
# A relative path yields `file:///path`; an absolute path yields `file:////path`.
|
|
41
41
|
# @param path [String] relative or absolute file-system path
|
|
42
42
|
# @return [String] corresponding file: URL
|
|
43
43
|
def file_url(path)
|
|
@@ -45,7 +45,7 @@ module Aspera
|
|
|
45
45
|
end
|
|
46
46
|
|
|
47
47
|
# Extract the file-system path from a file: URL.
|
|
48
|
-
# Accepts both the canonical
|
|
48
|
+
# Accepts both the canonical `file:///` form and the short `file:` form.
|
|
49
49
|
# Returns the literal path (relative or absolute) without working-directory expansion.
|
|
50
50
|
# @param url [String] a file: URL (canonical or short form)
|
|
51
51
|
# @return [String] the literal path encoded in the URL
|
|
@@ -57,12 +57,12 @@ module Aspera
|
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
# Read content from a URI and return it as a String.
|
|
60
|
-
# Supported schemes:
|
|
60
|
+
# Supported schemes: `http`, `https`, `data`, `file`, and bare paths (no scheme).
|
|
61
61
|
# For file: URLs the path is extracted via {file_path} to respect the module convention
|
|
62
62
|
# (see module-level documentation). Ruby's URI parser is not used for file: URLs because
|
|
63
63
|
# it interprets the three-slash prefix differently (it always produces an absolute path).
|
|
64
|
-
# Bare paths (no scheme) are passed to File.read directly; leading
|
|
65
|
-
# are expanded via
|
|
64
|
+
# Bare paths (no scheme) are passed to File.read directly; leading `/~/`, `/./`, `/../`
|
|
65
|
+
# are expanded via `File.expand_path` after stripping the synthetic leading slash added by
|
|
66
66
|
# URI.
|
|
67
67
|
def read(uri_to_read)
|
|
68
68
|
# Handle file: URLs directly to honour the file:///relative vs file:////absolute convention.
|
|
@@ -70,7 +70,7 @@ module Aspera
|
|
|
70
70
|
uri = URI.parse(uri_to_read)
|
|
71
71
|
case uri.scheme
|
|
72
72
|
when 'http', 'https'
|
|
73
|
-
return Rest.new(base_url: uri_to_read, redirect_max: 5).read(nil, headers: {'Accept' => '*/*'})
|
|
73
|
+
return Rest::Client.new(base_url: uri_to_read, redirect_max: 5).read(nil, headers: {'Accept' => '*/*'})
|
|
74
74
|
when 'data'
|
|
75
75
|
metadata, encoded_data = uri.opaque.split(',', 2)
|
|
76
76
|
if metadata.end_with?(';base64')
|
|
@@ -87,7 +87,7 @@ module Aspera
|
|
|
87
87
|
end
|
|
88
88
|
end
|
|
89
89
|
|
|
90
|
-
# Return the local file-system path for the content at
|
|
90
|
+
# Return the local file-system path for the content at `url`, downloading to a temp file if needed.
|
|
91
91
|
# For file: URLs the path is extracted directly (no download).
|
|
92
92
|
# For data: and http(s): URLs the content is written to a temporary file and its path is returned.
|
|
93
93
|
# @return [String] local path to a file containing the URL content
|
|
@@ -105,7 +105,7 @@ module Aspera
|
|
|
105
105
|
# download to temp file
|
|
106
106
|
# auto-delete on exit
|
|
107
107
|
temp_file = TempFileManager.instance.new_file_path_global(suffix: File.basename(url))
|
|
108
|
-
Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to: temp_file)
|
|
108
|
+
Aspera::Rest::Client.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to: temp_file)
|
|
109
109
|
return temp_file
|
|
110
110
|
end
|
|
111
111
|
end
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
|
|
4
|
+
<head>
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<link rel="icon" type="image/svg+xml"
|
|
7
|
+
href="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIzMiIgaGVpZ2h0PSIzMiIgdmlld0JveD0iMCAwIDMyIDMyIiBmaWxsPSJub25lIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iMyI+CiAgPGxpbmUgeDE9IjMiIHkxPSIzIiB4Mj0iMjkiIHkyPSIyOSIgIHN0cm9rZT0icmVkIi8+CiAgPGxpbmUgeDE9IjI5IiB5MT0iMyIgeDI9IjMiIHkyPSIyOSIgc3Ryb2tlPSJyZWQiIC8+Cjwvc3ZnPg==">
|
|
8
|
+
<title>Close Now</title>
|
|
9
|
+
<style>
|
|
10
|
+
body {
|
|
11
|
+
font-family: Arial, sans-serif;
|
|
12
|
+
text-align: center;
|
|
13
|
+
padding: 2rem;
|
|
14
|
+
margin: 0;
|
|
15
|
+
background: linear-gradient(135deg, #f0f4f8, #d9e2ec);
|
|
16
|
+
color: #333;
|
|
17
|
+
overflow: hidden;
|
|
18
|
+
/* Ensure no scrollbars for the background animation */
|
|
19
|
+
position: relative;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
h1 {
|
|
23
|
+
font-size: 2.5rem;
|
|
24
|
+
color: #0078d4;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
p {
|
|
28
|
+
font-size: 1.2rem;
|
|
29
|
+
margin-top: 1rem;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/* Styling for animated IBM logos */
|
|
33
|
+
.logo {
|
|
34
|
+
position: absolute;
|
|
35
|
+
bottom: -100px;
|
|
36
|
+
width: 40px;
|
|
37
|
+
height: 40px;
|
|
38
|
+
background: none;
|
|
39
|
+
display: flex;
|
|
40
|
+
justify-content: center;
|
|
41
|
+
align-items: center;
|
|
42
|
+
animation: rise 10s infinite ease-in-out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
.logo svg {
|
|
46
|
+
width: 100%;
|
|
47
|
+
height: 100%;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
.logo:nth-child(odd) {
|
|
51
|
+
animation-duration: 8s;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
.logo:nth-child(even) {
|
|
55
|
+
animation-duration: 12s;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
@keyframes rise {
|
|
59
|
+
0% {
|
|
60
|
+
transform: translateY(0) scale(1);
|
|
61
|
+
opacity: 1;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
50% {
|
|
65
|
+
opacity: 0.7;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
100% {
|
|
69
|
+
transform: translateY(-120vh) scale(0.7);
|
|
70
|
+
opacity: 0;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
</style>
|
|
74
|
+
</head>
|
|
75
|
+
|
|
76
|
+
<body>
|
|
77
|
+
<h1>Thank You!</h1>
|
|
78
|
+
<p>You can close this window.</p>
|
|
79
|
+
__ADD_INFO__
|
|
80
|
+
|
|
81
|
+
<!-- JavaScript to generate IBM logos -->
|
|
82
|
+
<script>
|
|
83
|
+
// Function to create logos dynamically
|
|
84
|
+
function createLogos() {
|
|
85
|
+
const body = document.body;
|
|
86
|
+
const svgContent = `
|
|
87
|
+
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="1000px" height="401.149px" viewBox="0 0 1000 401.149" xml:space="preserve">
|
|
88
|
+
<g>
|
|
89
|
+
<rect fill="#1F70C1" x="0" y="373.217" width="194.433" height="27.833"/>
|
|
90
|
+
<rect fill="#1F70C1" x="0" y="319.83" width="194.433" height="27.931"/>
|
|
91
|
+
<rect fill="#1F70C1" x="55.468" y="266.541" width="83.399" height="27.932"/>
|
|
92
|
+
<rect fill="#1F70C1" x="55.468" y="213.253" width="83.399" height="27.932"/>
|
|
93
|
+
<rect fill="#1F70C1" x="55.468" y="159.964" width="83.399" height="27.932"/>
|
|
94
|
+
<rect fill="#1F70C1" x="55.468" y="106.577" width="83.399" height="27.932"/>
|
|
95
|
+
<rect fill="#1F70C1" x="0" y="53.288" width="194.433" height="27.932"/>
|
|
96
|
+
<rect fill="#1F70C1" x="0" y="0" width="194.433" height="27.932"/>
|
|
97
|
+
<path fill="#1F70C1" d="M222.167,347.761h299.029c5.051-8.617,8.815-18.027,11.094-27.932H222.167V347.761z"/>
|
|
98
|
+
<path fill="#1F70C1" d="M497.92,213.253H277.734v27.932h243.463C514.857,230.487,507.032,221.078,497.92,213.253z"/>
|
|
99
|
+
<path fill="#1F70C1" d="M277.734,159.964v27.932H497.92c9.311-7.825,17.135-17.235,23.277-27.932H277.734z"/>
|
|
100
|
+
<path fill="#1F70C1" d="M521.197,53.288H222.167V81.22H532.29C529.715,71.315,525.951,61.906,521.197,53.288z"/>
|
|
101
|
+
<path fill="#1F70C1" d="M429.279,0H222.167v27.932h278.526C482.072,10.697,456.815,0,429.279,0z"/>
|
|
102
|
+
<rect fill="#1F70C1" x="277.734" y="106.577" width="83.3" height="27.932"/>
|
|
103
|
+
<path fill="#1F70C1" d="M444.433,134.509h87.163c2.476-8.914,3.764-18.324,3.764-27.932h-90.927z"/>
|
|
104
|
+
<rect fill="#1F70C1" x="277.734" y="266.541" width="83.3" height="27.932"/>
|
|
105
|
+
<path fill="#1F70C1" d="M444.433,266.541v27.932h90.927c0-9.608-1.288-19.017-3.764-27.932H444.433z"/>
|
|
106
|
+
<path fill="#1F70C1" d="M222.167,400.852h207.112c27.734,0,52.793-10.697,71.513-27.932H222.167V400.852z"/>
|
|
107
|
+
<rect fill="#1F70C1" x="555.567" y="373.217" width="138.866" height="27.833"/>
|
|
108
|
+
<rect fill="#1F70C1" x="555.567" y="319.83" width="138.866" height="27.931"/>
|
|
109
|
+
<rect fill="#1F70C1" x="611.034" y="266.541" width="83.399" height="27.932"/>
|
|
110
|
+
<rect fill="#1F70C1" x="611.034" y="213.253" width="83.399" height="27.932"/>
|
|
111
|
+
<polygon fill="#1F70C1" points="733.063,53.288 555.567,53.288 555.567,81.22 742.67,81.22"/>
|
|
112
|
+
<polygon fill="#1F70C1" points="714.639,0 555.567,0 555.567,27.932 724.247,27.932"/>
|
|
113
|
+
<rect fill="#1F70C1" x="861.034" y="373.217" width="138.866" height="27.833"/>
|
|
114
|
+
<rect fill="#1F70C1" x="861.034" y="319.83" width="138.866" height="27.931"/>
|
|
115
|
+
<rect fill="#1F70C1" x="861.034" y="266.541" width="83.399" height="27.932"/>
|
|
116
|
+
<rect fill="#1F70C1" x="861.034" y="213.253" width="83.399" height="27.932"/>
|
|
117
|
+
<polygon fill="#1F70C1" points="861.034,187.896 944.433,187.896 944.433,159.964 861.034,159.964 694.433,159.964 611.034,159.964 611.034,187.896 694.433,187.896 852.219,187.896"/>
|
|
118
|
+
<polygon fill="#1F70C1" points="944.433,106.577 803.982,106.577 794.374,134.509 944.433,134.509"/>
|
|
119
|
+
<polygon fill="#1F70C1" points="840.927,0 831.319,27.932 1000,27.932 1000,0"/>
|
|
120
|
+
<polygon fill="#1F70C1" points="777.734,400.852 787.341,373.217 768.126,373.217"/>
|
|
121
|
+
<polygon fill="#1F70C1" points="759.311,347.761 796.157,347.761 806.062,319.83 749.505,319.83"/>
|
|
122
|
+
<polygon fill="#1F70C1" points="740.59,294.473 814.877,294.473 824.683,266.541 730.784,266.541"/>
|
|
123
|
+
<polygon fill="#1F70C1" points="721.969,241.185 833.597,241.185 843.106,213.253 712.361,213.253"/>
|
|
124
|
+
<polygon fill="#1F70C1" points="611.034,134.509 761.093,134.509 751.486,106.577 611.034,106.577"/>
|
|
125
|
+
<polygon fill="#1F70C1" points="812.896,81.22 1000,81.22 1000,53.288 822.405,53.288"/>
|
|
126
|
+
</g>
|
|
127
|
+
</svg>
|
|
128
|
+
`;
|
|
129
|
+
for (let i = 0; i < 20; i++) {
|
|
130
|
+
const logo = document.createElement('div');
|
|
131
|
+
logo.className = 'logo';
|
|
132
|
+
const size = Math.random() * 30 + 20; // Random size between 20px and 50px
|
|
133
|
+
logo.style.width = `${size}px`;
|
|
134
|
+
logo.style.height = `${size}px`;
|
|
135
|
+
logo.style.left = `${Math.random() * 100}vw`;
|
|
136
|
+
logo.style.animationDelay = `${Math.random() * 5}s`;
|
|
137
|
+
logo.innerHTML = svgContent;
|
|
138
|
+
body.appendChild(logo);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Call the function to create logos on load
|
|
143
|
+
createLogos();
|
|
144
|
+
</script>
|
|
145
|
+
</body>
|
|
146
|
+
|
|
147
|
+
</html>
|