aspera-cli 4.27.2 → 4.27.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +133 -0
- data/CONTRIBUTING.md +5 -2
- data/bin/ascli +3 -1
- data/docs/README.md +1006 -766
- data/lib/aspera/agent/base.rb +7 -2
- data/lib/aspera/agent/connect.rb +6 -8
- data/lib/aspera/agent/desktop.rb +2 -6
- data/lib/aspera/agent/direct.rb +52 -22
- data/lib/aspera/agent/node.rb +9 -8
- data/lib/aspera/agent/transferd.rb +2 -2
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -12
- data/lib/aspera/api/ats.rb +1 -1
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +11 -7
- data/lib/aspera/api/httpgw.rb +38 -35
- data/lib/aspera/api/node.rb +39 -34
- data/lib/aspera/ascmd.rb +3 -1
- data/lib/aspera/ascp/installation.rb +63 -28
- data/lib/aspera/ascp/management.rb +1 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +20 -41
- data/lib/aspera/cli/async_transfer_store.rb +12 -11
- data/lib/aspera/cli/bootstrapper.rb +14 -16
- data/lib/aspera/cli/command_line.rb +252 -0
- data/lib/aspera/cli/command_registry.rb +215 -37
- data/lib/aspera/cli/command_spec.rb +104 -15
- data/lib/aspera/cli/completion/ascli.bash +12 -0
- data/lib/aspera/cli/completion/ascli.fish +16 -0
- data/lib/aspera/cli/completion/ascli.zsh +19 -0
- data/lib/aspera/cli/context.rb +3 -0
- data/lib/aspera/cli/deprecation.rb +37 -0
- data/lib/aspera/cli/extended_value.rb +6 -3
- data/lib/aspera/cli/formatter.rb +94 -80
- data/lib/aspera/cli/gem_checker.rb +1 -1
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/http.rb +22 -34
- data/lib/aspera/cli/info.rb +3 -0
- data/lib/aspera/cli/mcp_tool.rb +47 -83
- data/lib/aspera/cli/option_declarator.rb +33 -42
- data/lib/aspera/cli/option_registry.rb +69 -0
- data/lib/aspera/cli/option_types.rb +105 -0
- data/lib/aspera/cli/option_value.rb +278 -0
- data/lib/aspera/cli/options.schema.yaml +124 -15
- data/lib/aspera/cli/parser.rb +333 -862
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +545 -518
- data/lib/aspera/cli/plugins/ats.rb +59 -80
- data/lib/aspera/cli/plugins/base.rb +221 -265
- data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
- data/lib/aspera/cli/plugins/config.rb +263 -184
- data/lib/aspera/cli/plugins/console.rb +103 -39
- data/lib/aspera/cli/plugins/cos.rb +6 -23
- data/lib/aspera/cli/plugins/factory.rb +3 -0
- data/lib/aspera/cli/plugins/faspex5.rb +204 -182
- data/lib/aspera/cli/plugins/faspio.rb +6 -11
- data/lib/aspera/cli/plugins/httpgw.rb +8 -11
- data/lib/aspera/cli/plugins/mcp.rb +20 -55
- data/lib/aspera/cli/plugins/node.rb +300 -327
- data/lib/aspera/cli/plugins/orchestrator.rb +152 -110
- data/lib/aspera/cli/plugins/preview.rb +96 -105
- data/lib/aspera/cli/plugins/server.rb +78 -53
- data/lib/aspera/cli/plugins/shares.rb +80 -131
- data/lib/aspera/cli/preset_actions.rb +44 -27
- data/lib/aspera/cli/preset_manager.rb +44 -19
- data/lib/aspera/cli/prompt.rb +36 -0
- data/lib/aspera/cli/result.rb +42 -36
- data/lib/aspera/cli/runner.rb +32 -59
- data/lib/aspera/cli/special_values.rb +5 -0
- data/lib/aspera/cli/sync_actions.rb +51 -46
- data/lib/aspera/cli/terminal_formatter.rb +9 -3
- data/lib/aspera/cli/transfer_actions.rb +14 -9
- data/lib/aspera/cli/transfer_agent.rb +34 -38
- data/lib/aspera/cli/transfer_progress.rb +290 -55
- data/lib/aspera/cli/vault_manager.rb +0 -17
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +5 -3
- data/lib/aspera/coverage.rb +1 -1
- data/lib/aspera/environment.rb +35 -5
- data/lib/aspera/faspex_gw.rb +2 -1
- data/lib/aspera/faspex_postproc.rb +1 -0
- data/lib/aspera/graphql.rb +5 -5
- data/lib/aspera/json_rpc/client.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +2 -2
- data/lib/aspera/keychain/factory.rb +2 -1
- data/lib/aspera/keychain/one_password_api.rb +1 -1
- data/lib/aspera/link_header.rb +2 -2
- data/lib/aspera/log.rb +47 -27
- data/lib/aspera/markdown.rb +2 -0
- data/lib/aspera/mime.rb +25 -0
- data/lib/aspera/node_emulator.rb +759 -0
- data/lib/aspera/oauth/base.rb +37 -26
- data/lib/aspera/oauth/factory.rb +7 -3
- data/lib/aspera/oauth/generic.rb +1 -1
- data/lib/aspera/oauth/json_credentials.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +4 -5
- data/lib/aspera/oauth/web.rb +9 -8
- data/lib/aspera/oauth.rb +1 -0
- data/lib/aspera/persistency_folder.rb +1 -3
- data/lib/aspera/preview/file_types.rb +4 -4
- data/lib/aspera/preview/generator.rb +11 -1
- data/lib/aspera/preview/options.schema.yaml +119 -0
- data/lib/aspera/preview/terminal.rb +4 -3
- data/lib/aspera/preview/utils.rb +9 -6
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/rainbow.rb +7 -0
- data/lib/aspera/rest/aspera_errors.rb +72 -0
- data/lib/aspera/rest/call_error.rb +27 -0
- data/lib/aspera/rest/client.rb +523 -0
- data/lib/aspera/rest/error_analyzer.rb +113 -0
- data/lib/aspera/rest/list.rb +149 -0
- data/lib/aspera/rest/parameters.rb +55 -0
- data/lib/aspera/rest/util.rb +176 -0
- data/lib/aspera/rest.rb +7 -621
- data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -20
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +1784 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1230 -137
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +14 -13
- data/lib/aspera/schema/documentation.rb +13 -3
- data/lib/aspera/schema/reader.rb +12 -18
- data/lib/aspera/schema/registry.rb +23 -1
- data/lib/aspera/schema/validator.rb +92 -0
- data/lib/aspera/secret_hider.rb +36 -25
- data/lib/aspera/string_ext.rb +15 -0
- data/lib/aspera/temp_file_manager.rb +6 -5
- data/lib/aspera/transfer/parameters.rb +2 -0
- data/lib/aspera/transfer/spec.rb +1 -0
- data/lib/aspera/uri_reader.rb +11 -11
- data/lib/aspera/web_auth/index.html +147 -0
- data/lib/aspera/web_auth/server.rb +81 -0
- data.tar.gz.sig +0 -0
- metadata +43 -9
- metadata.gz.sig +0 -0
- data/lib/aspera/colors.rb +0 -79
- data/lib/aspera/node_simulator.rb +0 -344
- data/lib/aspera/preview/options.rb +0 -45
- data/lib/aspera/rest_call_error.rb +0 -25
- data/lib/aspera/rest_error_analyzer.rb +0 -111
- data/lib/aspera/rest_errors_aspera.rb +0 -58
- data/lib/aspera/rest_list.rb +0 -136
- data/lib/aspera/web_auth.rb +0 -211
|
@@ -2930,21 +2930,25 @@ components:
|
|
|
2930
2930
|
name:
|
|
2931
2931
|
type: string
|
|
2932
2932
|
minLength: 1
|
|
2933
|
-
description:
|
|
2934
|
-
type: string
|
|
2935
2933
|
node_id:
|
|
2936
2934
|
type: integer
|
|
2937
|
-
|
|
2935
|
+
directory:
|
|
2938
2936
|
type: string
|
|
2937
|
+
description: Path of share folder on node
|
|
2938
|
+
create_directory:
|
|
2939
|
+
type: boolean
|
|
2940
|
+
description: Create the share folder on node
|
|
2939
2941
|
ShareUpdate:
|
|
2940
2942
|
type: object
|
|
2941
2943
|
properties:
|
|
2942
2944
|
name:
|
|
2943
2945
|
type: string
|
|
2944
|
-
|
|
2945
|
-
type:
|
|
2946
|
-
|
|
2946
|
+
node_id:
|
|
2947
|
+
type: integer
|
|
2948
|
+
description: Admin only
|
|
2949
|
+
directory:
|
|
2947
2950
|
type: string
|
|
2951
|
+
description: Path of share folder on node (admin only)
|
|
2948
2952
|
# Node Schemas
|
|
2949
2953
|
Node:
|
|
2950
2954
|
type: object
|
|
@@ -3206,9 +3210,10 @@ components:
|
|
|
3206
3210
|
name:
|
|
3207
3211
|
type: string
|
|
3208
3212
|
example: engineers
|
|
3209
|
-
|
|
3210
|
-
type:
|
|
3211
|
-
|
|
3213
|
+
group_id:
|
|
3214
|
+
type: integer
|
|
3215
|
+
format: int32
|
|
3216
|
+
description: Identifier in groups from all sources
|
|
3212
3217
|
created_at:
|
|
3213
3218
|
type: string
|
|
3214
3219
|
format: date-time
|
|
@@ -3224,16 +3229,12 @@ components:
|
|
|
3224
3229
|
name:
|
|
3225
3230
|
type: string
|
|
3226
3231
|
example: engineers
|
|
3227
|
-
description:
|
|
3228
|
-
type: string
|
|
3229
3232
|
LocalGroupUpdate:
|
|
3230
3233
|
type: object
|
|
3231
3234
|
properties:
|
|
3232
3235
|
name:
|
|
3233
3236
|
type: string
|
|
3234
3237
|
example: engineers
|
|
3235
|
-
description:
|
|
3236
|
-
type: string
|
|
3237
3238
|
LdapConfig:
|
|
3238
3239
|
type: object
|
|
3239
3240
|
properties:
|
|
@@ -114,9 +114,9 @@ module Aspera
|
|
|
114
114
|
end
|
|
115
115
|
# Only keep lines that are usable in supported agents
|
|
116
116
|
next if agents.empty?
|
|
117
|
-
item['description'].push("Allowed values: #{node['enum'].map { |v|
|
|
118
|
-
item['description'].push("Default: #{
|
|
119
|
-
item['description'].push("Example: #{
|
|
117
|
+
item['description'].push("Allowed values: #{node['enum'].map { |v| value_text(v) }.join(', ')}.") if node.key?('enum')
|
|
118
|
+
item['description'].push("Default: #{value_text(node['default'])}.") if node.key?('default')
|
|
119
|
+
item['description'].push("Example: #{value_text(node['example'])}.") if node.key?('example')
|
|
120
120
|
if @include_option
|
|
121
121
|
envvar_prefix = ''
|
|
122
122
|
cli_option =
|
|
@@ -139,6 +139,16 @@ module Aspera
|
|
|
139
139
|
end
|
|
140
140
|
self
|
|
141
141
|
end
|
|
142
|
+
|
|
143
|
+
private
|
|
144
|
+
|
|
145
|
+
# Format a schema value (enum, default, example) as code
|
|
146
|
+
# @param value [Object] value from schema, `nil` for JSON `null`
|
|
147
|
+
# @return [String]
|
|
148
|
+
def value_text(value)
|
|
149
|
+
return @formatter.special_format('null') if value.nil?
|
|
150
|
+
@formatter.markdown_text("`#{value}`")
|
|
151
|
+
end
|
|
142
152
|
end
|
|
143
153
|
end
|
|
144
154
|
end
|
data/lib/aspera/schema/reader.rb
CHANGED
|
@@ -116,15 +116,12 @@ module Aspera
|
|
|
116
116
|
# Yield current property to block
|
|
117
117
|
yield(property_schema, name, property_full_name)
|
|
118
118
|
|
|
119
|
-
# Recursively process nested structures
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
if
|
|
125
|
-
array_item_schema = property_schema.dig('items')
|
|
126
|
-
array_item_schema.each_property("#{property_full_name}[].", on_variant: on_variant, &block) if array_item_schema.current['properties']
|
|
127
|
-
end
|
|
119
|
+
# Recursively process nested structures (`type` is a list for a nullable value, e.g. `[array, 'null']`)
|
|
120
|
+
types = Array(node['type'])
|
|
121
|
+
property_schema.each_property("#{property_full_name}.", on_variant: on_variant, &block) if types.include?('object') && node['properties']
|
|
122
|
+
if types.include?('array') && node['items']
|
|
123
|
+
array_item_schema = property_schema.dig('items')
|
|
124
|
+
array_item_schema.each_property("#{property_full_name}[].", on_variant: on_variant, &block) if array_item_schema.current['properties']
|
|
128
125
|
end
|
|
129
126
|
# allOf without explicit type: object — recurse to merge all branches
|
|
130
127
|
property_schema.each_property("#{property_full_name}.", on_variant: on_variant, &block) if node['allOf']
|
|
@@ -180,15 +177,12 @@ module Aspera
|
|
|
180
177
|
row['default'] = node['default'] if node.key?('default')
|
|
181
178
|
row['enum'] = node['enum'] if node.key?('enum')
|
|
182
179
|
rows << row
|
|
183
|
-
# Recurse into nested object or array-of-objects
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
if
|
|
189
|
-
item_reader = prop_reader.dig('items')
|
|
190
|
-
collect_rows(rows, item_reader, "#{full_name}[].") if item_reader.current['properties']
|
|
191
|
-
end
|
|
180
|
+
# Recurse into nested object or array-of-objects (`type` is a list for a nullable value)
|
|
181
|
+
types = Array(node['type'])
|
|
182
|
+
collect_rows(rows, prop_reader, "#{full_name}.") if types.include?('object') && node['properties']
|
|
183
|
+
if types.include?('array') && node['items']
|
|
184
|
+
item_reader = prop_reader.dig('items')
|
|
185
|
+
collect_rows(rows, item_reader, "#{full_name}[].") if item_reader.current['properties']
|
|
192
186
|
end
|
|
193
187
|
collect_rows(rows, prop_reader, "#{full_name}.") if node['allOf']
|
|
194
188
|
end
|
|
@@ -18,6 +18,12 @@ module Aspera
|
|
|
18
18
|
LOCATIONS.key?(sym)
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
+
# @param name_path [String] schema path, e.g. `opts:components.schemas.HttpOptions`
|
|
22
|
+
# @return [Boolean] `true` if schema is owned by ascli (not a vendor API)
|
|
23
|
+
def owned?(name_path)
|
|
24
|
+
OWNED.include?(name_path.split(':', 2).first.to_sym)
|
|
25
|
+
end
|
|
26
|
+
|
|
21
27
|
# Split a component string into registry key and optional path prefix.
|
|
22
28
|
# Syntax: 'key' or 'key+/prefix' (e.g. 'shares+/api/v1')
|
|
23
29
|
# @param component [String] registry key with optional '+/prefix'
|
|
@@ -52,27 +58,39 @@ module Aspera
|
|
|
52
58
|
args: 'aspera/sync/args.schema.yaml',
|
|
53
59
|
conf: 'aspera/sync/conf.schema.yaml',
|
|
54
60
|
opts: 'aspera/cli/options.schema.yaml',
|
|
61
|
+
preview: 'aspera/preview/options.schema.yaml',
|
|
55
62
|
aoc: 'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
|
|
63
|
+
automation: 'aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml',
|
|
56
64
|
faspex: 'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml',
|
|
57
65
|
faspio: 'aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml',
|
|
66
|
+
console: 'aspera/schema/IBM Aspera Console-enhanced.yaml',
|
|
58
67
|
node: 'aspera/schema/IBM Aspera Node API-4.4.6.yaml',
|
|
59
68
|
shares: 'aspera/schema/IBM_Aspera_Shares.yaml',
|
|
69
|
+
orchestrator: 'aspera/schema/IBM Aspera Orchestrator API-v1.yaml',
|
|
60
70
|
async_tables: 'aspera/schema/async_tables.yaml'
|
|
61
71
|
}
|
|
62
72
|
|
|
73
|
+
# Schemas owned by ascli: values are validated against them (vendor API schemas are validated by the API)
|
|
74
|
+
OWNED = %i[spec args conf opts preview async_tables].freeze
|
|
75
|
+
|
|
63
76
|
OPTIONS = 'opts'
|
|
64
77
|
TRANSFER_SPEC = 'spec'
|
|
65
78
|
SYNC_CONF = 'conf'
|
|
66
79
|
SYNC_ARGS = 'args'
|
|
80
|
+
PREVIEW_OPTIONS = 'preview'
|
|
67
81
|
AOC = 'aoc'
|
|
82
|
+
AUTOMATION = 'automation'
|
|
68
83
|
FASPEX = 'faspex'
|
|
69
84
|
FASPIO = 'faspio'
|
|
85
|
+
CONSOLE = 'console'
|
|
70
86
|
NODE = 'node'
|
|
87
|
+
ORCHESTRATOR = 'orchestrator'
|
|
71
88
|
SHARES = 'shares+/api/v1'
|
|
72
89
|
ASYNC_TABLES = 'async_tables'
|
|
73
90
|
LOG_OPTIONS = "#{OPTIONS}:components.schemas.LogOptions"
|
|
74
91
|
DIRECT_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.DirectAgentOptions"
|
|
75
92
|
NODE_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.NodeAgentOptions"
|
|
93
|
+
NODE_EMULATOR_OPTIONS = "#{OPTIONS}:components.schemas.NodeEmulatorOptions"
|
|
76
94
|
HTTPGW_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.HttpgwAgentOptions"
|
|
77
95
|
TRANSFERD_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.TransferdAgentOptions"
|
|
78
96
|
TRANSFER_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.TransferAgentOptions"
|
|
@@ -111,7 +129,11 @@ module Aspera
|
|
|
111
129
|
if path&.end_with?(QUERY_PARAMS_SUFFIX)
|
|
112
130
|
parent_path = path.delete_suffix(QUERY_PARAMS_SUFFIX)
|
|
113
131
|
node = @cache[sym].dig(*parent_path.split('.'))
|
|
114
|
-
|
|
132
|
+
# Parameters may be references to components.parameters
|
|
133
|
+
params = (node&.fetch('parameters', []) || []).map do |param|
|
|
134
|
+
param.key?('$ref') ? @cache[sym].dig(*param['$ref'].delete_prefix('#/').split('/')) : param
|
|
135
|
+
end
|
|
136
|
+
return Reader.from_query_params(params.compact)
|
|
115
137
|
end
|
|
116
138
|
reader = Reader.new(@cache[sym])
|
|
117
139
|
return reader unless path
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'singleton'
|
|
4
|
+
require 'json'
|
|
5
|
+
require 'aspera/schema/registry'
|
|
6
|
+
|
|
7
|
+
module Aspera
|
|
8
|
+
module Schema
|
|
9
|
+
# Validate values against the JSON schemas of the registry.
|
|
10
|
+
# Only schemas owned by ascli are validated: vendor API schemas are checked by the API itself.
|
|
11
|
+
# @!method self.instance
|
|
12
|
+
# Returns the singleton instance of Validator
|
|
13
|
+
# @return [Validator] the singleton instance
|
|
14
|
+
class Validator
|
|
15
|
+
include Singleton
|
|
16
|
+
|
|
17
|
+
def initialize
|
|
18
|
+
# compiled documents, key: [registry key, partial]
|
|
19
|
+
@documents = {}
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Validate a value against a schema.
|
|
23
|
+
# @param value [Object] value to validate
|
|
24
|
+
# @param path [String] schema path, e.g. `opts:components.schemas.HttpOptions`
|
|
25
|
+
# @param partial [Boolean] `true`: value may be incomplete, `required` is not enforced, `null` values (removal) are ignored
|
|
26
|
+
# @return [Array<String>] error messages, empty if valid or schema not validated
|
|
27
|
+
def errors(value, path, partial: false)
|
|
28
|
+
return [] unless path.is_a?(String) && Registry.owned?(path)
|
|
29
|
+
key, dotted = path.split(':', 2)
|
|
30
|
+
schema = document(key, partial)
|
|
31
|
+
schema = schema.ref("#/#{dotted.split('.').map { |s| s.gsub('~', '~0').gsub('/', '~1') }.join('/')}") if dotted
|
|
32
|
+
# JSON view of value: symbol keys and values become strings
|
|
33
|
+
data = JSON.parse(JSON.generate(value))
|
|
34
|
+
data = without_null(data) if partial
|
|
35
|
+
schema.validate(data).filter_map { |error| message(error, partial) }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
# @param key [String] registry key
|
|
41
|
+
# @param partial [Boolean] remove `required` from schema
|
|
42
|
+
# @return [JSONSchemer::Schema] compiled schema document
|
|
43
|
+
def document(key, partial)
|
|
44
|
+
@documents[[key, partial]] ||=
|
|
45
|
+
begin
|
|
46
|
+
require 'json_schemer'
|
|
47
|
+
root = Registry.instance.reader(key).current
|
|
48
|
+
root = without_required(root) if partial
|
|
49
|
+
meta =
|
|
50
|
+
case root['openapi']
|
|
51
|
+
when /^3\.0\./ then JSONSchemer.openapi30
|
|
52
|
+
when /^3\.1\./ then JSONSchemer.openapi31
|
|
53
|
+
end
|
|
54
|
+
meta ? JSONSchemer.schema(root, meta_schema: meta) : JSONSchemer.schema(root)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# @param node [Object] schema node
|
|
59
|
+
# @return [Object] copy of node without `required` lists
|
|
60
|
+
def without_required(node)
|
|
61
|
+
case node
|
|
62
|
+
when Hash then node.each_with_object({}) { |(k, v), h| h[k] = without_required(v) unless k.eql?('required') && v.is_a?(Array) }
|
|
63
|
+
when Array then node.map { |e| without_required(e) }
|
|
64
|
+
else node
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# @param node [Object] JSON value
|
|
69
|
+
# @return [Object] copy of node without Hash entries whose value is `null` (entry removal requested by user)
|
|
70
|
+
def without_null(node)
|
|
71
|
+
case node
|
|
72
|
+
when Hash then node.each_with_object({}) { |(k, v), h| h[k] = without_null(v) unless v.nil? }
|
|
73
|
+
when Array then node.map { |e| without_null(e) }
|
|
74
|
+
else node
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# @param error [Hash] json_schemer error
|
|
79
|
+
# @param partial [Boolean] partial validation
|
|
80
|
+
# @return [String, nil] message, or nil if error is ignored
|
|
81
|
+
def message(error, partial)
|
|
82
|
+
return error['error'] unless error['type'].eql?('discriminator')
|
|
83
|
+
property = error['schema'].dig('discriminator', 'propertyName')
|
|
84
|
+
object = error['data']
|
|
85
|
+
# Partial value: selector may come from another source or a default
|
|
86
|
+
return if partial && object.is_a?(Hash) && !object.key?(property)
|
|
87
|
+
allowed = error['schema'].dig('discriminator', 'mapping')&.keys
|
|
88
|
+
"value at `#{error['data_pointer']}/#{property}` is not one of: #{allowed}"
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
data/lib/aspera/secret_hider.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# cspell:ignore FILEPASS
|
|
4
4
|
require 'logger'
|
|
5
|
+
require 'set'
|
|
5
6
|
require 'singleton'
|
|
6
7
|
|
|
7
8
|
module Aspera
|
|
@@ -13,8 +14,6 @@ module Aspera
|
|
|
13
14
|
class SecretHider
|
|
14
15
|
include Singleton
|
|
15
16
|
|
|
16
|
-
# configurable:
|
|
17
|
-
ADDITIONAL_KEYS_TO_HIDE = []
|
|
18
17
|
# display string for hidden secrets
|
|
19
18
|
HIDDEN_PASSWORD = '🔑'
|
|
20
19
|
# env vars for ascp with secrets
|
|
@@ -23,8 +22,8 @@ module Aspera
|
|
|
23
22
|
KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
|
|
24
23
|
HTTP_SECRETS = %w[Authorization].freeze
|
|
25
24
|
ALL_SECRETS = (ASCP_ENV_SECRETS + KEY_SECRETS + HTTP_SECRETS).freeze
|
|
26
|
-
|
|
27
|
-
KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/].freeze
|
|
25
|
+
NON_ENV_SECRETS = (KEY_SECRETS + HTTP_SECRETS).freeze
|
|
26
|
+
KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/, /public_key$/, /^token_type$/].freeze
|
|
28
27
|
# min length of hidden secrets, use `+` or `{n,}`
|
|
29
28
|
SECRET_LENGTH = '{5,}'
|
|
30
29
|
# regex that define named captures :begin and :end
|
|
@@ -32,38 +31,42 @@ module Aspera
|
|
|
32
31
|
# private key values (place first)
|
|
33
32
|
/(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)\n*/,
|
|
34
33
|
# CLI manager get/set options
|
|
35
|
-
/(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)
|
|
34
|
+
/(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/i,
|
|
36
35
|
# env var ascp exec
|
|
37
|
-
/(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)[^ ]+(?<end> )/,
|
|
38
|
-
# rendered JSON or Ruby
|
|
39
|
-
/(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>?(?:=>|:) *")[^"]+(?<end>")
|
|
36
|
+
/(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)[^ \n]+(?<end> |$)/,
|
|
37
|
+
# rendered JSON or Ruby (quoted key or legacy symbol key: `"k":"v"`, `"k" => "v"`, `:k=>"v"`)
|
|
38
|
+
/(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>? *(?:=>|:) *")(?:[^"\\]|\\.)+(?<end>")/i,
|
|
39
|
+
# rendered Ruby 3.4+ symbol key: `k: "v"`
|
|
40
|
+
/(?<begin>\b\w*(?:#{ALL_SECRETS.join('|')})\w*: *")(?:[^"\\]|\\.)+(?<end>")/i,
|
|
40
41
|
# logged data
|
|
41
|
-
/(?<begin>(?:#{
|
|
42
|
+
/(?<begin>(?:#{NON_ENV_SECRETS.join('|')})[ =:]+)[^ "\n]#{SECRET_LENGTH}(?<end>$)/i,
|
|
42
43
|
# cred in http dump
|
|
43
44
|
/(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
|
|
44
45
|
].freeze
|
|
45
|
-
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :KEY_FALSE_POSITIVES, :REGEX_LOG_REPLACES
|
|
46
|
+
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :NON_ENV_SECRETS, :KEY_FALSE_POSITIVES, :SECRET_LENGTH, :REGEX_LOG_REPLACES
|
|
46
47
|
attr_accessor :log_secrets
|
|
47
48
|
|
|
48
49
|
# @return [Proc] new log formatter that hides secrets
|
|
49
50
|
def log_formatter(original_formatter)
|
|
50
51
|
original_formatter ||= Logger::Formatter.new
|
|
51
52
|
# NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
|
53
|
+
# Hiding is applied on the formatted line, so that non-String messages (e.g. Exception) are covered too
|
|
52
54
|
return lambda do |severity, date_time, program_name, msg|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
original_formatter.call(severity, date_time, program_name, msg)
|
|
55
|
+
line = original_formatter.call(severity, date_time, program_name, msg)
|
|
56
|
+
line = hide_secrets_in_string(line, all: true) if line.is_a?(String) && !@log_secrets
|
|
57
|
+
line
|
|
59
58
|
end
|
|
60
59
|
end
|
|
61
60
|
|
|
62
|
-
# Replace
|
|
63
|
-
# @param value [String] Input string possibly containing
|
|
64
|
-
# @
|
|
65
|
-
|
|
66
|
-
|
|
61
|
+
# Replace secrets in a string with the hidden password placeholder
|
|
62
|
+
# @param value [String] Input string possibly containing secrets
|
|
63
|
+
# @param all [Boolean] `false`: only private keys, `true`: all known secret patterns
|
|
64
|
+
# @return [String] String with secrets replaced by placeholder
|
|
65
|
+
def hide_secrets_in_string(value, all: false)
|
|
66
|
+
(all ? REGEX_LOG_REPLACES : REGEX_LOG_REPLACES.first(1)).each do |reg_ex|
|
|
67
|
+
value = value.gsub(reg_ex) { "#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}" }
|
|
68
|
+
end
|
|
69
|
+
return value
|
|
67
70
|
end
|
|
68
71
|
|
|
69
72
|
# @param keyword [String, Symbol] Key name to check
|
|
@@ -73,11 +76,12 @@ module Aspera
|
|
|
73
76
|
keyword = keyword.to_s if keyword.is_a?(Symbol)
|
|
74
77
|
# only Strings can be secrets, not booleans, or hash, arrays
|
|
75
78
|
return false unless keyword.is_a?(String) && value.is_a?(String)
|
|
79
|
+
return true if @additional_keys.include?(keyword)
|
|
80
|
+
keyword = keyword.downcase
|
|
76
81
|
# those are not secrets
|
|
77
82
|
return false if KEY_FALSE_POSITIVES.any? { |f| f.match?(keyword) }
|
|
78
|
-
return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
|
|
79
83
|
# check if keyword (name) contains an element that designate it as a secret
|
|
80
|
-
ALL_SECRETS.any? { |kw| keyword.include?(kw) }
|
|
84
|
+
ALL_SECRETS.any? { |kw| keyword.include?(kw.downcase) }
|
|
81
85
|
end
|
|
82
86
|
|
|
83
87
|
# Hides recursively secrets in Hash or Array of Hash
|
|
@@ -89,18 +93,25 @@ module Aspera
|
|
|
89
93
|
obj.each do |k, v|
|
|
90
94
|
if secret?(k, v)
|
|
91
95
|
obj[k] = HIDDEN_PASSWORD
|
|
92
|
-
elsif
|
|
93
|
-
deep_remove_secret(
|
|
96
|
+
elsif v.is_a?(Hash) || v.is_a?(Array)
|
|
97
|
+
deep_remove_secret(v)
|
|
94
98
|
end
|
|
95
99
|
end
|
|
96
100
|
end
|
|
97
101
|
return obj
|
|
98
102
|
end
|
|
99
103
|
|
|
104
|
+
# Declare additional exact key names whose values are secrets
|
|
105
|
+
# @param keys [Array<String, Symbol>] Key names
|
|
106
|
+
def add_secret_keys(keys)
|
|
107
|
+
@additional_keys.merge(keys.map(&:to_s))
|
|
108
|
+
end
|
|
109
|
+
|
|
100
110
|
private
|
|
101
111
|
|
|
102
112
|
def initialize
|
|
103
113
|
@log_secrets = false
|
|
114
|
+
@additional_keys = Set.new
|
|
104
115
|
end
|
|
105
116
|
end
|
|
106
117
|
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class ::String
|
|
4
|
+
# Transform capitalized to snake case
|
|
5
|
+
def capital_to_snake
|
|
6
|
+
return gsub(/([a-z\d])([A-Z])/, '\1_\2')
|
|
7
|
+
.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
|
|
8
|
+
.downcase
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# Transform snake case to capitalized
|
|
12
|
+
def snake_to_capital
|
|
13
|
+
split('_').map(&:capitalize).join
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'singleton'
|
|
4
4
|
require 'fileutils'
|
|
5
5
|
require 'etc'
|
|
6
|
-
require '
|
|
6
|
+
require 'tmpdir'
|
|
7
7
|
|
|
8
8
|
module Aspera
|
|
9
9
|
# create a temp file name for a given folder
|
|
@@ -58,10 +58,11 @@ module Aspera
|
|
|
58
58
|
def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
|
|
59
59
|
FileUtils.mkdir_p(temp_folder)
|
|
60
60
|
basename = [prefix || 'aspera', SecureRandom.uuid].join('-')
|
|
61
|
-
#
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
# Create the file atomically with O_EXCL (retry with another name if it exists)
|
|
62
|
+
# Note: Tempfile is not used, as its file is deleted when the object is garbage collected
|
|
63
|
+
new_file = Dir::Tmpname.create([basename, suffix ? "-#{suffix}" : ''], temp_folder) do |path|
|
|
64
|
+
File.open(path, File::WRONLY | File::CREAT | File::EXCL, 0o600) { nil }
|
|
65
|
+
end
|
|
65
66
|
@created_files.push(new_file)
|
|
66
67
|
new_file
|
|
67
68
|
end
|
data/lib/aspera/transfer/spec.rb
CHANGED
|
@@ -25,6 +25,7 @@ module Aspera
|
|
|
25
25
|
TRANSPORT_FIELDS = (%w[remote_host] + AK_TSPEC_BASE.keys + WSS_FIELDS).freeze
|
|
26
26
|
# reserved tag for Aspera
|
|
27
27
|
TAG_RESERVED = 'aspera'
|
|
28
|
+
# Specific fields to not display
|
|
28
29
|
SPECIFIC = %w{token paths direction source_root destination_root}.freeze
|
|
29
30
|
class << self
|
|
30
31
|
# wrong def in transferd
|
data/lib/aspera/uri_reader.rb
CHANGED
|
@@ -22,7 +22,7 @@ module Aspera
|
|
|
22
22
|
#
|
|
23
23
|
# The short form is consistent with RFC 8089 (file:///<path> = absolute,
|
|
24
24
|
# file:<path> = relative). Both forms are handled identically by this module.
|
|
25
|
-
# The canonical form built by {file_url} uses the
|
|
25
|
+
# The canonical form built by {file_url} uses the `file:///` prefix.
|
|
26
26
|
module UriReader
|
|
27
27
|
SCHEME_FILE = 'file'
|
|
28
28
|
SCHEME_FILE_PFX1 = "#{SCHEME_FILE}:"
|
|
@@ -31,13 +31,13 @@ module Aspera
|
|
|
31
31
|
SCHEME_FILE_PFX2 = "#{SCHEME_FILE_PFX1}///"
|
|
32
32
|
private_constant :SCHEME_FILE, :SCHEME_FILE_PFX1, :SCHEME_FILE_PFX2
|
|
33
33
|
class << self
|
|
34
|
-
# @return [Boolean] true if
|
|
34
|
+
# @return [Boolean] true if `url` uses the file: scheme recognised by this module
|
|
35
35
|
def file?(url)
|
|
36
36
|
url.start_with?(SCHEME_FILE_PFX1)
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
-
# Build a file: URL from
|
|
40
|
-
# A relative path yields
|
|
39
|
+
# Build a file: URL from `path`.
|
|
40
|
+
# A relative path yields `file:///path`; an absolute path yields `file:////path`.
|
|
41
41
|
# @param path [String] relative or absolute file-system path
|
|
42
42
|
# @return [String] corresponding file: URL
|
|
43
43
|
def file_url(path)
|
|
@@ -45,7 +45,7 @@ module Aspera
|
|
|
45
45
|
end
|
|
46
46
|
|
|
47
47
|
# Extract the file-system path from a file: URL.
|
|
48
|
-
# Accepts both the canonical
|
|
48
|
+
# Accepts both the canonical `file:///` form and the short `file:` form.
|
|
49
49
|
# Returns the literal path (relative or absolute) without working-directory expansion.
|
|
50
50
|
# @param url [String] a file: URL (canonical or short form)
|
|
51
51
|
# @return [String] the literal path encoded in the URL
|
|
@@ -57,12 +57,12 @@ module Aspera
|
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
# Read content from a URI and return it as a String.
|
|
60
|
-
# Supported schemes:
|
|
60
|
+
# Supported schemes: `http`, `https`, `data`, `file`, and bare paths (no scheme).
|
|
61
61
|
# For file: URLs the path is extracted via {file_path} to respect the module convention
|
|
62
62
|
# (see module-level documentation). Ruby's URI parser is not used for file: URLs because
|
|
63
63
|
# it interprets the three-slash prefix differently (it always produces an absolute path).
|
|
64
|
-
# Bare paths (no scheme) are passed to File.read directly; leading
|
|
65
|
-
# are expanded via
|
|
64
|
+
# Bare paths (no scheme) are passed to File.read directly; leading `/~/`, `/./`, `/../`
|
|
65
|
+
# are expanded via `File.expand_path` after stripping the synthetic leading slash added by
|
|
66
66
|
# URI.
|
|
67
67
|
def read(uri_to_read)
|
|
68
68
|
# Handle file: URLs directly to honour the file:///relative vs file:////absolute convention.
|
|
@@ -70,7 +70,7 @@ module Aspera
|
|
|
70
70
|
uri = URI.parse(uri_to_read)
|
|
71
71
|
case uri.scheme
|
|
72
72
|
when 'http', 'https'
|
|
73
|
-
return Rest.new(base_url: uri_to_read, redirect_max: 5).read(nil, headers: {'Accept' => '*/*'})
|
|
73
|
+
return Rest::Client.new(base_url: uri_to_read, redirect_max: 5).read(nil, headers: {'Accept' => '*/*'})
|
|
74
74
|
when 'data'
|
|
75
75
|
metadata, encoded_data = uri.opaque.split(',', 2)
|
|
76
76
|
if metadata.end_with?(';base64')
|
|
@@ -87,7 +87,7 @@ module Aspera
|
|
|
87
87
|
end
|
|
88
88
|
end
|
|
89
89
|
|
|
90
|
-
# Return the local file-system path for the content at
|
|
90
|
+
# Return the local file-system path for the content at `url`, downloading to a temp file if needed.
|
|
91
91
|
# For file: URLs the path is extracted directly (no download).
|
|
92
92
|
# For data: and http(s): URLs the content is written to a temporary file and its path is returned.
|
|
93
93
|
# @return [String] local path to a file containing the URL content
|
|
@@ -105,7 +105,7 @@ module Aspera
|
|
|
105
105
|
# download to temp file
|
|
106
106
|
# auto-delete on exit
|
|
107
107
|
temp_file = TempFileManager.instance.new_file_path_global(suffix: File.basename(url))
|
|
108
|
-
Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to: temp_file)
|
|
108
|
+
Aspera::Rest::Client.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to: temp_file)
|
|
109
109
|
return temp_file
|
|
110
110
|
end
|
|
111
111
|
end
|