aspera-cli 4.26.2 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +281 -21
- data/CONTRIBUTING.md +98 -7
- data/bin/ascli +3 -3
- data/docs/README.md +10757 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +20 -6
- data/lib/aspera/agent/connect.rb +53 -16
- data/lib/aspera/agent/desktop.rb +52 -17
- data/lib/aspera/agent/direct.rb +26 -24
- data/lib/aspera/agent/httpgw.rb +1 -1
- data/lib/aspera/agent/node.rb +32 -0
- data/lib/aspera/agent/transferd.rb +39 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -16
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/faspex.rb +15 -14
- data/lib/aspera/api/httpgw.rb +4 -4
- data/lib/aspera/api/node.rb +26 -23
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +21 -19
- data/lib/aspera/ascp/installation.rb +37 -38
- data/lib/aspera/ascp/management.rb +2 -3
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +81 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +138 -0
- data/lib/aspera/cli/context.rb +16 -3
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +2 -1
- data/lib/aspera/cli/formatter.rb +41 -27
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +2 -2
- data/lib/aspera/cli/http.rb +70 -19
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +198 -0
- data/lib/aspera/cli/options.schema.yaml +364 -7
- data/lib/aspera/cli/{manager.rb → parser.rb} +334 -142
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1166 -853
- data/lib/aspera/cli/plugins/ats.rb +200 -161
- data/lib/aspera/cli/plugins/base.rb +457 -100
- data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
- data/lib/aspera/cli/plugins/config.rb +428 -754
- data/lib/aspera/cli/plugins/console.rb +106 -64
- data/lib/aspera/cli/plugins/cos.rb +44 -32
- data/lib/aspera/cli/plugins/factory.rb +4 -4
- data/lib/aspera/cli/plugins/faspex.rb +294 -257
- data/lib/aspera/cli/plugins/faspex5.rb +585 -357
- data/lib/aspera/cli/plugins/faspio.rb +49 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +998 -794
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +111 -134
- data/lib/aspera/cli/plugins/preview.rb +180 -139
- data/lib/aspera/cli/plugins/server.rb +124 -79
- data/lib/aspera/cli/plugins/shares.rb +301 -107
- data/lib/aspera/cli/preset_actions.rb +139 -0
- data/lib/aspera/cli/preset_manager.rb +13 -12
- data/lib/aspera/cli/result.rb +61 -11
- data/lib/aspera/cli/runner.rb +275 -141
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +83 -59
- data/lib/aspera/cli/terminal_formatter.rb +1 -1
- data/lib/aspera/cli/transfer_actions.rb +83 -0
- data/lib/aspera/cli/transfer_agent.rb +116 -51
- data/lib/aspera/cli/transfer_progress.rb +3 -3
- data/lib/aspera/cli/vault_manager.rb +57 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +21 -20
- data/lib/aspera/command_line_builder.rb +22 -22
- data/lib/aspera/data_repository.rb +2 -1
- data/lib/aspera/dot_container.rb +4 -4
- data/lib/aspera/environment.rb +18 -11
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +4 -3
- data/lib/aspera/graphql.rb +35 -0
- data/lib/aspera/hash_ext.rb +6 -0
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/encrypted_hash.rb +1 -1
- data/lib/aspera/keychain/factory.rb +3 -3
- data/lib/aspera/keychain/macos_security.rb +3 -3
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +17 -3
- data/lib/aspera/markdown.rb +85 -2
- data/lib/aspera/node_simulator.rb +3 -1
- data/lib/aspera/oauth/base.rb +5 -5
- data/lib/aspera/oauth/boot.rb +2 -2
- data/lib/aspera/oauth/factory.rb +7 -5
- data/lib/aspera/oauth/jwt.rb +4 -4
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/persistency_action_once.rb +8 -8
- data/lib/aspera/persistency_folder.rb +10 -3
- data/lib/aspera/preview/file_types.rb +3 -3
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/utils.rb +1 -1
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/other.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +7 -6
- data/lib/aspera/rest.rb +10 -36
- data/lib/aspera/rest_error_analyzer.rb +3 -3
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +3 -3
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
- data/lib/aspera/schema/documentation.rb +57 -26
- data/lib/aspera/schema/reader.rb +59 -10
- data/lib/aspera/schema/registry.rb +39 -6
- data/lib/aspera/secret_hider.rb +7 -2
- data/lib/aspera/ssl.rb +4 -4
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/operations.rb +10 -13
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +23 -21
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +6 -6
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +12 -3
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +8 -5
- data/lib/aspera/yaml.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +25 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/json_rpc.rb +0 -52
|
@@ -7,6 +7,15 @@ module Aspera
|
|
|
7
7
|
module Schema
|
|
8
8
|
# Generate documentation from Schema, for Transfer Spec, or async Conf spec
|
|
9
9
|
class Documentation
|
|
10
|
+
# Map JSON Schema type names to user-friendly display names
|
|
11
|
+
JSON_TYPE_TO_DOC = {
|
|
12
|
+
'string' => 'String',
|
|
13
|
+
'integer' => 'Integer',
|
|
14
|
+
'number' => 'Number',
|
|
15
|
+
'boolean' => 'Bool',
|
|
16
|
+
'array' => 'Array',
|
|
17
|
+
'object' => 'Hash'
|
|
18
|
+
}.freeze
|
|
10
19
|
# @param formatter [Cli::Formatter] Formatter instance with methods: markdown_text, tick, check_row
|
|
11
20
|
# @param schema [Reader]
|
|
12
21
|
# @param include_option [Boolean] `true`: include CLI options (switches, env vars) in descriptions
|
|
@@ -18,25 +27,27 @@ module Aspera
|
|
|
18
27
|
@include_option = include_option
|
|
19
28
|
@agent_columns = agent_columns
|
|
20
29
|
@code_highlight = code_highlight
|
|
21
|
-
@columns = %
|
|
22
|
-
@columns.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short]}.sort) if @agent_columns
|
|
23
|
-
#
|
|
24
|
-
|
|
30
|
+
@columns = %w[name type description]
|
|
31
|
+
@columns.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short].to_s}.sort) if @agent_columns
|
|
32
|
+
# Sections: each entry is {header: row_or_nil, rows: []}
|
|
33
|
+
# A flat schema produces a single section with no header.
|
|
34
|
+
@sections = [{header: nil, rows: []}]
|
|
25
35
|
end
|
|
26
36
|
|
|
27
37
|
def rows
|
|
28
|
-
@
|
|
38
|
+
@sections.flat_map do |section|
|
|
39
|
+
sorted = section[:rows].sort_by{ |i| i['name']}
|
|
40
|
+
section[:header] ? [section[:header]] + sorted : sorted
|
|
41
|
+
end
|
|
29
42
|
end
|
|
30
43
|
|
|
31
44
|
# @return [Array<String>]
|
|
32
|
-
|
|
33
|
-
@columns.map(&:to_s)
|
|
34
|
-
end
|
|
45
|
+
attr_reader :columns
|
|
35
46
|
|
|
36
|
-
# First row is the titles
|
|
47
|
+
# First row is the titles (for Markdown table generation)
|
|
37
48
|
# @return [Array<Array<String>>]
|
|
38
49
|
def table
|
|
39
|
-
[@columns
|
|
50
|
+
[@columns] + rows.map{ |row| @columns.map{ |field| row[field]}}
|
|
40
51
|
end
|
|
41
52
|
|
|
42
53
|
# Generate a documentation table from a JSON schema for transfer specifications
|
|
@@ -49,37 +60,57 @@ module Aspera
|
|
|
49
60
|
def build(schema = nil)
|
|
50
61
|
code = @code_highlight ? ->(c){"`#{c}`"} : ->(c){c}
|
|
51
62
|
schema ||= @schema
|
|
52
|
-
|
|
63
|
+
render_title = ->(title){title.gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}}
|
|
64
|
+
on_variant = ->(variant_reader, discriminant_property, discriminant_value) do
|
|
65
|
+
title = variant_reader.current['title'] || variant_reader.current['description']
|
|
66
|
+
header =
|
|
67
|
+
if discriminant_property && discriminant_value
|
|
68
|
+
desc = render_title.call("`#{discriminant_value}`")
|
|
69
|
+
desc += ": #{render_title.call(title)}" if title
|
|
70
|
+
@formatter.check_row({
|
|
71
|
+
'name' => render_title.call("**#{discriminant_property}**"),
|
|
72
|
+
'type' => code.call(JSON_TYPE_TO_DOC['string']),
|
|
73
|
+
'description' => desc
|
|
74
|
+
})
|
|
75
|
+
elsif title
|
|
76
|
+
@formatter.check_row({'name' => "**#{render_title.call(title)}**", 'type' => ' ', 'description' => ' '})
|
|
77
|
+
end
|
|
78
|
+
@sections.push({header: header, rows: []})
|
|
79
|
+
end
|
|
80
|
+
schema.each_property(on_variant: on_variant) do |property_schema, _name, property_full_name|
|
|
53
81
|
node = property_schema.current
|
|
54
82
|
# Manual table
|
|
83
|
+
item_type = JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])
|
|
84
|
+
item_type = "#{item_type}[#{JSON_TYPE_TO_DOC.fetch(node.dig('items', 'type'), node.dig('items', 'type'))}]" if node['type'].eql?('array') && node.dig('items', 'type').is_a?(String)
|
|
55
85
|
item = {
|
|
56
|
-
name
|
|
57
|
-
type
|
|
58
|
-
description
|
|
86
|
+
'name' => code.call(property_full_name),
|
|
87
|
+
'type' => code.call(item_type),
|
|
88
|
+
'description' => []
|
|
59
89
|
}
|
|
60
90
|
# Render Markdown formatting and split lines
|
|
61
|
-
item[
|
|
91
|
+
item['description'] =
|
|
62
92
|
node['description']
|
|
63
93
|
.gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}
|
|
64
94
|
.split("\n") if node.key?('description')
|
|
65
|
-
item[
|
|
95
|
+
item['description'].unshift("DEPRECATED: #{node['x-deprecation']}") if node.key?('x-deprecation')
|
|
66
96
|
# Add flags for supported agents in doc
|
|
67
97
|
agents = []
|
|
68
98
|
Agent::Factory::ALL.each_key do |sym|
|
|
69
99
|
agents.push(sym) if node['x-agents'].nil? || node['x-agents'].include?(sym.to_s)
|
|
70
100
|
end
|
|
71
|
-
Aspera.assert(agents.include?(:direct)){"#{
|
|
101
|
+
Aspera.assert(agents.include?(:direct)){"#{property_full_name}: x-cli-option requires agent direct (or nil)"} if node['x-cli-option']
|
|
72
102
|
if @agent_columns
|
|
73
103
|
Agent::Factory::ALL.each do |sym, names|
|
|
74
|
-
item[names[:short]] = @formatter.tick(agents.include?(sym))
|
|
104
|
+
item[names[:short].to_s] = @formatter.tick(agents.include?(sym))
|
|
75
105
|
end
|
|
76
106
|
else
|
|
77
|
-
item[
|
|
107
|
+
item['description'].push("(#{agents.map{ |i| Agent::Factory::ALL[i][:short].to_s.upcase}.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
|
|
78
108
|
end
|
|
79
109
|
# Only keep lines that are usable in supported agents
|
|
80
|
-
next
|
|
81
|
-
item[
|
|
82
|
-
item[
|
|
110
|
+
next if agents.empty?
|
|
111
|
+
item['description'].push("Allowed values: #{node['enum'].map{ |v| @formatter.markdown_text("`#{v}`")}.join(', ')}.") if node.key?('enum')
|
|
112
|
+
item['description'].push("Default: #{@formatter.markdown_text("`#{node['default']}`")}.") if node.key?('default')
|
|
113
|
+
item['description'].push("Example: #{@formatter.markdown_text("`#{node['example']}`")}.") if node.key?('example')
|
|
83
114
|
if @include_option
|
|
84
115
|
envvar_prefix = ''
|
|
85
116
|
cli_option =
|
|
@@ -87,18 +118,18 @@ module Aspera
|
|
|
87
118
|
envvar_prefix = 'env:'
|
|
88
119
|
node['x-cli-envvar']
|
|
89
120
|
elsif node['x-cli-switch']
|
|
90
|
-
node['x-cli-
|
|
121
|
+
false_part = node.key?('x-cli-false') ? " / #{node['x-cli-false']}" : ''
|
|
122
|
+
"#{node['x-cli-option']}#{false_part}"
|
|
91
123
|
elsif node['x-cli-option']
|
|
92
124
|
arg_type = node.key?('enum') ? '{enum}' : "{#{[node['type']].flatten.join('|')}}"
|
|
93
|
-
# conversion_tag = node['x-cli-convert']
|
|
94
125
|
conversion_tag = node.key?('x-cli-convert') ? 'conversion' : nil
|
|
95
126
|
sep = node['x-cli-option'].start_with?('--') ? '=' : ' '
|
|
96
127
|
"#{node['x-cli-option']}#{sep}#{"(#{conversion_tag})" if conversion_tag}#{arg_type}"
|
|
97
128
|
end
|
|
98
129
|
short = node.key?('x-cli-short') ? "(#{node['x-cli-short']})" : nil
|
|
99
|
-
item[
|
|
130
|
+
item['description'].push("(#{'special:' if node['x-cli-special']}#{envvar_prefix}#{@formatter.markdown_text("`#{cli_option}`")})#{short}") if cli_option
|
|
100
131
|
end
|
|
101
|
-
@rows.push(@formatter.check_row(item))
|
|
132
|
+
@sections.last[:rows].push(@formatter.check_row(item))
|
|
102
133
|
end
|
|
103
134
|
self
|
|
104
135
|
end
|
data/lib/aspera/schema/reader.rb
CHANGED
|
@@ -7,11 +7,34 @@ module Aspera
|
|
|
7
7
|
class Reader
|
|
8
8
|
attr_reader :current
|
|
9
9
|
|
|
10
|
+
class << self
|
|
11
|
+
# Build a synthetic Reader from an OAS `parameters` array (entries with `in: query`).
|
|
12
|
+
# Produces a JSON Schema object whose `properties` map each query param name to its schema,
|
|
13
|
+
# with the OAS-level `description` and `required` merged in.
|
|
14
|
+
# @param params [Array<Hash>] raw OAS parameter objects (may contain path/header params too)
|
|
15
|
+
# @return [Reader]
|
|
16
|
+
def from_query_params(params)
|
|
17
|
+
properties = {}
|
|
18
|
+
required_names = []
|
|
19
|
+
params.each do |param|
|
|
20
|
+
next unless param['in'] == 'query'
|
|
21
|
+
name = param['name']
|
|
22
|
+
prop = (param['schema'] || {}).dup
|
|
23
|
+
prop['description'] = param['description'] if param['description'] && !prop.key?('description')
|
|
24
|
+
properties[name] = prop
|
|
25
|
+
required_names << name if param['required']
|
|
26
|
+
end
|
|
27
|
+
synthetic = {'type' => 'object', 'properties' => properties}
|
|
28
|
+
synthetic['required'] = required_names unless required_names.empty?
|
|
29
|
+
new(synthetic)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
10
33
|
# Shortcut to access current value at path
|
|
11
|
-
# @param
|
|
34
|
+
# @param key [String] path element
|
|
12
35
|
# @return [Hash, Array, String, Integer] current value at path
|
|
13
|
-
def [](
|
|
14
|
-
@current[
|
|
36
|
+
def [](key)
|
|
37
|
+
@current[key]
|
|
15
38
|
end
|
|
16
39
|
|
|
17
40
|
# Find sub path relative to current
|
|
@@ -31,6 +54,12 @@ module Aspera
|
|
|
31
54
|
Reader.new(@root, current)
|
|
32
55
|
end
|
|
33
56
|
|
|
57
|
+
# Resolve a $ref string to a Reader
|
|
58
|
+
def resolve_ref(ref)
|
|
59
|
+
Aspera.assert(ref.start_with?('#/')){"schema $ref must start with '#/': #{ref}"}
|
|
60
|
+
Reader.new(@root, @root.dig(*ref[2..].split('/')))
|
|
61
|
+
end
|
|
62
|
+
|
|
34
63
|
# Read schema from file or from cache
|
|
35
64
|
# @param root [Hash] root schema
|
|
36
65
|
# @param current [Hash, nil] current position in
|
|
@@ -40,15 +69,35 @@ module Aspera
|
|
|
40
69
|
@current = current || root
|
|
41
70
|
end
|
|
42
71
|
|
|
43
|
-
# Recursively traverse schema properties with a block
|
|
44
|
-
#
|
|
45
|
-
#
|
|
72
|
+
# Recursively traverse schema properties with a block.
|
|
73
|
+
# If the current node has `oneOf`, each variant is traversed in turn and
|
|
74
|
+
# `on_variant` is called (if given) before each variant's properties.
|
|
75
|
+
# @param prefix [String] Prefix for property names (e.g., 'parent.child.')
|
|
76
|
+
# @param on_variant [Proc, nil] Called with the variant Reader before its properties
|
|
46
77
|
# @yield [property_schema, name, full_name] Yields property info to block
|
|
47
|
-
# @yieldparam property_schema [Reader] Schema reader for this property
|
|
78
|
+
# @yieldparam property_schema [Reader] Schema reader for this property
|
|
48
79
|
# @yieldparam name [String] Property name
|
|
49
80
|
# @yieldparam full_name [String] Full property name with prefix
|
|
50
81
|
# @return [nil]
|
|
51
|
-
def each_property(prefix = '', &block)
|
|
82
|
+
def each_property(prefix = '', on_variant: nil, &block)
|
|
83
|
+
if @current.key?('oneOf')
|
|
84
|
+
# Build reverse map: $ref -> discriminant value, from discriminator.mapping if present
|
|
85
|
+
discriminant_by_ref = {}
|
|
86
|
+
if @current.dig('discriminator', 'mapping').is_a?(Hash)
|
|
87
|
+
@current['discriminator']['mapping'].each do |value, ref|
|
|
88
|
+
discriminant_by_ref[ref] = value
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
discriminant_property = @current.dig('discriminator', 'propertyName')
|
|
92
|
+
@current['oneOf'].each do |variant_node|
|
|
93
|
+
ref = variant_node['$ref']
|
|
94
|
+
variant_reader = ref ? resolve_ref(ref) : Reader.new(@root, variant_node)
|
|
95
|
+
discriminant_value = ref ? discriminant_by_ref[ref] : nil
|
|
96
|
+
on_variant&.call(variant_reader, discriminant_property, discriminant_value)
|
|
97
|
+
variant_reader.each_property(prefix, on_variant: on_variant, &block)
|
|
98
|
+
end
|
|
99
|
+
return
|
|
100
|
+
end
|
|
52
101
|
properties = dig('properties')
|
|
53
102
|
properties.current.each_key do |name|
|
|
54
103
|
property_full_name = "#{prefix}#{name}"
|
|
@@ -61,11 +110,11 @@ module Aspera
|
|
|
61
110
|
# Recursively process nested structures
|
|
62
111
|
case node['type']
|
|
63
112
|
when 'object'
|
|
64
|
-
property_schema.each_property("#{property_full_name}.", &block) if node['properties']
|
|
113
|
+
property_schema.each_property("#{property_full_name}.", on_variant: on_variant, &block) if node['properties']
|
|
65
114
|
when 'array'
|
|
66
115
|
if node['items']
|
|
67
116
|
array_item_schema = property_schema.dig('items')
|
|
68
|
-
array_item_schema.each_property("#{property_full_name}[].", &block) if array_item_schema.current['properties']
|
|
117
|
+
array_item_schema.each_property("#{property_full_name}[].", on_variant: on_variant, &block) if array_item_schema.current['properties']
|
|
69
118
|
end
|
|
70
119
|
end
|
|
71
120
|
end
|
|
@@ -18,11 +18,22 @@ module Aspera
|
|
|
18
18
|
LOCATIONS.key?(sym)
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
# Get path to request body, no check if it exists
|
|
22
|
-
# @
|
|
21
|
+
# Get path to request body schema, no check if it exists
|
|
22
|
+
# @param component [String] registry key (e.g. 'faspex', 'aoc')
|
|
23
|
+
# @param endpoint [String] endpoint path without leading slash (e.g. 'packages.post')
|
|
24
|
+
# @return [String] schema path usable in schema: keyword
|
|
23
25
|
def req_body(component, endpoint)
|
|
24
26
|
"#{component}:paths./#{endpoint}.requestBody.content.application/json.schema"
|
|
25
27
|
end
|
|
28
|
+
|
|
29
|
+
# Get path to query parameters for a GET endpoint
|
|
30
|
+
# @param component [String] registry key (e.g. 'faspex', 'aoc')
|
|
31
|
+
# @param endpoint [String] resource path without leading slash (e.g. 'packages')
|
|
32
|
+
# @param method [String] HTTP method (default: 'get')
|
|
33
|
+
# @return [String] schema path usable in query_schema: keyword
|
|
34
|
+
def query_params(component, endpoint, method: 'get')
|
|
35
|
+
"#{component}:paths./#{endpoint}.#{method}#{QUERY_PARAMS_SUFFIX}"
|
|
36
|
+
end
|
|
26
37
|
end
|
|
27
38
|
|
|
28
39
|
LOCATIONS = {
|
|
@@ -42,18 +53,34 @@ module Aspera
|
|
|
42
53
|
AOC = 'aoc'
|
|
43
54
|
FASPEX = 'faspex'
|
|
44
55
|
ASYNC_TABLES = 'async_tables'
|
|
45
|
-
|
|
56
|
+
LOG_OPTIONS = "#{OPTIONS}:components.schemas.LogOptions"
|
|
57
|
+
DIRECT_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.DirectAgentOptions"
|
|
58
|
+
NODE_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.NodeAgentOptions"
|
|
59
|
+
HTTPGW_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.HttpgwAgentOptions"
|
|
60
|
+
TRANSFERD_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.TransferdAgentOptions"
|
|
61
|
+
TRANSFER_AGENT_OPTIONS = "#{OPTIONS}:components.schemas.TransferAgentOptions"
|
|
62
|
+
SMTP_OPTIONS = "#{OPTIONS}:components.schemas.SmtpOptions"
|
|
63
|
+
HTTP_OPTIONS = "#{OPTIONS}:components.schemas.HttpOptions"
|
|
64
|
+
VAULT_OPTIONS = "#{OPTIONS}:components.schemas.VaultOptions"
|
|
65
|
+
IMAGE_OPTIONS = "#{OPTIONS}:components.schemas.ImageOptions"
|
|
66
|
+
PACKAGE_FOLDER_OPTIONS = "#{OPTIONS}:components.schemas.PackageFolderOptions"
|
|
46
67
|
|
|
47
68
|
REQ_BODY = '.requestBody.content.application/json.schema'
|
|
69
|
+
# Suffix appended to a dotted path to signal query-param extraction in reader()
|
|
70
|
+
QUERY_PARAMS_SUFFIX = '.parameters'
|
|
48
71
|
|
|
49
72
|
def initialize
|
|
50
73
|
@cache = {}
|
|
51
74
|
@main_folder = File.expand_path('../..', __dir__)
|
|
52
75
|
end
|
|
53
76
|
|
|
54
|
-
# Read schema from file or from cache
|
|
55
|
-
#
|
|
56
|
-
#
|
|
77
|
+
# Read schema from file or from cache.
|
|
78
|
+
# When name_path ends with QUERY_PARAMS_SUFFIX, the OAS `parameters` array at that path
|
|
79
|
+
# is synthesised into an object schema via Reader.from_query_params instead of navigating
|
|
80
|
+
# into the tree.
|
|
81
|
+
# @param name_path [String] registry key with optional colon-separated dotted path suffix,
|
|
82
|
+
# e.g. "faspex:paths./packages.get.parameters" or "faspex:paths./packages.post.requestBody..."
|
|
83
|
+
# @return [Reader] schema reader
|
|
57
84
|
def reader(name_path)
|
|
58
85
|
name, path = name_path.split(':', 2)
|
|
59
86
|
sym = name.to_sym
|
|
@@ -61,6 +88,12 @@ module Aspera
|
|
|
61
88
|
spec_file = File.join(@main_folder, LOCATIONS[sym])
|
|
62
89
|
@cache[sym] = Yaml.safe_load(File.read(spec_file)) if spec_file.end_with?('.yaml') && !@cache.key?(sym)
|
|
63
90
|
@cache[sym] = JSON.parse(File.read(spec_file)) if spec_file.end_with?('.json') && !@cache.key?(sym)
|
|
91
|
+
# Query-params path: strip the suffix, navigate to the operation node, extract parameters
|
|
92
|
+
if path&.end_with?(QUERY_PARAMS_SUFFIX)
|
|
93
|
+
parent_path = path.delete_suffix(QUERY_PARAMS_SUFFIX)
|
|
94
|
+
node = @cache[sym].dig(*parent_path.split('.'))
|
|
95
|
+
return Reader.from_query_params(node&.fetch('parameters', []) || [])
|
|
96
|
+
end
|
|
64
97
|
reader = Reader.new(@cache[sym])
|
|
65
98
|
return reader unless path
|
|
66
99
|
reader.dig(*path.split('.'))
|
data/lib/aspera/secret_hider.rb
CHANGED
|
@@ -45,7 +45,7 @@ module Aspera
|
|
|
45
45
|
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :KEY_FALSE_POSITIVES, :REGEX_LOG_REPLACES
|
|
46
46
|
attr_accessor :log_secrets
|
|
47
47
|
|
|
48
|
-
# @return new log formatter that hides secrets
|
|
48
|
+
# @return [Proc] new log formatter that hides secrets
|
|
49
49
|
def log_formatter(original_formatter)
|
|
50
50
|
original_formatter ||= Logger::Formatter.new
|
|
51
51
|
# NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
|
@@ -59,11 +59,16 @@ module Aspera
|
|
|
59
59
|
end
|
|
60
60
|
end
|
|
61
61
|
|
|
62
|
+
# Replace private key content in a string with the hidden password placeholder
|
|
63
|
+
# @param value [String] Input string possibly containing a private key
|
|
64
|
+
# @return [String] String with private key replaced by placeholder
|
|
62
65
|
def hide_secrets_in_string(value)
|
|
63
66
|
return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
|
|
64
67
|
end
|
|
65
68
|
|
|
66
|
-
# @
|
|
69
|
+
# @param keyword [String, Symbol] Key name to check
|
|
70
|
+
# @param value [String] Associated value (must be a String to be a secret)
|
|
71
|
+
# @return [Boolean] true if the key denotes a secret
|
|
67
72
|
def secret?(keyword, value)
|
|
68
73
|
keyword = keyword.to_s if keyword.is_a?(Symbol)
|
|
69
74
|
# only Strings can be secrets, not booleans, or hash, arrays
|
data/lib/aspera/ssl.rb
CHANGED
|
@@ -12,16 +12,16 @@ module Aspera
|
|
|
12
12
|
@extra_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
|
|
13
13
|
attr_reader :extra_options
|
|
14
14
|
|
|
15
|
-
def option_list=(
|
|
16
|
-
Aspera.assert_type(
|
|
17
|
-
|
|
15
|
+
def option_list=(options)
|
|
16
|
+
Aspera.assert_type(options, Array){'ssl_options'}
|
|
17
|
+
options.each do |opt|
|
|
18
18
|
Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
|
|
19
19
|
case opt
|
|
20
20
|
when Integer
|
|
21
21
|
@extra_options = opt
|
|
22
22
|
when String
|
|
23
23
|
name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
|
|
24
|
-
|
|
24
|
+
Aspera.assert(OpenSSL::SSL.const_defined?(name), type: Cli::BadArgument){"Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}"}
|
|
25
25
|
if opt.start_with?('-')
|
|
26
26
|
@extra_options &= ~OpenSSL::SSL.const_get(name)
|
|
27
27
|
else
|
|
@@ -637,7 +637,7 @@ properties:
|
|
|
637
637
|
Partial transfers may exist if communication disruptions caused the underlying ascp processes to terminate early.
|
|
638
638
|
Note that transfer resumption can only happen if the `reset` option is disabled.
|
|
639
639
|
If an async session starts with `reset` enabled and resume enabled, transfers interrupted during that
|
|
640
|
-
session will be
|
|
640
|
+
session will be resumable, but only if async is then restarted with 'reset' disabled.
|
|
641
641
|
type: object
|
|
642
642
|
properties:
|
|
643
643
|
enabled:
|
|
@@ -654,7 +654,7 @@ properties:
|
|
|
654
654
|
Sets the age limit in days for temporary files that will be preserved
|
|
655
655
|
on cleanup (usually at async's start and stop) for potential transfer resume.
|
|
656
656
|
Temp files older than the given value will be removed
|
|
657
|
-
regardless of whether they might be
|
|
657
|
+
regardless of whether they might be resumable.
|
|
658
658
|
type: integer
|
|
659
659
|
default: "5"
|
|
660
660
|
x-cli-option: --resume-age-days
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
require 'aspera/ascp/installation'
|
|
6
6
|
require 'aspera/agent/direct'
|
|
7
|
+
require 'aspera/exec_spec'
|
|
7
8
|
require 'aspera/command_line_converter'
|
|
8
9
|
require 'aspera/command_line_builder'
|
|
9
10
|
require 'aspera/log'
|
|
@@ -90,10 +91,7 @@ module Aspera
|
|
|
90
91
|
Log.dump(:sync_params_initial, sync_info)
|
|
91
92
|
Aspera.assert_type(sync_info, Hash)
|
|
92
93
|
Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
|
|
93
|
-
|
|
94
|
-
args: [],
|
|
95
|
-
env: {}
|
|
96
|
-
}
|
|
94
|
+
exec_spec = ExecSpec.new(exec: :async)
|
|
97
95
|
if sync_info.key?('local')
|
|
98
96
|
# `conf` format
|
|
99
97
|
Aspera.assert_type(sync_info['local'], Hash){'local'}
|
|
@@ -115,10 +113,10 @@ module Aspera
|
|
|
115
113
|
remote['private_key_paths'].concat(add_certificates)
|
|
116
114
|
end
|
|
117
115
|
# '--exclusive-mgmt-port=12345', '--arg-err-path=-',
|
|
118
|
-
|
|
116
|
+
exec_spec.args = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_info))}"]
|
|
119
117
|
Log.dump(:sync_conf, sync_info)
|
|
120
118
|
agent = Agent::Direct.new
|
|
121
|
-
agent.start_and_monitor_process(session: {},
|
|
119
|
+
agent.start_and_monitor_process(session: {}, exec_spec: exec_spec)
|
|
122
120
|
else
|
|
123
121
|
# `args` format
|
|
124
122
|
raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
|
|
@@ -141,16 +139,16 @@ module Aspera
|
|
|
141
139
|
Aspera.assert_type(sync_info['instance'], Hash)
|
|
142
140
|
instance_builder = CommandLineBuilder.new(sync_info['instance'], ARGS_INSTANCE_SCHEMA, CommandLineConverter)
|
|
143
141
|
instance_builder.process_params
|
|
144
|
-
instance_builder.
|
|
142
|
+
instance_builder.add_to_exec_spec(exec_spec)
|
|
145
143
|
end
|
|
146
144
|
sync_info['sessions'].each do |session_params|
|
|
147
145
|
Aspera.assert_type(session_params, Hash)
|
|
148
146
|
Aspera.assert(session_params.key?('name'), 'session must contain at least: name')
|
|
149
147
|
session_builder = CommandLineBuilder.new(session_params, ARGS_SESSION_SCHEMA, CommandLineConverter)
|
|
150
148
|
session_builder.process_params
|
|
151
|
-
session_builder.
|
|
149
|
+
session_builder.add_to_exec_spec(exec_spec)
|
|
152
150
|
end
|
|
153
|
-
Environment.secure_execute(Ascp::Installation.instance.path(:async), *
|
|
151
|
+
Environment.secure_execute(Ascp::Installation.instance.path(:async), *exec_spec.args, env: exec_spec.env)
|
|
154
152
|
end
|
|
155
153
|
return
|
|
156
154
|
end
|
|
@@ -216,7 +214,7 @@ module Aspera
|
|
|
216
214
|
return sync_info['local_db_dir']
|
|
217
215
|
elsif (local_path = sync_info.dig('local', 'path'))
|
|
218
216
|
return local_path
|
|
219
|
-
|
|
217
|
+
else
|
|
220
218
|
raise Error, 'Missing either local_db_dir or local.path'
|
|
221
219
|
end
|
|
222
220
|
else
|
|
@@ -226,11 +224,10 @@ module Aspera
|
|
|
226
224
|
return session['local_db_dir']
|
|
227
225
|
elsif session.key?('local_dir')
|
|
228
226
|
return session['local_dir']
|
|
229
|
-
|
|
227
|
+
else
|
|
230
228
|
raise Error, 'Missing either local_db_dir or local_dir'
|
|
231
229
|
end
|
|
232
230
|
end
|
|
233
|
-
nil
|
|
234
231
|
end
|
|
235
232
|
|
|
236
233
|
def session_name(sync_info)
|
|
@@ -325,7 +322,7 @@ module Aspera
|
|
|
325
322
|
end
|
|
326
323
|
end
|
|
327
324
|
path, props = find_option(CONF_SCHEMA, [], option)
|
|
328
|
-
|
|
325
|
+
Aspera.assert(!path.nil?){"Option not found: #{option}"}
|
|
329
326
|
last_key = path.pop
|
|
330
327
|
# navigate in the current result to insert the value
|
|
331
328
|
current = result
|
|
@@ -52,7 +52,7 @@ module Aspera
|
|
|
52
52
|
end
|
|
53
53
|
|
|
54
54
|
# Ensure that provided folder exists, or create it, generate a unique filename
|
|
55
|
-
# @return path to that unique file
|
|
55
|
+
# @return [String] path to that unique file
|
|
56
56
|
def new_file_path_in_folder(temp_folder, prefix: nil, suffix: nil)
|
|
57
57
|
FileUtils.mkdir_p(temp_folder)
|
|
58
58
|
new_file = File.join(temp_folder, [prefix, SecureRandom.uuid, suffix].compact.join('-'))
|
data/lib/aspera/timer_limiter.rb
CHANGED
|
@@ -11,42 +11,55 @@ module Aspera
|
|
|
11
11
|
SIZE_UNITS = %w[k m g t p e].freeze
|
|
12
12
|
private_constant :SCHEME, :PREFIX, :SIZE_UNITS
|
|
13
13
|
class << self
|
|
14
|
-
#
|
|
14
|
+
# Parse a faux URI and return a `FauxFile` instance, or `nil` if the URI does not use the faux scheme.
|
|
15
|
+
# URI format: `faux:///<path>?<size>`, where `<size>` is a decimal integer with an optional
|
|
16
|
+
# case-insensitive unit suffix: `k`, `m`, `g`, `t`, `p`, `e` (powers of 1024).
|
|
17
|
+
# When no suffix is given, the size is interpreted as raw bytes.
|
|
18
|
+
# Examples: `faux:///file.bin?10` (10 bytes), `faux:///file.bin?10m` (10 MiB)
|
|
19
|
+
# @param name [String] source file name, possibly a faux URI
|
|
20
|
+
# @return [FauxFile, nil] `nil` if not a faux scheme, else a `FauxFile` instance
|
|
15
21
|
def create(name)
|
|
16
22
|
return unless name.start_with?(PREFIX)
|
|
17
23
|
name_params = name.delete_prefix(PREFIX).split('?', 2)
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
SIZE_UNITS.each do |s|
|
|
23
|
-
size *= 1024
|
|
24
|
-
break if s.eql?(suffix)
|
|
25
|
-
end
|
|
24
|
+
Aspera.assert(name_params.length.eql?(2), type: Error){"Format: #{PREFIX}<file path>?<size>"}
|
|
25
|
+
m = name_params[1].downcase.match(/^(\d+)([#{SIZE_UNITS.join('')}]?)$/)
|
|
26
|
+
Aspera.assert(m, type: Error){"Format: <integer>[#{SIZE_UNITS.join(',')}]"}
|
|
27
|
+
size = m[2].empty? ? m[1].to_i : m[1].to_i * (1024**(SIZE_UNITS.index(m[2]) + 1))
|
|
26
28
|
return FauxFile.new(name_params[0], size)
|
|
27
29
|
end
|
|
28
30
|
end
|
|
31
|
+
# @return [String] virtual file path
|
|
32
|
+
# @return [Integer] total size in bytes
|
|
29
33
|
attr_reader :path, :size
|
|
30
34
|
|
|
35
|
+
# @param path [String] virtual file path (from the faux URI)
|
|
36
|
+
# @param size [Integer] total number of bytes to produce
|
|
31
37
|
def initialize(path, size)
|
|
32
38
|
@path = path
|
|
33
39
|
@size = size
|
|
34
40
|
@offset = 0
|
|
35
|
-
#
|
|
41
|
+
# cache chunks by size so repeated reads of the same chunk length reuse the same buffer
|
|
36
42
|
@chunk_by_size = {}
|
|
37
43
|
end
|
|
38
44
|
|
|
45
|
+
# Read up to `chunk_size` bytes from the stream and advance the internal offset.
|
|
46
|
+
# Returns `nil` when the stream is exhausted.
|
|
47
|
+
# @param chunk_size [Integer] maximum number of bytes to read
|
|
48
|
+
# @return [String, nil] null-byte string of the bytes actually read, or `nil` at EOF
|
|
39
49
|
def read(chunk_size)
|
|
40
50
|
return if eof?
|
|
41
51
|
bytes_to_read = [chunk_size, @size - @offset].min
|
|
42
52
|
@offset += bytes_to_read
|
|
43
|
-
@chunk_by_size[bytes_to_read]
|
|
53
|
+
@chunk_by_size[bytes_to_read] ||= "\x00" * bytes_to_read
|
|
44
54
|
return @chunk_by_size[bytes_to_read]
|
|
45
55
|
end
|
|
46
56
|
|
|
57
|
+
# No-op: required to satisfy the IO-like interface used by transfer agents.
|
|
58
|
+
# @return [nil]
|
|
47
59
|
def close
|
|
48
60
|
end
|
|
49
61
|
|
|
62
|
+
# @return [Boolean] true when all bytes have been read
|
|
50
63
|
def eof?
|
|
51
64
|
return @offset >= @size
|
|
52
65
|
end
|