aspera-cli 4.26.2 → 4.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +310 -21
- data/CONTRIBUTING.md +99 -14
- data/TODO.md +50 -0
- data/bin/ascli +3 -3
- data/docs/README.md +10977 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +29 -7
- data/lib/aspera/agent/connect.rb +58 -21
- data/lib/aspera/agent/desktop.rb +53 -18
- data/lib/aspera/agent/direct.rb +71 -40
- data/lib/aspera/agent/factory.rb +3 -3
- data/lib/aspera/agent/httpgw.rb +55 -14
- data/lib/aspera/agent/node.rb +34 -2
- data/lib/aspera/agent/transferd.rb +48 -12
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +42 -44
- data/lib/aspera/api/cos_node.rb +10 -9
- data/lib/aspera/api/faspex.rb +16 -15
- data/lib/aspera/api/httpgw.rb +23 -23
- data/lib/aspera/api/node.rb +51 -70
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +32 -30
- data/lib/aspera/ascp/installation.rb +75 -69
- data/lib/aspera/ascp/management.rb +7 -8
- data/lib/aspera/assert.rb +12 -13
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +125 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +141 -0
- data/lib/aspera/cli/context.rb +19 -6
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +27 -26
- data/lib/aspera/cli/formatter.rb +95 -59
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +3 -3
- data/lib/aspera/cli/http.rb +74 -30
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +225 -0
- data/lib/aspera/cli/option_declarator.rb +77 -0
- data/lib/aspera/cli/options.schema.yaml +634 -7
- data/lib/aspera/cli/parser.rb +1217 -0
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1178 -881
- data/lib/aspera/cli/plugins/ats.rb +205 -162
- data/lib/aspera/cli/plugins/base.rb +703 -175
- data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
- data/lib/aspera/cli/plugins/config.rb +434 -757
- data/lib/aspera/cli/plugins/console.rb +107 -65
- data/lib/aspera/cli/plugins/cos.rb +46 -33
- data/lib/aspera/cli/plugins/factory.rb +6 -6
- data/lib/aspera/cli/plugins/faspex5.rb +627 -388
- data/lib/aspera/cli/plugins/faspio.rb +54 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +988 -840
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
- data/lib/aspera/cli/plugins/preview.rb +195 -157
- data/lib/aspera/cli/plugins/server.rb +141 -91
- data/lib/aspera/cli/plugins/shares.rb +343 -110
- data/lib/aspera/cli/preset_actions.rb +159 -0
- data/lib/aspera/cli/preset_manager.rb +82 -39
- data/lib/aspera/cli/result.rb +78 -26
- data/lib/aspera/cli/runner.rb +323 -153
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +89 -64
- data/lib/aspera/cli/terminal_formatter.rb +3 -3
- data/lib/aspera/cli/transfer_actions.rb +90 -0
- data/lib/aspera/cli/transfer_agent.rb +129 -64
- data/lib/aspera/cli/transfer_progress.rb +9 -9
- data/lib/aspera/cli/vault_manager.rb +79 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +28 -30
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +34 -34
- data/lib/aspera/command_line_converter.rb +1 -1
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +2 -1
- data/lib/aspera/dot_container.rb +12 -12
- data/lib/aspera/environment.rb +30 -23
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +5 -5
- data/lib/aspera/faspex_postproc.rb +16 -10
- data/lib/aspera/graphql.rb +37 -0
- data/lib/aspera/hash_ext.rb +8 -2
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/base.rb +10 -3
- data/lib/aspera/keychain/encrypted_hash.rb +92 -17
- data/lib/aspera/keychain/factory.rb +20 -8
- data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
- data/lib/aspera/keychain/macos_security.rb +23 -25
- data/lib/aspera/keychain/one_password_api.rb +86 -0
- data/lib/aspera/keychain/one_password_base.rb +34 -0
- data/lib/aspera/keychain/one_password_cli.rb +98 -0
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +29 -15
- data/lib/aspera/markdown.rb +90 -7
- data/lib/aspera/nagios.rb +8 -8
- data/lib/aspera/node_simulator.rb +35 -27
- data/lib/aspera/oauth/base.rb +9 -10
- data/lib/aspera/oauth/boot.rb +5 -5
- data/lib/aspera/oauth/factory.rb +12 -10
- data/lib/aspera/oauth/jwt.rb +9 -11
- data/lib/aspera/oauth/web.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +12 -12
- data/lib/aspera/persistency_folder.rb +18 -11
- data/lib/aspera/preview/file_types.rb +5 -5
- data/lib/aspera/preview/generator.rb +52 -49
- data/lib/aspera/preview/options.rb +3 -2
- data/lib/aspera/preview/terminal.rb +5 -5
- data/lib/aspera/preview/utils.rb +66 -19
- data/lib/aspera/products/connect.rb +2 -2
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/other.rb +3 -3
- data/lib/aspera/products/transferd.rb +4 -4
- data/lib/aspera/proxy_auto_config.rb +15 -14
- data/lib/aspera/rest.rb +159 -151
- data/lib/aspera/rest_error_analyzer.rb +6 -6
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +11 -10
- data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
- data/lib/aspera/schema/documentation.rb +65 -28
- data/lib/aspera/schema/reader.rb +136 -13
- data/lib/aspera/schema/registry.rb +60 -8
- data/lib/aspera/secret_hider.rb +12 -7
- data/lib/aspera/ssh.rb +64 -31
- data/lib/aspera/ssl.rb +5 -5
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/database.rb +2 -2
- data/lib/aspera/sync/operations.rb +22 -25
- data/lib/aspera/temp_file_manager.rb +25 -5
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +36 -34
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +10 -10
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +12 -3
- data/lib/aspera/uri_reader.rb +52 -15
- data/lib/aspera/web_auth.rb +8 -7
- data/lib/aspera/web_server_simple.rb +15 -12
- data/lib/aspera/yaml.rb +5 -4
- data.tar.gz.sig +0 -0
- metadata +34 -6
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/manager.rb +0 -766
- data/lib/aspera/cli/plugins/faspex.rb +0 -539
- data/lib/aspera/json_rpc.rb +0 -52
- data/lib/aspera/transfer/uri.rb +0 -56
|
@@ -3,12 +3,13 @@
|
|
|
3
3
|
require 'aspera/log'
|
|
4
4
|
require 'aspera/assert'
|
|
5
5
|
require 'aspera/schema/registry'
|
|
6
|
+
require 'aspera/exec_spec'
|
|
6
7
|
require 'yaml'
|
|
7
8
|
module Aspera
|
|
8
9
|
# Helper class to build command line from a parameter list (key-value hash)
|
|
9
10
|
# Constructor takes hash: `{ 'param1':'value1', ...}`
|
|
10
11
|
# `process_param` is called repeatedly with all known parameters
|
|
11
|
-
# `
|
|
12
|
+
# `add_to_exec_spec` is called to get resulting param list and env var (also checks that all params were used)
|
|
12
13
|
class CommandLineBuilder
|
|
13
14
|
# Supported keys in JSON schema
|
|
14
15
|
PROPERTY_KEYS = [
|
|
@@ -22,6 +23,7 @@ module Aspera
|
|
|
22
23
|
'$comment', # [String]
|
|
23
24
|
'x-cli-envvar', # [String] Name of env var
|
|
24
25
|
'x-cli-option', # [String] Command line option (starts with "-")
|
|
26
|
+
'x-cli-false', # [String]. Option to use when switch value is false
|
|
25
27
|
'x-cli-short', # [String] Command line option (starts with "-")
|
|
26
28
|
'x-cli-switch', # [Boolean] `true` if option has no arg, else by default option has a value
|
|
27
29
|
'x-cli-special', # [Boolean] `true` if special handling (deferred)
|
|
@@ -36,7 +38,9 @@ module Aspera
|
|
|
36
38
|
|
|
37
39
|
class << self
|
|
38
40
|
# Called by provider of definition before constructor of this class so that schema has all mandatory fields
|
|
39
|
-
# @
|
|
41
|
+
# @param name_sym [Symbol] name of the schema to read
|
|
42
|
+
# @param ascp [Boolean] `true` if `ascp` mode
|
|
43
|
+
# @return [Aspera::Schema::Reader] validated schema reader
|
|
40
44
|
def read_schema(name_sym, ascp: false)
|
|
41
45
|
validate_schema(Schema::Registry.instance.reader(name_sym), ascp: ascp)
|
|
42
46
|
end
|
|
@@ -50,6 +54,7 @@ module Aspera
|
|
|
50
54
|
|
|
51
55
|
private
|
|
52
56
|
|
|
57
|
+
# `ascp`: mandatory type
|
|
53
58
|
DIRECT_PROPERTIES = %w[x-cli-option x-cli-envvar x-cli-special].freeze
|
|
54
59
|
|
|
55
60
|
# Fill default values for some fields in the schema
|
|
@@ -57,16 +62,16 @@ module Aspera
|
|
|
57
62
|
# @param ascp [Boolean] `true` if `ascp`
|
|
58
63
|
# @return [Schema::Reader] The JSON schema
|
|
59
64
|
def validate_schema(schema, ascp: false)
|
|
60
|
-
Aspera.assert_type(schema, Schema::Reader){'schema'}
|
|
61
|
-
Aspera.assert(schema.current.key?('properties')){"Schema must have 'properties': #{schema}"}
|
|
65
|
+
Aspera.assert_type(schema, Schema::Reader) { 'schema' }
|
|
66
|
+
Aspera.assert(schema.current.key?('properties')) { "Schema must have 'properties': #{schema}" }
|
|
62
67
|
schema.each_property do |property_schema, name, _full_name|
|
|
63
68
|
node = property_schema.current
|
|
64
69
|
unsupported_keys = node.keys - PROPERTY_KEYS
|
|
65
|
-
Aspera.assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
|
|
66
|
-
Aspera.assert(node.key?('type') || node.key?('enum')){"Missing type for #{name} in #{schema.current.dig('description').current}"}
|
|
67
|
-
Aspera.assert(node['type'].eql?('boolean')){"switch must be bool: #{name}"} if node['x-cli-switch'] && !node['x-cli-special']
|
|
70
|
+
Aspera.assert(unsupported_keys.empty?) { "Unsupported definition keys: #{unsupported_keys}" }
|
|
71
|
+
Aspera.assert(node.key?('type') || node.key?('enum')) { "Missing type for #{name} in #{schema.current.dig('description').current}" }
|
|
72
|
+
Aspera.assert(node['type'].eql?('boolean')) { "switch must be bool: #{name}" } if node['x-cli-switch'] && !node['x-cli-special']
|
|
68
73
|
node['x-cli-option'] = "--#{name.to_s.tr('_', '-')}" if node['x-cli-option'].eql?(true) || (node['x-cli-switch'].eql?(true) && !node.key?('x-cli-option'))
|
|
69
|
-
Aspera.assert(DIRECT_PROPERTIES.any?{ |i| node.key?(i)}, name, type: :warn) if ascp && supported_by_agent(:direct, node)
|
|
74
|
+
Aspera.assert(DIRECT_PROPERTIES.any? { |i| node.key?(i) }, name, type: :warn) if ascp && supported_by_agent(:direct, node)
|
|
70
75
|
node.freeze
|
|
71
76
|
end
|
|
72
77
|
schema
|
|
@@ -80,10 +85,7 @@ module Aspera
|
|
|
80
85
|
@object = object # keep reference so that it can be modified by caller before calling `process_params`
|
|
81
86
|
@schema = schema
|
|
82
87
|
@convert = convert
|
|
83
|
-
@result =
|
|
84
|
-
env: {},
|
|
85
|
-
args: []
|
|
86
|
-
}
|
|
88
|
+
@result = ExecSpec.new
|
|
87
89
|
@processed_parameters = []
|
|
88
90
|
end
|
|
89
91
|
|
|
@@ -97,16 +99,16 @@ module Aspera
|
|
|
97
99
|
end
|
|
98
100
|
|
|
99
101
|
# Add processed parameters to env and args, warns about unused parameters
|
|
100
|
-
# @param [
|
|
101
|
-
def
|
|
102
|
-
Log.dump(:
|
|
102
|
+
# @param exec_spec [ExecSpec]
|
|
103
|
+
def add_to_exec_spec(exec_spec)
|
|
104
|
+
Log.dump(:exec_spec, @result)
|
|
103
105
|
# warn about non translated arguments
|
|
104
106
|
@object.each_pair do |name, value|
|
|
105
|
-
Log.log.warn{"Unknown transfer spec parameter: #{name} = \"#{value}\""} unless @processed_parameters.include?(name)
|
|
107
|
+
Log.log.warn { "Unknown transfer spec parameter: #{name} = \"#{value}\"" } unless @processed_parameters.include?(name)
|
|
106
108
|
end
|
|
107
109
|
# set result
|
|
108
|
-
|
|
109
|
-
|
|
110
|
+
exec_spec.env.merge!(@result.env)
|
|
111
|
+
exec_spec.args.concat(@result.args)
|
|
110
112
|
return
|
|
111
113
|
end
|
|
112
114
|
|
|
@@ -114,7 +116,7 @@ module Aspera
|
|
|
114
116
|
def add_command_line_options(*options)
|
|
115
117
|
options = options.first if options.first.is_a?(Array) && options.length.eql?(1)
|
|
116
118
|
Aspera.assert_type(options, Array)
|
|
117
|
-
options.each{ |o| @result
|
|
119
|
+
options.each { |o| @result.args.push(o.to_s) }
|
|
118
120
|
end
|
|
119
121
|
|
|
120
122
|
def process_params
|
|
@@ -136,11 +138,11 @@ module Aspera
|
|
|
136
138
|
properties = @schema['properties'][name]
|
|
137
139
|
# should not happen
|
|
138
140
|
if properties.nil?
|
|
139
|
-
Log.log.warn{"Unknown parameter #{name}"}
|
|
141
|
+
Log.log.warn { "Unknown parameter #{name}" }
|
|
140
142
|
return
|
|
141
143
|
end
|
|
142
144
|
# check mandatory parameter (nil is valid value), TODO: change exception ?
|
|
143
|
-
|
|
145
|
+
Aspera.assert(!(@schema['required']&.include?(name) && !properties['x-cli-special'] && !@object.key?(name)), type: Transfer::Error) { "Missing mandatory parameter: #{name}" }
|
|
144
146
|
parameter_value = @object[name]
|
|
145
147
|
# no default setting
|
|
146
148
|
# parameter_value=properties['default'] if parameter_value.nil? and properties.has_key?('default')
|
|
@@ -153,7 +155,7 @@ module Aspera
|
|
|
153
155
|
when 'object' then [Hash]
|
|
154
156
|
when 'integer' then [Integer]
|
|
155
157
|
when 'boolean' then [TrueClass, FalseClass]
|
|
156
|
-
else Aspera.error_unexpected_value(properties['type']){"Property #{name}"}
|
|
158
|
+
else Aspera.error_unexpected_value(properties['type']) { "Property #{name}" }
|
|
157
159
|
end
|
|
158
160
|
end.flatten
|
|
159
161
|
# check that value is of expected type
|
|
@@ -166,7 +168,7 @@ module Aspera
|
|
|
166
168
|
return if parameter_value.nil?
|
|
167
169
|
|
|
168
170
|
# check that value is of an accepted type (string, integer, boolean)
|
|
169
|
-
|
|
171
|
+
Aspera.assert(!(properties.key?('enum') && !properties['enum'].include?(parameter_value))) { "Enum value #{parameter_value} is not allowed for #{name}" }
|
|
170
172
|
|
|
171
173
|
# convert some values if value on command line needs processing from value in structure
|
|
172
174
|
if (convert = properties['x-cli-convert'])
|
|
@@ -174,9 +176,9 @@ module Aspera
|
|
|
174
176
|
case convert
|
|
175
177
|
when Hash then convert[parameter_value]
|
|
176
178
|
when String then @convert.send(convert, parameter_value)
|
|
177
|
-
else Aspera.error_unexpected_value(convert){"Conversion type for #{name} is Hash or String only."}
|
|
179
|
+
else Aspera.error_unexpected_value(convert) { "Conversion type for #{name} is Hash or String only." }
|
|
178
180
|
end
|
|
179
|
-
|
|
181
|
+
Aspera.assert(!converted_value.nil?) { "No conversion for: #{name}=#{parameter_value}" }
|
|
180
182
|
parameter_value = converted_value
|
|
181
183
|
end
|
|
182
184
|
|
|
@@ -190,23 +192,21 @@ module Aspera
|
|
|
190
192
|
return
|
|
191
193
|
elsif properties.key?('x-cli-envvar')
|
|
192
194
|
# set in env var
|
|
193
|
-
@result
|
|
195
|
+
@result.env[properties['x-cli-envvar']] = parameter_value
|
|
194
196
|
elsif properties['x-cli-switch']
|
|
195
|
-
# if present and true : just add option without value
|
|
196
|
-
add_param = false
|
|
197
197
|
case parameter_value
|
|
198
|
-
when
|
|
199
|
-
|
|
200
|
-
|
|
198
|
+
when true
|
|
199
|
+
add_command_line_options(properties['x-cli-option'])
|
|
200
|
+
when false
|
|
201
|
+
add_command_line_options(properties['x-cli-false']) if properties.key?('x-cli-false')
|
|
202
|
+
else Aspera.error_unexpected_value(parameter_value) { name }
|
|
201
203
|
end
|
|
202
|
-
# add_param = !add_param if properties[:add_on_false]
|
|
203
|
-
add_command_line_options(properties['x-cli-option']) if add_param
|
|
204
204
|
else
|
|
205
205
|
# transform into command line option with value
|
|
206
206
|
# parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
|
|
207
207
|
parameter_value = [parameter_value] unless parameter_value.is_a?(Array)
|
|
208
208
|
# if transfer_spec value is an array, applies option many times
|
|
209
|
-
parameter_value.each{ |v| add_command_line_options(properties['x-cli-option'], v)}
|
|
209
|
+
parameter_value.each { |v| add_command_line_options(properties['x-cli-option'], v) }
|
|
210
210
|
end
|
|
211
211
|
end
|
|
212
212
|
end
|
data/lib/aspera/coverage.rb
CHANGED
|
@@ -27,12 +27,11 @@ if ENV.key?('ENABLE_COVERAGE')
|
|
|
27
27
|
# assert usually do not trigger
|
|
28
28
|
add_filter do |source_file|
|
|
29
29
|
source_file.lines.each do |line|
|
|
30
|
-
line.skipped! if no_cov_functions.any?{ |i| line.src.include?(i)}
|
|
30
|
+
line.skipped! if no_cov_functions.any? { |i| line.src.include?(i) }
|
|
31
31
|
end
|
|
32
32
|
false
|
|
33
33
|
end
|
|
34
34
|
# no coverage in these files
|
|
35
|
-
add_filter 'lib/aspera/cli/plugins/faspex.rb'
|
|
36
35
|
add_filter 'lib/aspera/node_simulator.rb'
|
|
37
36
|
add_filter 'lib/aspera/keychain/macos_security.rb'
|
|
38
37
|
add_filter 'lib/aspera/assert.rb'
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'aspera/assert'
|
|
4
4
|
require 'singleton'
|
|
5
5
|
require 'openssl'
|
|
6
|
+
require 'zlib'
|
|
6
7
|
|
|
7
8
|
module Aspera
|
|
8
9
|
# a simple binary data repository
|
|
@@ -23,7 +24,7 @@ module Aspera
|
|
|
23
24
|
# @return [String] decoded data
|
|
24
25
|
def item(name)
|
|
25
26
|
index = ELEMENTS.index(name)
|
|
26
|
-
|
|
27
|
+
Aspera.assert(index, type: ParameterError) { "Unknown data item #{name} (#{name.class})" }
|
|
27
28
|
raw_data = data(START_INDEX + index)
|
|
28
29
|
case name
|
|
29
30
|
when :dsa, :rsa
|
data/lib/aspera/dot_container.rb
CHANGED
|
@@ -14,7 +14,7 @@ module Aspera
|
|
|
14
14
|
def dotted_to_container(path, value, result = nil)
|
|
15
15
|
Aspera.assert_array_all(path, String)
|
|
16
16
|
# Typed keys
|
|
17
|
-
keys = path.map{ |k| int_or_string(k)}
|
|
17
|
+
keys = path.map { |k| int_or_string(k) }
|
|
18
18
|
# Create, or re-use first level container
|
|
19
19
|
current = (result ||= new_hash_or_array_from_key(keys.first))
|
|
20
20
|
# walk the path, and create sub-containers if necessary
|
|
@@ -48,7 +48,7 @@ module Aspera
|
|
|
48
48
|
end
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
-
# @param [Hash,Array] Container object
|
|
51
|
+
# @param container [Hash,Array] Container object
|
|
52
52
|
def initialize(container)
|
|
53
53
|
Aspera.assert_type(container, Hash)
|
|
54
54
|
# tail (pop,push) contains the next element to display
|
|
@@ -72,20 +72,20 @@ module Aspera
|
|
|
72
72
|
when Hash
|
|
73
73
|
add_elements(path, current)
|
|
74
74
|
when Array
|
|
75
|
-
# Array has no nested structures
|
|
76
|
-
if current.none?{ |i| i.is_a?(Array) || i.is_a?(Hash)}
|
|
75
|
+
# Array has no nested structures: list of Strings
|
|
76
|
+
if current.none? { |i| i.is_a?(Array) || i.is_a?(Hash) }
|
|
77
77
|
to_insert = current.map(&:to_s)
|
|
78
|
-
# Array of Hashes with only 'name' keys
|
|
79
|
-
elsif current.all?{ |i| i.is_a?(Hash) && i.keys == ['name']}
|
|
80
|
-
to_insert = current.map{ |i| i['name']}
|
|
81
|
-
# Array of Hashes with only 'name' and 'value' keys
|
|
82
|
-
elsif current.all?{ |i| i.is_a?(Hash) && i.key?('name') && i.key?('value') && i.length <= 3}
|
|
78
|
+
# Array of Hashes with only 'name' keys: list of Strings
|
|
79
|
+
elsif current.all? { |i| i.is_a?(Hash) && i.keys == ['name'] }
|
|
80
|
+
to_insert = current.map { |i| i['name'] }
|
|
81
|
+
# Array of Hashes with only 'name' and 'value' keys: Hash of key/values
|
|
82
|
+
elsif current.all? { |i| i.is_a?(Hash) && i.key?('name') && i.key?('value') && i.length <= 3 }
|
|
83
83
|
# if there is an extra key, other than 'name' and 'value', insert that key as is
|
|
84
|
-
add_elements(path, current.flat_map{ |h| h.except('name', 'value').to_a})
|
|
84
|
+
add_elements(path, current.flat_map { |h| h.except('name', 'value').to_a })
|
|
85
85
|
# Insert name/value pairs as Hash
|
|
86
|
-
add_elements(path, current.to_h{ |h| h.values_at('name', 'value')})
|
|
86
|
+
add_elements(path, current.to_h { |h| h.values_at('name', 'value') })
|
|
87
87
|
else
|
|
88
|
-
add_elements(path, current.each_with_index.map{ |v, i| [i, v]})
|
|
88
|
+
add_elements(path, current.each_with_index.map { |v, i| [i, v] })
|
|
89
89
|
end
|
|
90
90
|
else
|
|
91
91
|
to_insert = current
|
data/lib/aspera/environment.rb
CHANGED
|
@@ -78,7 +78,9 @@ module Aspera
|
|
|
78
78
|
argv
|
|
79
79
|
end
|
|
80
80
|
|
|
81
|
-
#
|
|
81
|
+
# Like `Shellwords.shellescape`, but does not escape `=`
|
|
82
|
+
# @param str [String] String to escape for shell usage
|
|
83
|
+
# @return [String] Shell-safe string
|
|
82
84
|
def shell_escape_pretty(str)
|
|
83
85
|
# Safe unquoted characters + '=' explicitly allowed
|
|
84
86
|
return str if str.match?(%r{\A[A-Za-z0-9_.,:/@+=-]+\z})
|
|
@@ -116,11 +118,11 @@ module Aspera
|
|
|
116
118
|
def secure_execute(*cmd, mode: :execute, **kwargs)
|
|
117
119
|
cmd = cmd.map(&:to_s)
|
|
118
120
|
Aspera.assert(cmd.size.positive?, 'executable must be present', type: ArgumentError)
|
|
119
|
-
Aspera.assert_values(mode, PROCESS_MODES, type: ArgumentError){'mode'}
|
|
121
|
+
Aspera.assert_values(mode, PROCESS_MODES, type: ArgumentError) { 'mode' }
|
|
120
122
|
Log.log.debug do
|
|
121
123
|
parts = [mode.to_s, 'command:']
|
|
122
|
-
kwargs[:env]&.each{ |k, v| parts << "#{k}=#{shell_escape_pretty(v.to_s)}"}
|
|
123
|
-
cmd.each{ |a| parts << shell_escape_pretty(a)}
|
|
124
|
+
kwargs[:env]&.each { |k, v| parts << "#{k}=#{shell_escape_pretty(v.to_s)}" }
|
|
125
|
+
cmd.each { |a| parts << shell_escape_pretty(a) }
|
|
124
126
|
parts.join(' ')
|
|
125
127
|
end
|
|
126
128
|
case mode
|
|
@@ -140,7 +142,7 @@ module Aspera
|
|
|
140
142
|
# https://docs.ruby-lang.org/en/master/Open3.html#method-c-capture3
|
|
141
143
|
# https://docs.ruby-lang.org/en/master/Process.html#module-Process-label-Execution+Options
|
|
142
144
|
argv = [kwargs.delete(:env)].compact + cmd
|
|
143
|
-
exception = kwargs.delete(:exception){true}
|
|
145
|
+
exception = kwargs.delete(:exception) { true }
|
|
144
146
|
result = Open3.capture3(*argv, **kwargs)
|
|
145
147
|
Log.dump(:stdout, result[0], level: :trace1)
|
|
146
148
|
Log.dump(:stderr, result[1], level: :trace1)
|
|
@@ -168,7 +170,10 @@ module Aspera
|
|
|
168
170
|
return path
|
|
169
171
|
end
|
|
170
172
|
|
|
171
|
-
#
|
|
173
|
+
# Restrict access to a file or folder to the current user only (chmod 600/700)
|
|
174
|
+
# @param path [String] Path to the file or directory
|
|
175
|
+
# @param mode [Integer, nil] Octal permission mode; if nil, inferred from path type
|
|
176
|
+
# @return [nil]
|
|
172
177
|
def restrict_file_access(path, mode: nil)
|
|
173
178
|
if mode.nil?
|
|
174
179
|
# or FileUtils ?
|
|
@@ -177,37 +182,38 @@ module Aspera
|
|
|
177
182
|
elsif File.directory?(path)
|
|
178
183
|
mode = 0o700
|
|
179
184
|
else
|
|
180
|
-
Log.log.debug{"No restriction can be set for #{path}"}
|
|
185
|
+
Log.log.debug { "No restriction can be set for #{path}" }
|
|
181
186
|
end
|
|
182
187
|
end
|
|
183
188
|
File.chmod(mode, path) unless mode.nil?
|
|
189
|
+
nil
|
|
184
190
|
rescue => e
|
|
185
191
|
Log.log.warn(e.message)
|
|
186
192
|
end
|
|
187
193
|
|
|
188
|
-
# @return true if we are in a terminal
|
|
194
|
+
# @return [Boolean] true if we are in a terminal
|
|
189
195
|
def terminal?
|
|
190
196
|
$stdout.tty?
|
|
191
197
|
end
|
|
192
198
|
|
|
193
199
|
# force locale to C so that unicode characters are not used
|
|
194
200
|
def force_terminal_c
|
|
195
|
-
I18N_VARS.each{ |var| ENV[var] = 'C'}
|
|
201
|
+
I18N_VARS.each { |var| ENV[var] = 'C' }
|
|
196
202
|
end
|
|
197
203
|
|
|
198
|
-
# @return true if we can display Unicode characters
|
|
204
|
+
# @return [Boolean] true if we can display Unicode characters
|
|
199
205
|
# Uses Encoding.locale_charmap for OS-independent detection.
|
|
200
206
|
# Falls back to locale env vars for systems where charmap is not available.
|
|
201
207
|
# https://www.gnu.org/software/libc/manual/html_node/Locale-Categories.html
|
|
202
208
|
# https://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html
|
|
203
209
|
def terminal_supports_unicode?
|
|
204
210
|
return false unless terminal?
|
|
205
|
-
locale_charmap_utf8? || I18N_VARS.any?{ |var| ENV[var]&.include?('UTF-8')}
|
|
211
|
+
locale_charmap_utf8? || I18N_VARS.any? { |var| ENV[var]&.include?('UTF-8') }
|
|
206
212
|
end
|
|
207
213
|
|
|
208
214
|
private
|
|
209
215
|
|
|
210
|
-
# @return true if the locale charmap resolves to UTF-8
|
|
216
|
+
# @return [Boolean] true if the locale charmap resolves to UTF-8
|
|
211
217
|
# Unix: nl_langinfo(CODESET) returns "UTF-8"
|
|
212
218
|
# Windows: GetACP returns "CP65001" which Encoding.find resolves as UTF-8
|
|
213
219
|
def locale_charmap_utf8?
|
|
@@ -235,7 +241,7 @@ module Aspera
|
|
|
235
241
|
OS_LINUX
|
|
236
242
|
when /aix/
|
|
237
243
|
OS_AIX
|
|
238
|
-
else Aspera.error_unexpected_value(RbConfig::CONFIG['host_os']){'host_os'}
|
|
244
|
+
else Aspera.error_unexpected_value(RbConfig::CONFIG['host_os']) { 'host_os' }
|
|
239
245
|
end
|
|
240
246
|
@cpu =
|
|
241
247
|
case RbConfig::CONFIG['host_cpu']
|
|
@@ -247,7 +253,7 @@ module Aspera
|
|
|
247
253
|
CPU_S390
|
|
248
254
|
when /arm/, /aarch64/
|
|
249
255
|
CPU_ARM64
|
|
250
|
-
else Aspera.error_unexpected_value(RbConfig::CONFIG['host_cpu']){'host_cpu'}
|
|
256
|
+
else Aspera.error_unexpected_value(RbConfig::CONFIG['host_cpu']) { 'host_cpu' }
|
|
251
257
|
end
|
|
252
258
|
@executable_extension = @os.eql?(OS_WINDOWS) ? '.exe' : nil
|
|
253
259
|
# :text or :graphical depending on the environment
|
|
@@ -283,7 +289,7 @@ module Aspera
|
|
|
283
289
|
def fix_home
|
|
284
290
|
return unless @os.eql?(OS_WINDOWS) && ENV.key?('USERPROFILE') && Dir.exist?(ENV.fetch('USERPROFILE', nil))
|
|
285
291
|
ENV['HOME'] = ENV.fetch('USERPROFILE', nil)
|
|
286
|
-
Log.log.debug{"Windows: set HOME to USERPROFILE: #{Dir.home}"}
|
|
292
|
+
Log.log.debug { "Windows: set HOME to USERPROFILE: #{Dir.home}" }
|
|
287
293
|
nil
|
|
288
294
|
end
|
|
289
295
|
|
|
@@ -291,16 +297,17 @@ module Aspera
|
|
|
291
297
|
@default_gui_mode == :graphical
|
|
292
298
|
end
|
|
293
299
|
|
|
294
|
-
# Open a URI in
|
|
295
|
-
#
|
|
296
|
-
# @
|
|
300
|
+
# Open a URI in the system's default graphical browser (non-blocking)
|
|
301
|
+
# @param uri [String, URI] the URI to open
|
|
302
|
+
# @return [nil]
|
|
297
303
|
def open_uri_graphical(uri)
|
|
298
304
|
case @os
|
|
299
|
-
when Environment::OS_MACOS then
|
|
300
|
-
when Environment::OS_WINDOWS then
|
|
301
|
-
when Environment::OS_LINUX then
|
|
302
|
-
else Aspera.error_unexpected_value(os){'no graphical open method'}
|
|
305
|
+
when Environment::OS_MACOS then self.class.secure_execute('open', uri.to_s)
|
|
306
|
+
when Environment::OS_WINDOWS then self.class.secure_execute('start', 'explorer', %Q{"#{uri}"})
|
|
307
|
+
when Environment::OS_LINUX then self.class.secure_execute('xdg-open', uri.to_s)
|
|
308
|
+
else Aspera.error_unexpected_value(os) { 'no graphical open method' }
|
|
303
309
|
end
|
|
310
|
+
nil
|
|
304
311
|
end
|
|
305
312
|
|
|
306
313
|
# open a file in an editor
|
|
@@ -329,7 +336,7 @@ module Aspera
|
|
|
329
336
|
else
|
|
330
337
|
puts "USER ACTION: open this:\n#{the_url.to_s.red}\n"
|
|
331
338
|
end
|
|
332
|
-
else Aspera.error_unexpected_value(@url_method){'URL open method'}
|
|
339
|
+
else Aspera.error_unexpected_value(@url_method) { 'URL open method' }
|
|
333
340
|
end
|
|
334
341
|
end
|
|
335
342
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aspera
|
|
4
|
+
# Holds the executable name, environment variables and command line arguments for a spawned process.
|
|
5
|
+
ExecSpec = Struct.new(:exec, :env, :args, keyword_init: true) do
|
|
6
|
+
def initialize(exec: nil, env: {}, args: [], **) = super
|
|
7
|
+
|
|
8
|
+
# @return [ExecSpec] deep copy with independent :args and :env
|
|
9
|
+
def deep_clone
|
|
10
|
+
self.class.new(exec: exec, env: env.dup, args: args.dup)
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
data/lib/aspera/faspex_gw.rb
CHANGED
|
@@ -44,7 +44,7 @@ module Aspera
|
|
|
44
44
|
package_data = {
|
|
45
45
|
'title' => faspex_pkg_delivery['title'],
|
|
46
46
|
'note' => faspex_pkg_delivery['note'],
|
|
47
|
-
'recipients' => faspex_pkg_delivery['recipients'].map{ |name| {'name'=>name}}
|
|
47
|
+
'recipients' => faspex_pkg_delivery['recipients'].map { |name| {'name'=>name} }
|
|
48
48
|
}
|
|
49
49
|
package = @app_api.create('packages', package_data)
|
|
50
50
|
# TODO: option to send from remote source or httpgw
|
|
@@ -70,7 +70,7 @@ module Aspera
|
|
|
70
70
|
begin
|
|
71
71
|
Aspera.assert(!request.body.nil?, 'payload missing')
|
|
72
72
|
faspex_pkg_parameters = JSON.parse(request.body)
|
|
73
|
-
Log.
|
|
73
|
+
Log.dump(:faspex_pkg_parameters, faspex_pkg_parameters)
|
|
74
74
|
# compare string, as class is not yet known here
|
|
75
75
|
faspex_package_create_result =
|
|
76
76
|
case @app_api.class.name
|
|
@@ -80,16 +80,16 @@ module Aspera
|
|
|
80
80
|
faspex4_send_to_faspex5(faspex_pkg_parameters)
|
|
81
81
|
else Aspera.error_unexpected_value(@app_api.class.name)
|
|
82
82
|
end
|
|
83
|
-
Log.
|
|
83
|
+
Log.dump(:faspex_package_create_result, faspex_package_create_result)
|
|
84
84
|
response.status = 200
|
|
85
85
|
response.content_type = Mime::JSON
|
|
86
86
|
response.body = JSON.generate(faspex_package_create_result)
|
|
87
|
-
rescue => e
|
|
87
|
+
rescue StandardError => e
|
|
88
88
|
response.status = 500
|
|
89
89
|
response['Content-Type'] = Mime::JSON
|
|
90
90
|
response.body = {error: e.message, stacktrace: e.backtrace}.to_json
|
|
91
91
|
Log.log.error(e.message)
|
|
92
|
-
Log.log.debug{e.backtrace.join("\n")}
|
|
92
|
+
Log.log.debug { e.backtrace.join("\n") }
|
|
93
93
|
end
|
|
94
94
|
else
|
|
95
95
|
response.status = 400
|
|
@@ -16,18 +16,18 @@ module Aspera
|
|
|
16
16
|
@parameters = parameters.symbolize_keys
|
|
17
17
|
Log.dump(:post_proc_parameters, @parameters)
|
|
18
18
|
not_allowed = @parameters.keys - ALLOWED_PARAMETERS
|
|
19
|
-
|
|
19
|
+
Aspera.assert(not_allowed.empty?) { "unsupported parameters: #{not_allowed.join(', ')}" }
|
|
20
20
|
@parameters[:script_folder] ||= '.'
|
|
21
21
|
@parameters[:fail_on_error] ||= false
|
|
22
22
|
@parameters[:timeout_seconds] ||= 60
|
|
23
23
|
super(server)
|
|
24
|
-
Log.log.debug{'Faspex4PostProcServlet initialized'}
|
|
24
|
+
Log.log.debug { 'Faspex4PostProcServlet initialized' }
|
|
25
25
|
end
|
|
26
26
|
|
|
27
27
|
# :reek:UncommunicativeMethodName
|
|
28
28
|
def do_POST(request, response)
|
|
29
|
-
Log.log.debug{"request=#{request.path}"}
|
|
30
|
-
Log.
|
|
29
|
+
Log.log.debug { "request=#{request.path}" }
|
|
30
|
+
Log.dump(:query, request.query)
|
|
31
31
|
begin
|
|
32
32
|
# Only accept requests on the root
|
|
33
33
|
if !request.path.start_with?(@parameters[:root])
|
|
@@ -44,17 +44,23 @@ module Aspera
|
|
|
44
44
|
end
|
|
45
45
|
# build script path by removing domain and adding script folder
|
|
46
46
|
script_file = request.path[@parameters[:root].size..]
|
|
47
|
-
Log.log.debug{"script file=#{script_file}"}
|
|
47
|
+
Log.log.debug { "script file=#{script_file}" }
|
|
48
48
|
script_path = File.join(@parameters[:script_folder], script_file)
|
|
49
|
-
|
|
49
|
+
# Resolve the real path and ensure it stays within script_folder (prevents path traversal)
|
|
50
|
+
resolved_script_folder = File.realpath(@parameters[:script_folder])
|
|
51
|
+
resolved_script_path = File.realpath(script_path)
|
|
52
|
+
Aspera.assert(resolved_script_path.start_with?("#{resolved_script_folder}/")) { 'Script path traversal attempt detected' }
|
|
53
|
+
script_path = resolved_script_path
|
|
54
|
+
Log.log.debug { "script=#{script_path}" }
|
|
50
55
|
webhook_parameters = JSON.parse(request.body)
|
|
51
56
|
Log.dump(:webhook_parameters, webhook_parameters)
|
|
57
|
+
process_status = nil
|
|
52
58
|
if request.query.key?('lambda')
|
|
53
59
|
# Code can throw exception, source code must return a lambda
|
|
54
60
|
Environment.secure_eval(File.read(script_path), __FILE__, __LINE__).call(webhook_parameters)
|
|
55
61
|
else
|
|
56
62
|
# env expects only strings
|
|
57
|
-
environment = webhook_parameters.each_with_object({}){ |(k, v), h| h[k] = v.to_s}
|
|
63
|
+
environment = webhook_parameters.each_with_object({}) { |(k, v), h| h[k] = v.to_s }
|
|
58
64
|
post_proc_pid = Environment.secure_execute(script_path, mode: :background, env: environment)
|
|
59
65
|
Timeout.timeout(@parameters[:timeout_seconds]) do
|
|
60
66
|
# "wait" for process to avoid zombie
|
|
@@ -62,12 +68,12 @@ module Aspera
|
|
|
62
68
|
post_proc_pid = nil
|
|
63
69
|
end
|
|
64
70
|
process_status = $CHILD_STATUS
|
|
65
|
-
|
|
71
|
+
Aspera.assert(process_status.success? || !@parameters[:fail_on_error]) { "script #{script_path} failed with code #{process_status.exitstatus}" }
|
|
66
72
|
end
|
|
67
73
|
response.status = 200
|
|
68
74
|
response.content_type = Mime::JSON
|
|
69
|
-
response.body = JSON.generate({status: 'success', script: script_path, exit_code: process_status
|
|
70
|
-
Log.log.debug{'Script executed successfully'}
|
|
75
|
+
response.body = JSON.generate({status: 'success', script: script_path, exit_code: process_status&.exitstatus})
|
|
76
|
+
Log.log.debug { 'Script executed successfully' }
|
|
71
77
|
rescue => e
|
|
72
78
|
Log.log.error("Script failed: #{e.class}:#{e.message}")
|
|
73
79
|
if !post_proc_pid.nil?
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aspera/rest_call_error'
|
|
4
|
+
|
|
5
|
+
module Aspera
|
|
6
|
+
# Helper to execute GraphQL queries via a Rest object.
|
|
7
|
+
# Follows the GraphQL-over-HTTP convention: POST {query, variables} to endpoint,
|
|
8
|
+
# parse top-level `data` key, raise on top-level `errors` array.
|
|
9
|
+
module GraphQL
|
|
10
|
+
QUERIES_FOLDER = File.join(__dir__, 'api', 'queries')
|
|
11
|
+
private_constant :QUERIES_FOLDER
|
|
12
|
+
|
|
13
|
+
class << self
|
|
14
|
+
# Execute a GraphQL query loaded from a .graphql file in lib/aspera/api/queries/
|
|
15
|
+
# @param rest [Rest] API object pointing at the GraphQL endpoint
|
|
16
|
+
# @param name [String] Base filename without extension (e.g. 'my_query')
|
|
17
|
+
# @param variables [Hash] GraphQL variables
|
|
18
|
+
# @return [Hash] The `data` object from the response
|
|
19
|
+
# @raise [RestCallError] if the response contains GraphQL errors
|
|
20
|
+
def execute(rest, name, variables = {})
|
|
21
|
+
query = load_query(name)
|
|
22
|
+
response = rest.create(nil, {query: query, variables: variables})
|
|
23
|
+
errors = response['errors']
|
|
24
|
+
raise RestCallError, errors.map { |e| e['message'] }.join("\n") if errors.is_a?(Array) && !errors.empty?
|
|
25
|
+
response['data']
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def load_query(name)
|
|
31
|
+
candidate = File.realpath(File.join(QUERIES_FOLDER, "#{name}.graphql"))
|
|
32
|
+
Aspera.assert(candidate.start_with?("#{QUERIES_FOLDER}/")) { "Invalid GraphQL query name: #{name}" }
|
|
33
|
+
File.read(candidate)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
data/lib/aspera/hash_ext.rb
CHANGED
|
@@ -1,12 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
class ::Hash
|
|
4
|
+
# Recursively merge another hash, with nested hashes merged deeply
|
|
5
|
+
# @param second [Hash] Hash to merge into self
|
|
6
|
+
# @return [Hash] New merged hash
|
|
4
7
|
def deep_merge(second)
|
|
5
|
-
merge(second){ |_key, v1, v2| v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge(v2) : v2}
|
|
8
|
+
merge(second) { |_key, v1, v2| v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge(v2) : v2 }
|
|
6
9
|
end
|
|
7
10
|
|
|
11
|
+
# Recursively merge another hash in-place, with nested hashes merged deeply
|
|
12
|
+
# @param second [Hash] Hash to merge into self
|
|
13
|
+
# @return [Hash] self
|
|
8
14
|
def deep_merge!(second)
|
|
9
|
-
merge!(second){ |_key, v1, v2| v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge!(v2) : v2}
|
|
15
|
+
merge!(second) { |_key, v1, v2| v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge!(v2) : v2 }
|
|
10
16
|
end
|
|
11
17
|
|
|
12
18
|
# Recursively iterate through hash and execute block on leaf values
|