aspera-cli 4.21.1 → 4.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +1 -1
- data/CHANGELOG.md +52 -22
- data/CONTRIBUTING.md +69 -148
- data/README.md +929 -668
- data/bin/ascli +5 -14
- data/bin/asession +1 -3
- data/examples/get_proto_file.rb +4 -3
- data/examples/proxy.pac +20 -20
- data/lib/aspera/agent/base.rb +11 -5
- data/lib/aspera/agent/connect.rb +30 -28
- data/lib/aspera/agent/{alpha.rb → desktop.rb} +35 -31
- data/lib/aspera/agent/direct.rb +141 -121
- data/lib/aspera/agent/httpgw.rb +22 -26
- data/lib/aspera/agent/node.rb +14 -11
- data/lib/aspera/agent/transferd.rb +30 -19
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +6 -6
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/httpgw.rb +7 -3
- data/lib/aspera/api/node.rb +10 -8
- data/lib/aspera/ascmd.rb +3 -3
- data/lib/aspera/ascp/installation.rb +53 -72
- data/lib/aspera/ascp/management.rb +1 -1
- data/lib/aspera/assert.rb +11 -2
- data/lib/aspera/cli/error.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +46 -21
- data/lib/aspera/cli/formatter.rb +55 -48
- data/lib/aspera/cli/hints.rb +1 -1
- data/lib/aspera/cli/info.rb +1 -0
- data/lib/aspera/cli/main.rb +192 -170
- data/lib/aspera/cli/manager.rb +18 -18
- data/lib/aspera/cli/plugin.rb +23 -20
- data/lib/aspera/cli/plugin_factory.rb +1 -1
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +247 -159
- data/lib/aspera/cli/plugins/ats.rb +19 -17
- data/lib/aspera/cli/plugins/config.rb +76 -113
- data/lib/aspera/cli/plugins/console.rb +5 -3
- data/lib/aspera/cli/plugins/faspex.rb +39 -35
- data/lib/aspera/cli/plugins/faspex5.rb +111 -84
- data/lib/aspera/cli/plugins/faspio.rb +13 -1
- data/lib/aspera/cli/plugins/httpgw.rb +13 -1
- data/lib/aspera/cli/plugins/node.rb +312 -182
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +6 -6
- data/lib/aspera/cli/plugins/shares.rb +5 -5
- data/lib/aspera/cli/sync_actions.rb +19 -18
- data/lib/aspera/cli/transfer_agent.rb +5 -5
- data/lib/aspera/cli/transfer_progress.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +116 -95
- data/lib/aspera/coverage.rb +8 -5
- data/lib/aspera/environment.rb +26 -17
- data/lib/aspera/faspex_gw.rb +14 -14
- data/lib/aspera/faspex_postproc.rb +10 -11
- data/lib/aspera/hash_ext.rb +4 -14
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +47 -34
- data/lib/aspera/keychain/factory.rb +41 -0
- data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
- data/lib/aspera/keychain/macos_security.rb +19 -11
- data/lib/aspera/log.rb +28 -34
- data/lib/aspera/nagios.rb +6 -6
- data/lib/aspera/node_simulator.rb +8 -8
- data/lib/aspera/oauth/base.rb +14 -7
- data/lib/aspera/oauth/factory.rb +5 -6
- data/lib/aspera/oauth/url_json.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +6 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +13 -10
- data/lib/aspera/preview/options.rb +16 -16
- data/lib/aspera/preview/terminal.rb +4 -4
- data/lib/aspera/preview/utils.rb +15 -17
- data/lib/aspera/products/connect.rb +35 -1
- data/lib/aspera/products/{alpha.rb → desktop.rb} +3 -3
- data/lib/aspera/products/transferd.rb +9 -2
- data/lib/aspera/proxy_auto_config.rb +2 -2
- data/lib/aspera/rest.rb +56 -47
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/secret_hider.rb +12 -5
- data/lib/aspera/ssh.rb +4 -4
- data/lib/aspera/temp_file_manager.rb +5 -4
- data/lib/aspera/transfer/convert.rb +29 -0
- data/lib/aspera/transfer/error_info.rb +66 -66
- data/lib/aspera/transfer/parameters.rb +13 -68
- data/lib/aspera/transfer/spec.rb +5 -6
- data/lib/aspera/transfer/spec.schema.yaml +753 -0
- data/lib/aspera/transfer/spec_doc.rb +62 -0
- data/lib/aspera/transfer/sync.rb +23 -72
- data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
- data/lib/aspera/transfer/uri.rb +6 -6
- data/lib/aspera/uri_reader.rb +18 -1
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +53 -44
- data.tar.gz.sig +0 -0
- metadata +28 -165
- metadata.gz.sig +0 -0
- data/examples/build_exec +0 -74
- data/examples/build_exec_rubyc +0 -40
- data/examples/build_package.sh +0 -28
- data/lib/aspera/transfer/spec.yaml +0 -718
@@ -2,79 +2,101 @@
|
|
2
2
|
|
3
3
|
require 'aspera/log'
|
4
4
|
require 'aspera/assert'
|
5
|
+
require 'yaml'
|
5
6
|
module Aspera
|
6
7
|
# helper class to build command line from a parameter list (key-value hash)
|
7
8
|
# constructor takes hash: { 'param1':'value1', ...}
|
8
9
|
# process_param is called repeatedly with all known parameters
|
9
10
|
# add_env_args is called to get resulting param list and env var (also checks that all params were used)
|
10
11
|
class CommandLineBuilder
|
11
|
-
#
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
12
|
+
# description [String] Description
|
13
|
+
# type [String,Array] Accepted type(s) for non-enum
|
14
|
+
# default [String] Default value if not specified
|
15
|
+
# enum [Array] Set with list of values for enum types accepted in transfer spec
|
16
|
+
# x-cli-envvar [String] Name of env var
|
17
|
+
# x-cli-option [String] Command line option (starts with "-")
|
18
|
+
# x-cli-switch [Bool] true if option has no arg, else by default option has a value
|
19
|
+
# x-cli-special [Bool] true if special handling (defered)
|
20
|
+
# x-cli-convert [String,Hash] Method name for Convert object or Conversion for enum ts to arg
|
21
|
+
# x-agents [Array] Supported agents (for doc only), if not specified: all
|
22
|
+
# x-tspec [Bool,String] (async) true if same name in transfer spec, else name in transfer spec, else ignored
|
23
|
+
# x-deprecation [String] Deprecation message for doc
|
24
|
+
SCHEMA_KEYS = %w[
|
25
|
+
description
|
26
|
+
type
|
27
|
+
default
|
28
|
+
enum
|
29
|
+
required
|
30
|
+
$comment
|
31
|
+
x-cli-envvar
|
32
|
+
x-cli-option
|
33
|
+
x-cli-switch
|
34
|
+
x-cli-special
|
35
|
+
x-cli-convert
|
36
|
+
x-agents
|
37
|
+
x-tspec
|
38
|
+
x-deprecation
|
39
|
+
].freeze
|
40
|
+
|
41
|
+
CLI_AGENT = 'direct'
|
42
|
+
|
43
|
+
private_constant :SCHEMA_KEYS, :CLI_AGENT
|
18
44
|
|
19
45
|
class << self
|
20
|
-
#
|
21
|
-
def
|
22
|
-
|
23
|
-
when 'yes' then return true
|
24
|
-
when 'no' then return false
|
25
|
-
else Aspera.error_unexpected_value(value){'only: yes or no: '}
|
26
|
-
end
|
46
|
+
# @return true if given agent supports that field
|
47
|
+
def supported_by_agent(agent, properties)
|
48
|
+
!properties.key?('x-agents') || properties['x-agents'].include?(agent)
|
27
49
|
end
|
28
50
|
|
29
|
-
# Called by provider of definition before constructor of this class so that
|
30
|
-
def
|
31
|
-
|
32
|
-
|
33
|
-
|
51
|
+
# Called by provider of definition before constructor of this class so that schema has all mandatory fields
|
52
|
+
def read_schema(source_path, suffix=nil)
|
53
|
+
suffix = "_#{suffix}" unless suffix.nil?
|
54
|
+
schema = YAML.load_file("#{source_path[0..-4]}#{suffix}.schema.yaml")
|
55
|
+
schema['properties'].each do |name, properties|
|
56
|
+
Aspera.assert_type(properties, Hash){name}
|
57
|
+
unsupported_keys = properties.keys - SCHEMA_KEYS
|
34
58
|
Aspera.assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
|
35
|
-
Aspera.assert(options.key?(:cli)){"Missing key: cli for #{name}"}
|
36
|
-
Aspera.assert_type(options[:cli], Hash){'Key: cli'}
|
37
|
-
Aspera.assert(options[:cli].key?(:type)){'Missing key: cli.type'}
|
38
|
-
Aspera.assert_values(options[:cli][:type], CLI_OPTION_TYPES){"Unsupported processing type for #{name}"}
|
39
|
-
# by default : optional
|
40
|
-
options[:mandatory] ||= false
|
41
|
-
options[:desc] ||= ''
|
42
|
-
options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
|
43
|
-
cli = options[:cli]
|
44
|
-
unsupported_cli_keys = cli.keys - CLI_KEYS
|
45
|
-
Aspera.assert(unsupported_cli_keys.empty?){"Unsupported cli keys: #{unsupported_cli_keys}"}
|
46
59
|
# by default : string, unless it's without arg
|
47
|
-
|
48
|
-
#
|
49
|
-
|
50
|
-
|
51
|
-
if !cli.key?(:switch) && cli.key?(:type) && CLI_OPTION_TYPE_SWITCH.include?(cli[:type])
|
52
|
-
cli[:switch] = '--' + name.to_s.tr('_', '-')
|
53
|
-
end
|
60
|
+
properties['type'] ||= properties['x-cli-switch'] ? 'boolean' : 'string'
|
61
|
+
# add default cli option name if not present, and if supported in "direct".
|
62
|
+
properties['x-cli-option'] = '--' + name.to_s.tr('_', '-') if !properties.key?('x-cli-option') && !properties['x-cli-envvar'] && (properties.key?('x-cli-switch') || supported_by_agent(CLI_AGENT, properties))
|
63
|
+
properties.freeze
|
54
64
|
end
|
65
|
+
schema['required'] = [] unless schema.key?('required')
|
66
|
+
schema.freeze
|
55
67
|
end
|
56
68
|
end
|
57
69
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
@
|
64
|
-
@params_definition = params_definition
|
70
|
+
# @param [Hash] object with parameters
|
71
|
+
# @param [Hash] schema JSON schema
|
72
|
+
def initialize(object, schema, convert)
|
73
|
+
@object = object # keep reference so that it can be modified by caller before calling `process_params`
|
74
|
+
@schema = schema
|
75
|
+
@convert = convert
|
65
76
|
@result = {
|
66
77
|
env: {},
|
67
78
|
args: []
|
68
79
|
}
|
69
|
-
@
|
80
|
+
@processed_parameters = []
|
81
|
+
end
|
82
|
+
|
83
|
+
# Change required-ness of property in schema
|
84
|
+
def required(name, required)
|
85
|
+
if required
|
86
|
+
@schema['required'].push(name) unless @schema['required'].include?(name)
|
87
|
+
else
|
88
|
+
@schema['required'].delete(name)
|
89
|
+
end
|
70
90
|
end
|
71
91
|
|
72
|
-
#
|
92
|
+
# Add processed parameters to env and args, warns about unused parameters
|
73
93
|
# @param [Hash] env_args with :env and :args
|
74
94
|
def add_env_args(env_args)
|
75
95
|
Log.log.debug{"add_env_args: ENV=#{@result[:env]}, ARGS=#{@result[:args]}"}
|
76
96
|
# warn about non translated arguments
|
77
|
-
@
|
97
|
+
@object.each_pair do |name, value|
|
98
|
+
Log.log.warn{raise "Unknown transfer spec parameter: #{name} = \"#{value}\""} unless @processed_parameters.include?(name)
|
99
|
+
end
|
78
100
|
# set result
|
79
101
|
env_args[:env].merge!(@result[:env])
|
80
102
|
env_args[:args].concat(@result[:args])
|
@@ -84,11 +106,11 @@ module Aspera
|
|
84
106
|
# add options directly to command line
|
85
107
|
def add_command_line_options(options)
|
86
108
|
return if options.nil?
|
87
|
-
options.each{|o
|
109
|
+
options.each{ |o| @result[:args].push(o.to_s)}
|
88
110
|
end
|
89
111
|
|
90
112
|
def process_params
|
91
|
-
@
|
113
|
+
@schema['properties'].each_key do |k|
|
92
114
|
process_param(k)
|
93
115
|
end
|
94
116
|
end
|
@@ -103,81 +125,80 @@ module Aspera
|
|
103
125
|
# @param name [String] of parameter
|
104
126
|
# @param read [TrueClass,FalseClass] read and return value of parameter instead of normal processing (for special)
|
105
127
|
def process_param(name, read: false)
|
106
|
-
|
128
|
+
properties = @schema['properties'][name]
|
107
129
|
# should not happen
|
108
|
-
if
|
130
|
+
if properties.nil?
|
109
131
|
Log.log.warn{"Unknown parameter #{name}"}
|
110
132
|
return
|
111
133
|
end
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
parameter_value = @param_hash[name]
|
134
|
+
# check mandatory parameter (nil is valid value), TODO: change exception ?
|
135
|
+
raise Transfer::Error, "Missing mandatory parameter: #{name}" if @schema['required'].include?(name) && !properties['x-cli-special'] && !@object.key?(name)
|
136
|
+
parameter_value = @object[name]
|
116
137
|
# no default setting
|
117
|
-
# parameter_value=
|
138
|
+
# parameter_value=properties['default'] if parameter_value.nil? and properties.has_key?('default')
|
118
139
|
# Check parameter type
|
119
|
-
expected_classes =
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
140
|
+
expected_classes =
|
141
|
+
[properties['type']].flatten.map do |type|
|
142
|
+
case type
|
143
|
+
when 'string' then [String]
|
144
|
+
when 'array' then [Array]
|
145
|
+
when 'object' then [Hash]
|
146
|
+
when 'integer' then [Integer]
|
147
|
+
when 'boolean' then [TrueClass, FalseClass]
|
148
|
+
else Aspera.error_unexpected_value(properties['type'])
|
149
|
+
end
|
150
|
+
end.flatten
|
129
151
|
# check that value is of expected type
|
130
|
-
raise Transfer::Error, "#{name} is : #{parameter_value.class} (#{parameter_value}), shall be #{
|
152
|
+
raise Transfer::Error, "#{name} is : #{parameter_value.class} (#{parameter_value}), shall be #{properties['type']}, " \
|
131
153
|
unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
|
132
154
|
# special processing will be requested with type get_value
|
133
|
-
@
|
155
|
+
@processed_parameters.push(name) if !properties['x-cli-special'] || read
|
134
156
|
|
135
157
|
# process only non-nil values
|
136
158
|
return nil if parameter_value.nil?
|
137
159
|
|
138
|
-
# check that value is of an accepted type (string,
|
139
|
-
raise "Enum value #{parameter_value} is not allowed for #{name}" if
|
160
|
+
# check that value is of an accepted type (string, integer, boolean)
|
161
|
+
raise "Enum value #{parameter_value} is not allowed for #{name}" if properties.key?('enum') && !properties['enum'].include?(parameter_value)
|
140
162
|
|
141
163
|
# convert some values if value on command line needs processing from value in structure
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
conversion_class, conversion_method = options[:cli][:convert].split('.')
|
151
|
-
converted_value = Kernel.const_get(conversion_class).send(conversion_method, parameter_value)
|
152
|
-
raise Transfer::Error, "unsupported #{name}: #{parameter_value}" if converted_value.nil?
|
164
|
+
if (convert = properties['x-cli-convert'])
|
165
|
+
converted_value =
|
166
|
+
case convert
|
167
|
+
when Hash then convert[parameter_value]
|
168
|
+
when String then @convert.send(convert, parameter_value)
|
169
|
+
else Aspera.error_unexpected_value(convert){"Conversion type for #{name} is Hash or String only."}
|
170
|
+
end
|
171
|
+
raise "No conversion for: #{name}=#{parameter_value}" if converted_value.nil?
|
153
172
|
parameter_value = converted_value
|
154
|
-
when NilClass
|
155
|
-
else Aspera.error_unexpected_value(options[:cli][:convert].class)
|
156
173
|
end
|
157
174
|
|
158
|
-
|
159
|
-
|
175
|
+
return unless self.class.supported_by_agent(CLI_AGENT, properties)
|
176
|
+
|
177
|
+
if read
|
178
|
+
# just get value (deferred)
|
160
179
|
return parameter_value
|
161
|
-
|
180
|
+
elsif properties['x-cli-special']
|
181
|
+
# process later
|
162
182
|
return
|
163
|
-
|
164
|
-
|
165
|
-
@result[:env][
|
166
|
-
|
183
|
+
elsif properties.key?('x-cli-envvar')
|
184
|
+
# set in env var
|
185
|
+
@result[:env][properties['x-cli-envvar']] = parameter_value
|
186
|
+
elsif properties['x-cli-switch']
|
187
|
+
# if present and true : just add option without value
|
167
188
|
add_param = false
|
168
189
|
case parameter_value
|
169
190
|
when false then nil # nothing to put on command line, no creation by default
|
170
191
|
when true then add_param = true
|
171
192
|
else Aspera.error_unexpected_value(parameter_value){name}
|
172
193
|
end
|
173
|
-
add_param = !add_param if
|
174
|
-
add_command_line_options([
|
175
|
-
|
194
|
+
# add_param = !add_param if properties[:add_on_false]
|
195
|
+
add_command_line_options([properties['x-cli-option']]) if add_param
|
196
|
+
else
|
197
|
+
# transform into command line option with value
|
176
198
|
# parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
|
177
199
|
parameter_value = [parameter_value] unless parameter_value.is_a?(Array)
|
178
200
|
# if transfer_spec value is an array, applies option many times
|
179
|
-
parameter_value.each{|v|add_command_line_options([
|
180
|
-
else Aspera.error_unexpected_value(processing_type){processing_type.class.name}
|
201
|
+
parameter_value.each{ |v| add_command_line_options([properties['x-cli-option'], v])}
|
181
202
|
end
|
182
203
|
end
|
183
204
|
end
|
data/lib/aspera/coverage.rb
CHANGED
@@ -5,7 +5,7 @@ if ENV.key?('ENABLE_COVERAGE')
|
|
5
5
|
require 'simplecov'
|
6
6
|
require 'securerandom'
|
7
7
|
# compute development top folder based on this source location
|
8
|
-
development_root = 3.times.inject(File.realpath(__FILE__))
|
8
|
+
development_root = 3.times.inject(File.realpath(__FILE__)){ |p, _| File.dirname(p)}
|
9
9
|
SimpleCov.root(development_root)
|
10
10
|
SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
|
11
11
|
# keep cache data for 1 day (must be longer that time to run the whole test suite)
|
@@ -20,14 +20,17 @@ if ENV.key?('ENABLE_COVERAGE')
|
|
20
20
|
# lines with those words are ignored from coverage
|
21
21
|
no_cov_functions = %w[error_unreachable_line error_unexpected_value Log.log.trace].freeze
|
22
22
|
SimpleCov.start do
|
23
|
-
|
24
|
-
add_filter 'lib/aspera/node_simulator.rb'
|
25
|
-
add_filter 'lib/aspera/keychain/macos_security.rb'
|
23
|
+
# assert usually do not trigger
|
26
24
|
add_filter do |source_file|
|
27
25
|
source_file.lines.each do |line|
|
28
|
-
line.skipped! if no_cov_functions.any?{|i|line.src.include?(i)}
|
26
|
+
line.skipped! if no_cov_functions.any?{ |i| line.src.include?(i)}
|
29
27
|
end
|
30
28
|
false
|
31
29
|
end
|
30
|
+
# no coverage in these files
|
31
|
+
add_filter 'lib/aspera/cli/plugins/faspex.rb'
|
32
|
+
add_filter 'lib/aspera/node_simulator.rb'
|
33
|
+
add_filter 'lib/aspera/keychain/macos_security.rb'
|
34
|
+
add_filter 'lib/aspera/assert.rb'
|
32
35
|
end
|
33
36
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -103,29 +103,34 @@ module Aspera
|
|
103
103
|
def log_spawn(exec:, args: nil, env: nil)
|
104
104
|
[
|
105
105
|
'execute:'.red,
|
106
|
-
env&.map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
|
106
|
+
env&.map{ |k, v| "#{k}=#{Shellwords.shellescape(v)}"},
|
107
107
|
Shellwords.shellescape(exec),
|
108
|
-
args&.map{|a|Shellwords.shellescape(a)}
|
108
|
+
args&.map{ |a| Shellwords.shellescape(a)}
|
109
109
|
].compact.flatten.join(' ')
|
110
110
|
end
|
111
111
|
|
112
|
-
#
|
112
|
+
# Start process in background
|
113
113
|
# caller can call Process.wait on returned value
|
114
|
-
# @param
|
115
|
-
# @param
|
116
|
-
# @param
|
117
|
-
# @
|
118
|
-
|
114
|
+
# @param exec [String] path to executable
|
115
|
+
# @param args [Array, nil] arguments for executable
|
116
|
+
# @param env [Hash, nil] environment variables
|
117
|
+
# @param options [Hash, nil] spawn options
|
118
|
+
# @return [String] PID of process
|
119
|
+
# @raise [Exception] if problem
|
120
|
+
def secure_spawn(exec:, args: nil, env: nil, **options)
|
119
121
|
Aspera.assert_type(exec, String)
|
120
122
|
Aspera.assert_type(args, Array) unless args.nil?
|
121
123
|
Aspera.assert_type(env, Hash) unless env.nil?
|
122
|
-
|
124
|
+
Aspera.assert_type(options, Hash) unless options.nil?
|
125
|
+
Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
|
123
126
|
# start ascp in separate process
|
124
127
|
spawn_args = []
|
125
128
|
spawn_args.push(env) unless env.nil?
|
126
129
|
spawn_args.push([exec, exec])
|
127
130
|
spawn_args.concat(args) unless args.nil?
|
128
|
-
|
131
|
+
opts = {close_others: true}
|
132
|
+
opts.merge!(options) unless options.nil?
|
133
|
+
ascp_pid = Process.spawn(*spawn_args, **opts)
|
129
134
|
Log.log.debug{"pid: #{ascp_pid}"}
|
130
135
|
return ascp_pid
|
131
136
|
end
|
@@ -135,29 +140,33 @@ module Aspera
|
|
135
140
|
# @param exec [String] path to executable
|
136
141
|
# @param args [Array, nil] arguments
|
137
142
|
# @return [String] PID of process
|
138
|
-
def secure_execute(exec:, args: nil, env: nil)
|
143
|
+
def secure_execute(exec:, args: nil, env: nil, **system_args)
|
139
144
|
Aspera.assert_type(exec, String)
|
140
145
|
Aspera.assert_type(args, Array) unless args.nil?
|
141
146
|
Aspera.assert_type(env, Hash) unless env.nil?
|
142
|
-
Log.log.debug
|
143
|
-
# start
|
147
|
+
Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
|
148
|
+
# start in separate process
|
144
149
|
spawn_args = []
|
145
150
|
spawn_args.push(env) unless env.nil?
|
151
|
+
# ensure no shell expansion
|
146
152
|
spawn_args.push([exec, exec])
|
147
153
|
spawn_args.concat(args) unless args.nil?
|
148
|
-
|
154
|
+
kwargs = {exception: true}
|
155
|
+
kwargs.merge!(system_args)
|
156
|
+
Kernel.system(*spawn_args, **kwargs)
|
149
157
|
nil
|
150
158
|
end
|
151
159
|
|
160
|
+
# Execute process and capture stdout
|
152
161
|
# @param exec [String] path to executable
|
153
162
|
# @param args [Array] arguments to executable
|
154
163
|
# @param opts [Hash] options to capture3
|
155
|
-
# @return stdout of executable or raise
|
164
|
+
# @return stdout of executable or raise exception
|
156
165
|
def secure_capture(exec:, args: [], **opts)
|
157
166
|
Aspera.assert_type(exec, String)
|
158
167
|
Aspera.assert_type(args, Array)
|
159
168
|
Aspera.assert_type(opts, Hash)
|
160
|
-
Log.log.debug
|
169
|
+
Log.log.debug{log_spawn(exec: exec, args: args)}
|
161
170
|
stdout, stderr, status = Open3.capture3(exec, *args, **opts)
|
162
171
|
Log.log.debug{"status=#{status}, stderr=#{stderr}"}
|
163
172
|
Log.log.trace1{"stdout=#{stdout}"}
|
@@ -247,7 +256,7 @@ module Aspera
|
|
247
256
|
# https://www.gnu.org/software/libc/manual/html_node/Locale-Categories.html
|
248
257
|
# https://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html
|
249
258
|
def terminal_supports_unicode?
|
250
|
-
@terminal_supports_unicode = self.class.terminal? && %w(LC_ALL LC_CTYPE LANG).any?{|var|ENV[var]&.include?('UTF-8')} if @terminal_supports_unicode.nil?
|
259
|
+
@terminal_supports_unicode = self.class.terminal? && %w(LC_ALL LC_CTYPE LANG).any?{ |var| ENV[var]&.include?('UTF-8')} if @terminal_supports_unicode.nil?
|
251
260
|
return @terminal_supports_unicode
|
252
261
|
end
|
253
262
|
|
data/lib/aspera/faspex_gw.rb
CHANGED
@@ -8,8 +8,8 @@ require 'json'
|
|
8
8
|
module Aspera
|
9
9
|
# Simulate the Faspex 4 /send API and creates a package on Aspera on Cloud or Faspex 5
|
10
10
|
class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
|
11
|
-
# @param app_api
|
12
|
-
# @param app_context [String]
|
11
|
+
# @param app_api [Rest] API object
|
12
|
+
# @param app_context [String] workspace id (aoc only)
|
13
13
|
def initialize(server, app_api, app_context)
|
14
14
|
Aspera.assert_values(app_api.class.name, ['Aspera::Api::AoC', 'Aspera::Rest'])
|
15
15
|
super(server)
|
@@ -32,7 +32,7 @@ module Aspera
|
|
32
32
|
created_package = @app_api.create_package_simple(package_data, true, nil)
|
33
33
|
# but we place it in a Faspex package creation response
|
34
34
|
return {
|
35
|
-
'links' => {
|
35
|
+
'links' => {'status' => 'unused'},
|
36
36
|
'xfer_sessions' => [created_package[:spec]]
|
37
37
|
}
|
38
38
|
end
|
@@ -42,22 +42,22 @@ module Aspera
|
|
42
42
|
package_data = {
|
43
43
|
'title' => faspex_pkg_delivery['title'],
|
44
44
|
'note' => faspex_pkg_delivery['note'],
|
45
|
-
'recipients' => faspex_pkg_delivery['recipients'].map{|name|{'name'=>name}}
|
45
|
+
'recipients' => faspex_pkg_delivery['recipients'].map{ |name| {'name'=>name}}
|
46
46
|
}
|
47
47
|
package = @app_api.create('packages', package_data)
|
48
48
|
# TODO: option to send from remote source or httpgw
|
49
49
|
transfer_spec = @app_api.call(
|
50
|
-
operation:
|
51
|
-
subpath:
|
52
|
-
|
53
|
-
|
54
|
-
body:
|
55
|
-
|
50
|
+
operation: 'POST',
|
51
|
+
subpath: "packages/#{package['id']}/transfer_spec/upload",
|
52
|
+
query: {transfer_type: Cli::Plugins::Faspex5::TRANSFER_CONNECT},
|
53
|
+
content_type: Rest::MIME_JSON,
|
54
|
+
body: {paths: [{'destination'=>'/'}]},
|
55
|
+
headers: {'Accept' => Rest::MIME_JSON}
|
56
56
|
)[:data]
|
57
57
|
transfer_spec.delete('authentication')
|
58
58
|
# but we place it in a Faspex package creation response
|
59
59
|
return {
|
60
|
-
'links' => {
|
60
|
+
'links' => {'status' => 'unused'},
|
61
61
|
'xfer_sessions' => [transfer_spec]
|
62
62
|
}
|
63
63
|
end
|
@@ -80,18 +80,18 @@ module Aspera
|
|
80
80
|
end
|
81
81
|
Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
|
82
82
|
response.status = 200
|
83
|
-
response.content_type =
|
83
|
+
response.content_type = Rest::MIME_JSON
|
84
84
|
response.body = JSON.generate(faspex_package_create_result)
|
85
85
|
rescue => e
|
86
86
|
response.status = 500
|
87
|
-
response['Content-Type'] =
|
87
|
+
response['Content-Type'] = Rest::MIME_JSON
|
88
88
|
response.body = {error: e.message, stacktrace: e.backtrace}.to_json
|
89
89
|
Log.log.error(e.message)
|
90
90
|
Log.log.debug{e.backtrace.join("\n")}
|
91
91
|
end
|
92
92
|
else
|
93
93
|
response.status = 400
|
94
|
-
response['Content-Type'] =
|
94
|
+
response['Content-Type'] = Rest::MIME_JSON
|
95
95
|
response.body = {error: 'Unsupported endpoint'}.to_json
|
96
96
|
end
|
97
97
|
end
|
@@ -15,7 +15,8 @@ module Aspera
|
|
15
15
|
Aspera.assert_type(parameters, Hash)
|
16
16
|
@parameters = parameters.symbolize_keys
|
17
17
|
Log.log.debug{Log.dump(:post_proc_parameters, @parameters)}
|
18
|
-
|
18
|
+
not_allowed = @parameters.keys - ALLOWED_PARAMETERS
|
19
|
+
raise "unsupported parameters: #{not_allowed.join(', ')}" unless not_allowed.empty?
|
19
20
|
@parameters[:script_folder] ||= '.'
|
20
21
|
@parameters[:fail_on_error] ||= false
|
21
22
|
@parameters[:timeout_seconds] ||= 60
|
@@ -29,17 +30,17 @@ module Aspera
|
|
29
30
|
# only accept requests on the root
|
30
31
|
if !request.path.start_with?(@parameters[:root])
|
31
32
|
response.status = 400
|
32
|
-
response['Content-Type'] =
|
33
|
+
response['Content-Type'] = Rest::MIME_JSON
|
33
34
|
response.body = {status: 'error', message: 'Request outside domain'}.to_json
|
34
35
|
return
|
35
36
|
end
|
36
37
|
if request.body.nil?
|
37
38
|
response.status = 400
|
38
|
-
response['Content-Type'] =
|
39
|
+
response['Content-Type'] = Rest::MIME_JSON
|
39
40
|
response.body = {status: 'error', message: 'Empty request'}.to_json
|
40
41
|
return
|
41
42
|
end
|
42
|
-
# build script path by removing domain
|
43
|
+
# build script path by removing domain and adding script folder
|
43
44
|
script_file = request.path[@parameters[:root].size..]
|
44
45
|
Log.log.debug{"script file=#{script_file}"}
|
45
46
|
script_path = File.join(@parameters[:script_folder], script_file)
|
@@ -47,19 +48,17 @@ module Aspera
|
|
47
48
|
webhook_parameters = JSON.parse(request.body)
|
48
49
|
Log.log.debug{Log.dump(:webhook_parameters, webhook_parameters)}
|
49
50
|
# env expects only strings
|
50
|
-
environment = webhook_parameters.each_with_object({})
|
51
|
-
post_proc_pid =
|
52
|
-
Log.log.debug{"pid=#{post_proc_pid}"}
|
53
|
-
raise 'no pid' if post_proc_pid.nil?
|
54
|
-
# "wait" for process to avoid zombie
|
51
|
+
environment = webhook_parameters.each_with_object({}){ |(k, v), h| h[k] = v.to_s}
|
52
|
+
post_proc_pid = Environment.secure_spawn(env: environment, exec: script_path)
|
55
53
|
Timeout.timeout(@parameters[:timeout_seconds]) do
|
54
|
+
# "wait" for process to avoid zombie
|
56
55
|
Process.wait(post_proc_pid)
|
57
56
|
post_proc_pid = nil
|
58
57
|
end
|
59
58
|
process_status = $CHILD_STATUS
|
60
59
|
raise "script #{script_path} failed with code #{process_status.exitstatus}" if !process_status.success? && @parameters[:fail_on_error]
|
61
60
|
response.status = 200
|
62
|
-
response.content_type =
|
61
|
+
response.content_type = Rest::MIME_JSON
|
63
62
|
response.body = JSON.generate({status: 'success', script: script_path, exit_code: process_status.exitstatus})
|
64
63
|
Log.log.debug{'Script executed successfully'}
|
65
64
|
rescue => e
|
@@ -70,7 +69,7 @@ module Aspera
|
|
70
69
|
Log.log.error("Killed process: #{post_proc_pid}")
|
71
70
|
end
|
72
71
|
response.status = 500
|
73
|
-
response['Content-Type'] =
|
72
|
+
response['Content-Type'] = Rest::MIME_JSON
|
74
73
|
response.body = {status: 'error', script: script_path, message: e.message}.to_json
|
75
74
|
end
|
76
75
|
end
|
data/lib/aspera/hash_ext.rb
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
class ::Hash
|
4
4
|
def deep_merge(second)
|
5
|
-
merge(second){|_key, v1, v2|v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge(v2) : v2}
|
5
|
+
merge(second){ |_key, v1, v2| v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge(v2) : v2}
|
6
6
|
end
|
7
7
|
|
8
8
|
def deep_merge!(second)
|
9
|
-
merge!(second){|_key, v1, v2|v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge!(v2) : v2}
|
9
|
+
merge!(second){ |_key, v1, v2| v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge!(v2) : v2}
|
10
10
|
end
|
11
11
|
|
12
12
|
def deep_do(memory=nil, &block)
|
@@ -20,17 +20,7 @@ class ::Hash
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
# in
|
24
|
-
unless Hash.method_defined?(:transform_keys)
|
25
|
-
class Hash
|
26
|
-
def transform_keys
|
27
|
-
raise 'missing block' unless block_given?
|
28
|
-
return each_with_object({}){|(k, v), memo|memo[yield(k)] = v}
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
# rails
|
23
|
+
# Exists in Rails
|
34
24
|
unless Hash.method_defined?(:symbolize_keys)
|
35
25
|
class Hash
|
36
26
|
def symbolize_keys
|
@@ -39,7 +29,7 @@ unless Hash.method_defined?(:symbolize_keys)
|
|
39
29
|
end
|
40
30
|
end
|
41
31
|
|
42
|
-
#
|
32
|
+
# Exists in Rails
|
43
33
|
unless Hash.method_defined?(:stringify_keys)
|
44
34
|
class Hash
|
45
35
|
def stringify_keys
|