aspera-cli 4.21.2 → 4.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +1 -1
- data/CHANGELOG.md +34 -16
- data/CONTRIBUTING.md +6 -10
- data/README.md +805 -574
- data/examples/get_proto_file.rb +1 -1
- data/lib/aspera/agent/base.rb +9 -5
- data/lib/aspera/agent/connect.rb +30 -28
- data/lib/aspera/agent/desktop.rb +29 -25
- data/lib/aspera/agent/direct.rb +137 -125
- data/lib/aspera/agent/httpgw.rb +22 -26
- data/lib/aspera/agent/node.rb +14 -11
- data/lib/aspera/agent/transferd.rb +6 -2
- data/lib/aspera/api/aoc.rb +6 -6
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/httpgw.rb +7 -3
- data/lib/aspera/api/node.rb +6 -4
- data/lib/aspera/ascmd.rb +3 -3
- data/lib/aspera/ascp/installation.rb +15 -16
- data/lib/aspera/ascp/management.rb +1 -1
- data/lib/aspera/assert.rb +11 -2
- data/lib/aspera/cli/error.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +38 -19
- data/lib/aspera/cli/formatter.rb +48 -48
- data/lib/aspera/cli/hints.rb +1 -1
- data/lib/aspera/cli/main.rb +190 -168
- data/lib/aspera/cli/manager.rb +15 -15
- data/lib/aspera/cli/plugin.rb +23 -20
- data/lib/aspera/cli/plugin_factory.rb +1 -1
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +144 -107
- data/lib/aspera/cli/plugins/ats.rb +19 -17
- data/lib/aspera/cli/plugins/config.rb +67 -83
- data/lib/aspera/cli/plugins/console.rb +5 -3
- data/lib/aspera/cli/plugins/faspex.rb +39 -35
- data/lib/aspera/cli/plugins/faspex5.rb +104 -80
- data/lib/aspera/cli/plugins/faspio.rb +13 -1
- data/lib/aspera/cli/plugins/httpgw.rb +13 -1
- data/lib/aspera/cli/plugins/node.rb +306 -179
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +6 -6
- data/lib/aspera/cli/plugins/shares.rb +5 -5
- data/lib/aspera/cli/sync_actions.rb +19 -18
- data/lib/aspera/cli/transfer_agent.rb +5 -5
- data/lib/aspera/cli/transfer_progress.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +116 -95
- data/lib/aspera/coverage.rb +4 -3
- data/lib/aspera/environment.rb +6 -6
- data/lib/aspera/faspex_gw.rb +14 -14
- data/lib/aspera/faspex_postproc.rb +7 -6
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +47 -34
- data/lib/aspera/keychain/factory.rb +41 -0
- data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
- data/lib/aspera/keychain/macos_security.rb +19 -11
- data/lib/aspera/log.rb +28 -34
- data/lib/aspera/nagios.rb +6 -6
- data/lib/aspera/node_simulator.rb +8 -8
- data/lib/aspera/oauth/base.rb +8 -6
- data/lib/aspera/oauth/factory.rb +5 -6
- data/lib/aspera/oauth/url_json.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +6 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +1 -1
- data/lib/aspera/preview/options.rb +16 -16
- data/lib/aspera/preview/terminal.rb +3 -3
- data/lib/aspera/preview/utils.rb +11 -13
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +2 -2
- data/lib/aspera/rest.rb +52 -43
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/secret_hider.rb +5 -5
- data/lib/aspera/ssh.rb +4 -4
- data/lib/aspera/transfer/convert.rb +29 -0
- data/lib/aspera/transfer/error_info.rb +66 -66
- data/lib/aspera/transfer/parameters.rb +13 -68
- data/lib/aspera/transfer/spec.rb +5 -6
- data/lib/aspera/transfer/spec.schema.yaml +753 -0
- data/lib/aspera/transfer/spec_doc.rb +62 -0
- data/lib/aspera/transfer/sync.rb +23 -72
- data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
- data/lib/aspera/transfer/uri.rb +6 -6
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +53 -44
- data.tar.gz.sig +1 -2
- metadata +37 -4
- metadata.gz.sig +0 -0
- data/examples/build_package.sh +0 -28
- data/lib/aspera/transfer/spec.yaml +0 -718
@@ -2,79 +2,101 @@
|
|
2
2
|
|
3
3
|
require 'aspera/log'
|
4
4
|
require 'aspera/assert'
|
5
|
+
require 'yaml'
|
5
6
|
module Aspera
|
6
7
|
# helper class to build command line from a parameter list (key-value hash)
|
7
8
|
# constructor takes hash: { 'param1':'value1', ...}
|
8
9
|
# process_param is called repeatedly with all known parameters
|
9
10
|
# add_env_args is called to get resulting param list and env var (also checks that all params were used)
|
10
11
|
class CommandLineBuilder
|
11
|
-
#
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
12
|
+
# description [String] Description
|
13
|
+
# type [String,Array] Accepted type(s) for non-enum
|
14
|
+
# default [String] Default value if not specified
|
15
|
+
# enum [Array] Set with list of values for enum types accepted in transfer spec
|
16
|
+
# x-cli-envvar [String] Name of env var
|
17
|
+
# x-cli-option [String] Command line option (starts with "-")
|
18
|
+
# x-cli-switch [Bool] true if option has no arg, else by default option has a value
|
19
|
+
# x-cli-special [Bool] true if special handling (defered)
|
20
|
+
# x-cli-convert [String,Hash] Method name for Convert object or Conversion for enum ts to arg
|
21
|
+
# x-agents [Array] Supported agents (for doc only), if not specified: all
|
22
|
+
# x-tspec [Bool,String] (async) true if same name in transfer spec, else name in transfer spec, else ignored
|
23
|
+
# x-deprecation [String] Deprecation message for doc
|
24
|
+
SCHEMA_KEYS = %w[
|
25
|
+
description
|
26
|
+
type
|
27
|
+
default
|
28
|
+
enum
|
29
|
+
required
|
30
|
+
$comment
|
31
|
+
x-cli-envvar
|
32
|
+
x-cli-option
|
33
|
+
x-cli-switch
|
34
|
+
x-cli-special
|
35
|
+
x-cli-convert
|
36
|
+
x-agents
|
37
|
+
x-tspec
|
38
|
+
x-deprecation
|
39
|
+
].freeze
|
40
|
+
|
41
|
+
CLI_AGENT = 'direct'
|
42
|
+
|
43
|
+
private_constant :SCHEMA_KEYS, :CLI_AGENT
|
18
44
|
|
19
45
|
class << self
|
20
|
-
#
|
21
|
-
def
|
22
|
-
|
23
|
-
when 'yes' then return true
|
24
|
-
when 'no' then return false
|
25
|
-
else Aspera.error_unexpected_value(value){'only: yes or no: '}
|
26
|
-
end
|
46
|
+
# @return true if given agent supports that field
|
47
|
+
def supported_by_agent(agent, properties)
|
48
|
+
!properties.key?('x-agents') || properties['x-agents'].include?(agent)
|
27
49
|
end
|
28
50
|
|
29
|
-
# Called by provider of definition before constructor of this class so that
|
30
|
-
def
|
31
|
-
|
32
|
-
|
33
|
-
|
51
|
+
# Called by provider of definition before constructor of this class so that schema has all mandatory fields
|
52
|
+
def read_schema(source_path, suffix=nil)
|
53
|
+
suffix = "_#{suffix}" unless suffix.nil?
|
54
|
+
schema = YAML.load_file("#{source_path[0..-4]}#{suffix}.schema.yaml")
|
55
|
+
schema['properties'].each do |name, properties|
|
56
|
+
Aspera.assert_type(properties, Hash){name}
|
57
|
+
unsupported_keys = properties.keys - SCHEMA_KEYS
|
34
58
|
Aspera.assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
|
35
|
-
Aspera.assert(options.key?(:cli)){"Missing key: cli for #{name}"}
|
36
|
-
Aspera.assert_type(options[:cli], Hash){'Key: cli'}
|
37
|
-
Aspera.assert(options[:cli].key?(:type)){'Missing key: cli.type'}
|
38
|
-
Aspera.assert_values(options[:cli][:type], CLI_OPTION_TYPES){"Unsupported processing type for #{name}"}
|
39
|
-
# by default : optional
|
40
|
-
options[:mandatory] ||= false
|
41
|
-
options[:desc] ||= ''
|
42
|
-
options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
|
43
|
-
cli = options[:cli]
|
44
|
-
unsupported_cli_keys = cli.keys - CLI_KEYS
|
45
|
-
Aspera.assert(unsupported_cli_keys.empty?){"Unsupported cli keys: #{unsupported_cli_keys}"}
|
46
59
|
# by default : string, unless it's without arg
|
47
|
-
|
48
|
-
#
|
49
|
-
|
50
|
-
|
51
|
-
if !cli.key?(:switch) && cli.key?(:type) && CLI_OPTION_TYPE_SWITCH.include?(cli[:type])
|
52
|
-
cli[:switch] = '--' + name.to_s.tr('_', '-')
|
53
|
-
end
|
60
|
+
properties['type'] ||= properties['x-cli-switch'] ? 'boolean' : 'string'
|
61
|
+
# add default cli option name if not present, and if supported in "direct".
|
62
|
+
properties['x-cli-option'] = '--' + name.to_s.tr('_', '-') if !properties.key?('x-cli-option') && !properties['x-cli-envvar'] && (properties.key?('x-cli-switch') || supported_by_agent(CLI_AGENT, properties))
|
63
|
+
properties.freeze
|
54
64
|
end
|
65
|
+
schema['required'] = [] unless schema.key?('required')
|
66
|
+
schema.freeze
|
55
67
|
end
|
56
68
|
end
|
57
69
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
@
|
64
|
-
@params_definition = params_definition
|
70
|
+
# @param [Hash] object with parameters
|
71
|
+
# @param [Hash] schema JSON schema
|
72
|
+
def initialize(object, schema, convert)
|
73
|
+
@object = object # keep reference so that it can be modified by caller before calling `process_params`
|
74
|
+
@schema = schema
|
75
|
+
@convert = convert
|
65
76
|
@result = {
|
66
77
|
env: {},
|
67
78
|
args: []
|
68
79
|
}
|
69
|
-
@
|
80
|
+
@processed_parameters = []
|
81
|
+
end
|
82
|
+
|
83
|
+
# Change required-ness of property in schema
|
84
|
+
def required(name, required)
|
85
|
+
if required
|
86
|
+
@schema['required'].push(name) unless @schema['required'].include?(name)
|
87
|
+
else
|
88
|
+
@schema['required'].delete(name)
|
89
|
+
end
|
70
90
|
end
|
71
91
|
|
72
|
-
#
|
92
|
+
# Add processed parameters to env and args, warns about unused parameters
|
73
93
|
# @param [Hash] env_args with :env and :args
|
74
94
|
def add_env_args(env_args)
|
75
95
|
Log.log.debug{"add_env_args: ENV=#{@result[:env]}, ARGS=#{@result[:args]}"}
|
76
96
|
# warn about non translated arguments
|
77
|
-
@
|
97
|
+
@object.each_pair do |name, value|
|
98
|
+
Log.log.warn{raise "Unknown transfer spec parameter: #{name} = \"#{value}\""} unless @processed_parameters.include?(name)
|
99
|
+
end
|
78
100
|
# set result
|
79
101
|
env_args[:env].merge!(@result[:env])
|
80
102
|
env_args[:args].concat(@result[:args])
|
@@ -84,11 +106,11 @@ module Aspera
|
|
84
106
|
# add options directly to command line
|
85
107
|
def add_command_line_options(options)
|
86
108
|
return if options.nil?
|
87
|
-
options.each{|o
|
109
|
+
options.each{ |o| @result[:args].push(o.to_s)}
|
88
110
|
end
|
89
111
|
|
90
112
|
def process_params
|
91
|
-
@
|
113
|
+
@schema['properties'].each_key do |k|
|
92
114
|
process_param(k)
|
93
115
|
end
|
94
116
|
end
|
@@ -103,81 +125,80 @@ module Aspera
|
|
103
125
|
# @param name [String] of parameter
|
104
126
|
# @param read [TrueClass,FalseClass] read and return value of parameter instead of normal processing (for special)
|
105
127
|
def process_param(name, read: false)
|
106
|
-
|
128
|
+
properties = @schema['properties'][name]
|
107
129
|
# should not happen
|
108
|
-
if
|
130
|
+
if properties.nil?
|
109
131
|
Log.log.warn{"Unknown parameter #{name}"}
|
110
132
|
return
|
111
133
|
end
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
parameter_value = @param_hash[name]
|
134
|
+
# check mandatory parameter (nil is valid value), TODO: change exception ?
|
135
|
+
raise Transfer::Error, "Missing mandatory parameter: #{name}" if @schema['required'].include?(name) && !properties['x-cli-special'] && !@object.key?(name)
|
136
|
+
parameter_value = @object[name]
|
116
137
|
# no default setting
|
117
|
-
# parameter_value=
|
138
|
+
# parameter_value=properties['default'] if parameter_value.nil? and properties.has_key?('default')
|
118
139
|
# Check parameter type
|
119
|
-
expected_classes =
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
140
|
+
expected_classes =
|
141
|
+
[properties['type']].flatten.map do |type|
|
142
|
+
case type
|
143
|
+
when 'string' then [String]
|
144
|
+
when 'array' then [Array]
|
145
|
+
when 'object' then [Hash]
|
146
|
+
when 'integer' then [Integer]
|
147
|
+
when 'boolean' then [TrueClass, FalseClass]
|
148
|
+
else Aspera.error_unexpected_value(properties['type'])
|
149
|
+
end
|
150
|
+
end.flatten
|
129
151
|
# check that value is of expected type
|
130
|
-
raise Transfer::Error, "#{name} is : #{parameter_value.class} (#{parameter_value}), shall be #{
|
152
|
+
raise Transfer::Error, "#{name} is : #{parameter_value.class} (#{parameter_value}), shall be #{properties['type']}, " \
|
131
153
|
unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
|
132
154
|
# special processing will be requested with type get_value
|
133
|
-
@
|
155
|
+
@processed_parameters.push(name) if !properties['x-cli-special'] || read
|
134
156
|
|
135
157
|
# process only non-nil values
|
136
158
|
return nil if parameter_value.nil?
|
137
159
|
|
138
|
-
# check that value is of an accepted type (string,
|
139
|
-
raise "Enum value #{parameter_value} is not allowed for #{name}" if
|
160
|
+
# check that value is of an accepted type (string, integer, boolean)
|
161
|
+
raise "Enum value #{parameter_value} is not allowed for #{name}" if properties.key?('enum') && !properties['enum'].include?(parameter_value)
|
140
162
|
|
141
163
|
# convert some values if value on command line needs processing from value in structure
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
conversion_class, conversion_method = options[:cli][:convert].split('.')
|
151
|
-
converted_value = Kernel.const_get(conversion_class).send(conversion_method, parameter_value)
|
152
|
-
raise Transfer::Error, "unsupported #{name}: #{parameter_value}" if converted_value.nil?
|
164
|
+
if (convert = properties['x-cli-convert'])
|
165
|
+
converted_value =
|
166
|
+
case convert
|
167
|
+
when Hash then convert[parameter_value]
|
168
|
+
when String then @convert.send(convert, parameter_value)
|
169
|
+
else Aspera.error_unexpected_value(convert){"Conversion type for #{name} is Hash or String only."}
|
170
|
+
end
|
171
|
+
raise "No conversion for: #{name}=#{parameter_value}" if converted_value.nil?
|
153
172
|
parameter_value = converted_value
|
154
|
-
when NilClass
|
155
|
-
else Aspera.error_unexpected_value(options[:cli][:convert].class)
|
156
173
|
end
|
157
174
|
|
158
|
-
|
159
|
-
|
175
|
+
return unless self.class.supported_by_agent(CLI_AGENT, properties)
|
176
|
+
|
177
|
+
if read
|
178
|
+
# just get value (deferred)
|
160
179
|
return parameter_value
|
161
|
-
|
180
|
+
elsif properties['x-cli-special']
|
181
|
+
# process later
|
162
182
|
return
|
163
|
-
|
164
|
-
|
165
|
-
@result[:env][
|
166
|
-
|
183
|
+
elsif properties.key?('x-cli-envvar')
|
184
|
+
# set in env var
|
185
|
+
@result[:env][properties['x-cli-envvar']] = parameter_value
|
186
|
+
elsif properties['x-cli-switch']
|
187
|
+
# if present and true : just add option without value
|
167
188
|
add_param = false
|
168
189
|
case parameter_value
|
169
190
|
when false then nil # nothing to put on command line, no creation by default
|
170
191
|
when true then add_param = true
|
171
192
|
else Aspera.error_unexpected_value(parameter_value){name}
|
172
193
|
end
|
173
|
-
add_param = !add_param if
|
174
|
-
add_command_line_options([
|
175
|
-
|
194
|
+
# add_param = !add_param if properties[:add_on_false]
|
195
|
+
add_command_line_options([properties['x-cli-option']]) if add_param
|
196
|
+
else
|
197
|
+
# transform into command line option with value
|
176
198
|
# parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
|
177
199
|
parameter_value = [parameter_value] unless parameter_value.is_a?(Array)
|
178
200
|
# if transfer_spec value is an array, applies option many times
|
179
|
-
parameter_value.each{|v|add_command_line_options([
|
180
|
-
else Aspera.error_unexpected_value(processing_type){processing_type.class.name}
|
201
|
+
parameter_value.each{ |v| add_command_line_options([properties['x-cli-option'], v])}
|
181
202
|
end
|
182
203
|
end
|
183
204
|
end
|
data/lib/aspera/coverage.rb
CHANGED
@@ -5,7 +5,7 @@ if ENV.key?('ENABLE_COVERAGE')
|
|
5
5
|
require 'simplecov'
|
6
6
|
require 'securerandom'
|
7
7
|
# compute development top folder based on this source location
|
8
|
-
development_root = 3.times.inject(File.realpath(__FILE__))
|
8
|
+
development_root = 3.times.inject(File.realpath(__FILE__)){ |p, _| File.dirname(p)}
|
9
9
|
SimpleCov.root(development_root)
|
10
10
|
SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
|
11
11
|
# keep cache data for 1 day (must be longer that time to run the whole test suite)
|
@@ -23,13 +23,14 @@ if ENV.key?('ENABLE_COVERAGE')
|
|
23
23
|
# assert usually do not trigger
|
24
24
|
add_filter do |source_file|
|
25
25
|
source_file.lines.each do |line|
|
26
|
-
line.skipped! if no_cov_functions.any?{|i|line.src.include?(i)}
|
26
|
+
line.skipped! if no_cov_functions.any?{ |i| line.src.include?(i)}
|
27
27
|
end
|
28
28
|
false
|
29
29
|
end
|
30
|
-
# no coverage
|
30
|
+
# no coverage in these files
|
31
31
|
add_filter 'lib/aspera/cli/plugins/faspex.rb'
|
32
32
|
add_filter 'lib/aspera/node_simulator.rb'
|
33
33
|
add_filter 'lib/aspera/keychain/macos_security.rb'
|
34
|
+
add_filter 'lib/aspera/assert.rb'
|
34
35
|
end
|
35
36
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -103,9 +103,9 @@ module Aspera
|
|
103
103
|
def log_spawn(exec:, args: nil, env: nil)
|
104
104
|
[
|
105
105
|
'execute:'.red,
|
106
|
-
env&.map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
|
106
|
+
env&.map{ |k, v| "#{k}=#{Shellwords.shellescape(v)}"},
|
107
107
|
Shellwords.shellescape(exec),
|
108
|
-
args&.map{|a|Shellwords.shellescape(a)}
|
108
|
+
args&.map{ |a| Shellwords.shellescape(a)}
|
109
109
|
].compact.flatten.join(' ')
|
110
110
|
end
|
111
111
|
|
@@ -122,7 +122,7 @@ module Aspera
|
|
122
122
|
Aspera.assert_type(args, Array) unless args.nil?
|
123
123
|
Aspera.assert_type(env, Hash) unless env.nil?
|
124
124
|
Aspera.assert_type(options, Hash) unless options.nil?
|
125
|
-
Log.log.debug
|
125
|
+
Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
|
126
126
|
# start ascp in separate process
|
127
127
|
spawn_args = []
|
128
128
|
spawn_args.push(env) unless env.nil?
|
@@ -144,7 +144,7 @@ module Aspera
|
|
144
144
|
Aspera.assert_type(exec, String)
|
145
145
|
Aspera.assert_type(args, Array) unless args.nil?
|
146
146
|
Aspera.assert_type(env, Hash) unless env.nil?
|
147
|
-
Log.log.debug
|
147
|
+
Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
|
148
148
|
# start in separate process
|
149
149
|
spawn_args = []
|
150
150
|
spawn_args.push(env) unless env.nil?
|
@@ -166,7 +166,7 @@ module Aspera
|
|
166
166
|
Aspera.assert_type(exec, String)
|
167
167
|
Aspera.assert_type(args, Array)
|
168
168
|
Aspera.assert_type(opts, Hash)
|
169
|
-
Log.log.debug
|
169
|
+
Log.log.debug{log_spawn(exec: exec, args: args)}
|
170
170
|
stdout, stderr, status = Open3.capture3(exec, *args, **opts)
|
171
171
|
Log.log.debug{"status=#{status}, stderr=#{stderr}"}
|
172
172
|
Log.log.trace1{"stdout=#{stdout}"}
|
@@ -256,7 +256,7 @@ module Aspera
|
|
256
256
|
# https://www.gnu.org/software/libc/manual/html_node/Locale-Categories.html
|
257
257
|
# https://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html
|
258
258
|
def terminal_supports_unicode?
|
259
|
-
@terminal_supports_unicode = self.class.terminal? && %w(LC_ALL LC_CTYPE LANG).any?{|var|ENV[var]&.include?('UTF-8')} if @terminal_supports_unicode.nil?
|
259
|
+
@terminal_supports_unicode = self.class.terminal? && %w(LC_ALL LC_CTYPE LANG).any?{ |var| ENV[var]&.include?('UTF-8')} if @terminal_supports_unicode.nil?
|
260
260
|
return @terminal_supports_unicode
|
261
261
|
end
|
262
262
|
|
data/lib/aspera/faspex_gw.rb
CHANGED
@@ -8,8 +8,8 @@ require 'json'
|
|
8
8
|
module Aspera
|
9
9
|
# Simulate the Faspex 4 /send API and creates a package on Aspera on Cloud or Faspex 5
|
10
10
|
class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
|
11
|
-
# @param app_api
|
12
|
-
# @param app_context [String]
|
11
|
+
# @param app_api [Rest] API object
|
12
|
+
# @param app_context [String] workspace id (aoc only)
|
13
13
|
def initialize(server, app_api, app_context)
|
14
14
|
Aspera.assert_values(app_api.class.name, ['Aspera::Api::AoC', 'Aspera::Rest'])
|
15
15
|
super(server)
|
@@ -32,7 +32,7 @@ module Aspera
|
|
32
32
|
created_package = @app_api.create_package_simple(package_data, true, nil)
|
33
33
|
# but we place it in a Faspex package creation response
|
34
34
|
return {
|
35
|
-
'links' => {
|
35
|
+
'links' => {'status' => 'unused'},
|
36
36
|
'xfer_sessions' => [created_package[:spec]]
|
37
37
|
}
|
38
38
|
end
|
@@ -42,22 +42,22 @@ module Aspera
|
|
42
42
|
package_data = {
|
43
43
|
'title' => faspex_pkg_delivery['title'],
|
44
44
|
'note' => faspex_pkg_delivery['note'],
|
45
|
-
'recipients' => faspex_pkg_delivery['recipients'].map{|name|{'name'=>name}}
|
45
|
+
'recipients' => faspex_pkg_delivery['recipients'].map{ |name| {'name'=>name}}
|
46
46
|
}
|
47
47
|
package = @app_api.create('packages', package_data)
|
48
48
|
# TODO: option to send from remote source or httpgw
|
49
49
|
transfer_spec = @app_api.call(
|
50
|
-
operation:
|
51
|
-
subpath:
|
52
|
-
|
53
|
-
|
54
|
-
body:
|
55
|
-
|
50
|
+
operation: 'POST',
|
51
|
+
subpath: "packages/#{package['id']}/transfer_spec/upload",
|
52
|
+
query: {transfer_type: Cli::Plugins::Faspex5::TRANSFER_CONNECT},
|
53
|
+
content_type: Rest::MIME_JSON,
|
54
|
+
body: {paths: [{'destination'=>'/'}]},
|
55
|
+
headers: {'Accept' => Rest::MIME_JSON}
|
56
56
|
)[:data]
|
57
57
|
transfer_spec.delete('authentication')
|
58
58
|
# but we place it in a Faspex package creation response
|
59
59
|
return {
|
60
|
-
'links' => {
|
60
|
+
'links' => {'status' => 'unused'},
|
61
61
|
'xfer_sessions' => [transfer_spec]
|
62
62
|
}
|
63
63
|
end
|
@@ -80,18 +80,18 @@ module Aspera
|
|
80
80
|
end
|
81
81
|
Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
|
82
82
|
response.status = 200
|
83
|
-
response.content_type =
|
83
|
+
response.content_type = Rest::MIME_JSON
|
84
84
|
response.body = JSON.generate(faspex_package_create_result)
|
85
85
|
rescue => e
|
86
86
|
response.status = 500
|
87
|
-
response['Content-Type'] =
|
87
|
+
response['Content-Type'] = Rest::MIME_JSON
|
88
88
|
response.body = {error: e.message, stacktrace: e.backtrace}.to_json
|
89
89
|
Log.log.error(e.message)
|
90
90
|
Log.log.debug{e.backtrace.join("\n")}
|
91
91
|
end
|
92
92
|
else
|
93
93
|
response.status = 400
|
94
|
-
response['Content-Type'] =
|
94
|
+
response['Content-Type'] = Rest::MIME_JSON
|
95
95
|
response.body = {error: 'Unsupported endpoint'}.to_json
|
96
96
|
end
|
97
97
|
end
|
@@ -15,7 +15,8 @@ module Aspera
|
|
15
15
|
Aspera.assert_type(parameters, Hash)
|
16
16
|
@parameters = parameters.symbolize_keys
|
17
17
|
Log.log.debug{Log.dump(:post_proc_parameters, @parameters)}
|
18
|
-
|
18
|
+
not_allowed = @parameters.keys - ALLOWED_PARAMETERS
|
19
|
+
raise "unsupported parameters: #{not_allowed.join(', ')}" unless not_allowed.empty?
|
19
20
|
@parameters[:script_folder] ||= '.'
|
20
21
|
@parameters[:fail_on_error] ||= false
|
21
22
|
@parameters[:timeout_seconds] ||= 60
|
@@ -29,13 +30,13 @@ module Aspera
|
|
29
30
|
# only accept requests on the root
|
30
31
|
if !request.path.start_with?(@parameters[:root])
|
31
32
|
response.status = 400
|
32
|
-
response['Content-Type'] =
|
33
|
+
response['Content-Type'] = Rest::MIME_JSON
|
33
34
|
response.body = {status: 'error', message: 'Request outside domain'}.to_json
|
34
35
|
return
|
35
36
|
end
|
36
37
|
if request.body.nil?
|
37
38
|
response.status = 400
|
38
|
-
response['Content-Type'] =
|
39
|
+
response['Content-Type'] = Rest::MIME_JSON
|
39
40
|
response.body = {status: 'error', message: 'Empty request'}.to_json
|
40
41
|
return
|
41
42
|
end
|
@@ -47,7 +48,7 @@ module Aspera
|
|
47
48
|
webhook_parameters = JSON.parse(request.body)
|
48
49
|
Log.log.debug{Log.dump(:webhook_parameters, webhook_parameters)}
|
49
50
|
# env expects only strings
|
50
|
-
environment = webhook_parameters.each_with_object({})
|
51
|
+
environment = webhook_parameters.each_with_object({}){ |(k, v), h| h[k] = v.to_s}
|
51
52
|
post_proc_pid = Environment.secure_spawn(env: environment, exec: script_path)
|
52
53
|
Timeout.timeout(@parameters[:timeout_seconds]) do
|
53
54
|
# "wait" for process to avoid zombie
|
@@ -57,7 +58,7 @@ module Aspera
|
|
57
58
|
process_status = $CHILD_STATUS
|
58
59
|
raise "script #{script_path} failed with code #{process_status.exitstatus}" if !process_status.success? && @parameters[:fail_on_error]
|
59
60
|
response.status = 200
|
60
|
-
response.content_type =
|
61
|
+
response.content_type = Rest::MIME_JSON
|
61
62
|
response.body = JSON.generate({status: 'success', script: script_path, exit_code: process_status.exitstatus})
|
62
63
|
Log.log.debug{'Script executed successfully'}
|
63
64
|
rescue => e
|
@@ -68,7 +69,7 @@ module Aspera
|
|
68
69
|
Log.log.error("Killed process: #{post_proc_pid}")
|
69
70
|
end
|
70
71
|
response.status = 500
|
71
|
-
response['Content-Type'] =
|
72
|
+
response['Content-Type'] = Rest::MIME_JSON
|
72
73
|
response.body = {status: 'error', script: script_path, message: e.message}.to_json
|
73
74
|
end
|
74
75
|
end
|
data/lib/aspera/hash_ext.rb
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
class ::Hash
|
4
4
|
def deep_merge(second)
|
5
|
-
merge(second){|_key, v1, v2|v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge(v2) : v2}
|
5
|
+
merge(second){ |_key, v1, v2| v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge(v2) : v2}
|
6
6
|
end
|
7
7
|
|
8
8
|
def deep_merge!(second)
|
9
|
-
merge!(second){|_key, v1, v2|v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge!(v2) : v2}
|
9
|
+
merge!(second){ |_key, v1, v2| v1.is_a?(Hash) && v2.is_a?(Hash) ? v1.deep_merge!(v2) : v2}
|
10
10
|
end
|
11
11
|
|
12
12
|
def deep_do(memory=nil, &block)
|
data/lib/aspera/json_rpc.rb
CHANGED
@@ -17,9 +17,9 @@ module Aspera
|
|
17
17
|
CONTENT_KEYS = %i[label username password url description].freeze
|
18
18
|
FILE_KEYS = %w[version type cipher data].sort.freeze
|
19
19
|
private_constant :LEGACY_CIPHER_NAME, :DEFAULT_CIPHER_NAME, :FILE_TYPE, :CONTENT_KEYS, :FILE_KEYS
|
20
|
-
def initialize(
|
21
|
-
Aspera.assert_type(
|
22
|
-
@path =
|
20
|
+
def initialize(file:, password:)
|
21
|
+
Aspera.assert_type(file, String){'path to vault file'}
|
22
|
+
@path = file
|
23
23
|
@all_secrets = {}
|
24
24
|
@cipher_name = DEFAULT_CIPHER_NAME
|
25
25
|
vault_encrypted_data = nil
|
@@ -37,31 +37,27 @@ module Aspera
|
|
37
37
|
end
|
38
38
|
end
|
39
39
|
# setting password also creates the cipher
|
40
|
-
|
40
|
+
@cipher = cipher(password)
|
41
41
|
if !vault_encrypted_data.nil?
|
42
42
|
@all_secrets = YAML.load_stream(@cipher.decrypt(vault_encrypted_data)).first
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
# derive key from passphrase, add trailing zeros
|
51
|
-
key = "#{new_password}#{"\x0" * key_bytes}"[0..(key_bytes - 1)]
|
52
|
-
Log.log.trace1{"secret=[#{key}],#{key.length}"}
|
53
|
-
@cipher = SymmetricEncryption.cipher = SymmetricEncryption::Cipher.new(cipher_name: DEFAULT_CIPHER_NAME, key: key, encoding: :none)
|
46
|
+
def info
|
47
|
+
return {
|
48
|
+
file: @path
|
49
|
+
}
|
54
50
|
end
|
55
51
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
'
|
62
|
-
|
63
|
-
|
64
|
-
|
52
|
+
def list
|
53
|
+
result = []
|
54
|
+
@all_secrets.each do |label, values|
|
55
|
+
normal = values.symbolize_keys
|
56
|
+
normal[:label] = label
|
57
|
+
CONTENT_KEYS.each{ |k| normal[k] = '' unless normal.key?(k)}
|
58
|
+
result.push(normal)
|
59
|
+
end
|
60
|
+
return result
|
65
61
|
end
|
66
62
|
|
67
63
|
# set a secret
|
@@ -79,14 +75,10 @@ module Aspera
|
|
79
75
|
save
|
80
76
|
end
|
81
77
|
|
82
|
-
def
|
83
|
-
|
84
|
-
@all_secrets.
|
85
|
-
|
86
|
-
normal[:label] = label
|
87
|
-
CONTENT_KEYS.each{|k|normal[k] = '' unless normal.key?(k)}
|
88
|
-
result.push(normal)
|
89
|
-
end
|
78
|
+
def get(label:, exception: true)
|
79
|
+
Aspera.assert(@all_secrets.key?(label)){"Label not found: #{label}"} if exception
|
80
|
+
result = @all_secrets[label].clone
|
81
|
+
result[:label] = label if result.is_a?(Hash)
|
90
82
|
return result
|
91
83
|
end
|
92
84
|
|
@@ -95,11 +87,32 @@ module Aspera
|
|
95
87
|
save
|
96
88
|
end
|
97
89
|
|
98
|
-
def
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
90
|
+
def change_password(password)
|
91
|
+
@cipher = cipher(password)
|
92
|
+
save
|
93
|
+
end
|
94
|
+
|
95
|
+
private
|
96
|
+
|
97
|
+
# set the password and cipher
|
98
|
+
def cipher(new_password)
|
99
|
+
# number of bits in second position
|
100
|
+
key_bytes = @cipher_name.split('-')[1].to_i / Environment::BITS_PER_BYTE
|
101
|
+
# derive key from passphrase, add trailing zeros
|
102
|
+
key = "#{new_password}#{"\x0" * key_bytes}"[0..(key_bytes - 1)]
|
103
|
+
Log.log.trace1{"secret=[#{key}],#{key.length}"}
|
104
|
+
SymmetricEncryption.cipher = SymmetricEncryption::Cipher.new(cipher_name: @cipher_name, key: key, encoding: :none)
|
105
|
+
end
|
106
|
+
|
107
|
+
# save current data to file with format
|
108
|
+
def save
|
109
|
+
vault_info = {
|
110
|
+
'version' => '1.0.0',
|
111
|
+
'type' => FILE_TYPE,
|
112
|
+
'cipher' => @cipher_name,
|
113
|
+
'data' => @cipher.encrypt(YAML.dump(@all_secrets))
|
114
|
+
}
|
115
|
+
File.write(@path, YAML.dump(vault_info))
|
103
116
|
end
|
104
117
|
end
|
105
118
|
end
|