aspera-cli 4.15.0 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +29 -3
- data/CHANGELOG.md +375 -280
- data/CONTRIBUTING.md +71 -18
- data/README.md +1978 -1656
- data/bin/ascli +13 -31
- data/bin/asession +32 -22
- data/examples/dascli +2 -2
- data/lib/aspera/agent/alpha.rb +117 -0
- data/lib/aspera/agent/base.rb +61 -0
- data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
- data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +116 -116
- data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +21 -19
- data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +21 -33
- data/lib/aspera/agent/trsdk.rb +188 -0
- data/lib/aspera/api/aoc.rb +586 -0
- data/lib/aspera/api/ats.rb +46 -0
- data/lib/aspera/api/cos_node.rb +95 -0
- data/lib/aspera/api/node.rb +344 -0
- data/lib/aspera/ascmd.rb +47 -14
- data/lib/aspera/{fasp → ascp}/installation.rb +54 -15
- data/lib/aspera/{fasp → ascp}/management.rb +14 -14
- data/lib/aspera/{fasp → ascp}/products.rb +1 -1
- data/lib/aspera/assert.rb +45 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formatter.rb +27 -14
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/main.rb +49 -29
- data/lib/aspera/cli/manager.rb +46 -36
- data/lib/aspera/cli/plugin.rb +34 -20
- data/lib/aspera/cli/plugin_factory.rb +61 -0
- data/lib/aspera/cli/plugins/alee.rb +7 -7
- data/lib/aspera/cli/plugins/aoc.rb +168 -132
- data/lib/aspera/cli/plugins/ats.rb +33 -33
- data/lib/aspera/cli/plugins/bss.rb +3 -4
- data/lib/aspera/cli/plugins/config.rb +250 -272
- data/lib/aspera/cli/plugins/console.rb +8 -6
- data/lib/aspera/cli/plugins/cos.rb +20 -19
- data/lib/aspera/cli/plugins/faspex.rb +71 -60
- data/lib/aspera/cli/plugins/faspex5.rb +212 -133
- data/lib/aspera/cli/plugins/node.rb +83 -75
- data/lib/aspera/cli/plugins/orchestrator.rb +36 -44
- data/lib/aspera/cli/plugins/preview.rb +33 -31
- data/lib/aspera/cli/plugins/server.rb +33 -32
- data/lib/aspera/cli/plugins/shares.rb +39 -33
- data/lib/aspera/cli/sync_actions.rb +9 -9
- data/lib/aspera/cli/transfer_agent.rb +45 -25
- data/lib/aspera/cli/transfer_progress.rb +2 -3
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -0
- data/lib/aspera/command_line_builder.rb +16 -14
- data/lib/aspera/coverage.rb +21 -0
- data/lib/aspera/data_repository.rb +33 -2
- data/lib/aspera/environment.rb +5 -4
- data/lib/aspera/faspex_gw.rb +13 -11
- data/lib/aspera/faspex_postproc.rb +6 -5
- data/lib/aspera/id_generator.rb +4 -2
- data/lib/aspera/json_rpc.rb +10 -8
- data/lib/aspera/keychain/encrypted_hash.rb +46 -11
- data/lib/aspera/keychain/macos_security.rb +29 -22
- data/lib/aspera/log.rb +5 -4
- data/lib/aspera/nagios.rb +7 -2
- data/lib/aspera/node_simulator.rb +213 -0
- data/lib/aspera/oauth/base.rb +143 -0
- data/lib/aspera/oauth/factory.rb +124 -0
- data/lib/aspera/oauth/generic.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +51 -0
- data/lib/aspera/oauth/url_json.rb +31 -0
- data/lib/aspera/oauth/web.rb +50 -0
- data/lib/aspera/oauth.rb +5 -328
- data/lib/aspera/open_application.rb +7 -7
- data/lib/aspera/persistency_action_once.rb +13 -14
- data/lib/aspera/persistency_folder.rb +3 -2
- data/lib/aspera/preview/file_types.rb +53 -267
- data/lib/aspera/preview/generator.rb +7 -5
- data/lib/aspera/preview/terminal.rb +17 -7
- data/lib/aspera/preview/utils.rb +8 -7
- data/lib/aspera/proxy_auto_config.rb +6 -3
- data/lib/aspera/rest.rb +187 -140
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/rest_errors_aspera.rb +5 -3
- data/lib/aspera/resumer.rb +77 -0
- data/lib/aspera/secret_hider.rb +5 -2
- data/lib/aspera/ssh.rb +15 -8
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/{fasp → transfer}/error.rb +3 -3
- data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
- data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
- data/lib/aspera/{fasp → transfer}/parameters.rb +95 -120
- data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +23 -19
- data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
- data/lib/aspera/transfer/sync.rb +273 -0
- data/lib/aspera/{fasp → transfer}/uri.rb +10 -9
- data/lib/aspera/web_server_simple.rb +12 -3
- data.tar.gz.sig +0 -0
- metadata +92 -68
- metadata.gz.sig +0 -0
- data/lib/aspera/aoc.rb +0 -606
- data/lib/aspera/ats_api.rb +0 -47
- data/lib/aspera/cos_node.rb +0 -93
- data/lib/aspera/fasp/agent_aspera.rb +0 -126
- data/lib/aspera/fasp/agent_base.rb +0 -48
- data/lib/aspera/fasp/agent_trsdk.rb +0 -146
- data/lib/aspera/fasp/resume_policy.rb +0 -77
- data/lib/aspera/node.rb +0 -338
- data/lib/aspera/sync.rb +0 -219
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'aspera/
|
|
3
|
+
require 'aspera/agent/base'
|
|
4
|
+
require 'aspera/transfer/spec'
|
|
4
5
|
require 'aspera/cli/info'
|
|
6
|
+
require 'aspera/log'
|
|
7
|
+
require 'aspera/assert'
|
|
5
8
|
|
|
6
9
|
module Aspera
|
|
7
10
|
module Cli
|
|
@@ -28,7 +31,7 @@ module Aspera
|
|
|
28
31
|
:FILE_LIST_FROM_TRANSFER_SPEC,
|
|
29
32
|
:FILE_LIST_OPTIONS,
|
|
30
33
|
:DEFAULT_TRANSFER_NOTIFY_TEMPLATE
|
|
31
|
-
TRANSFER_AGENTS =
|
|
34
|
+
TRANSFER_AGENTS = Agent::Base.agent_list.freeze
|
|
32
35
|
|
|
33
36
|
class << self
|
|
34
37
|
# @return :success if all sessions statuses returned by "start" are success
|
|
@@ -52,6 +55,8 @@ module Aspera
|
|
|
52
55
|
@agent = nil
|
|
53
56
|
# source/destination pair, like "paths" of transfer spec
|
|
54
57
|
@transfer_paths = nil
|
|
58
|
+
# HTTPGW URL provided by webapp
|
|
59
|
+
@httpgw_url_lambda = nil
|
|
55
60
|
@opt_mgr.declare(:ts, 'Override transfer spec values', types: Hash, handler: {o: self, m: :option_transfer_spec})
|
|
56
61
|
@opt_mgr.declare(:to_folder, 'Destination folder for transferred files')
|
|
57
62
|
@opt_mgr.declare(:sources, "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})")
|
|
@@ -65,7 +70,7 @@ module Aspera
|
|
|
65
70
|
|
|
66
71
|
# multiple option are merged
|
|
67
72
|
def option_transfer_spec=(value)
|
|
68
|
-
|
|
73
|
+
Aspera.assert_type(value, Hash){'ts'}
|
|
69
74
|
@transfer_spec_command_line.deep_merge!(value)
|
|
70
75
|
end
|
|
71
76
|
|
|
@@ -92,11 +97,12 @@ module Aspera
|
|
|
92
97
|
end
|
|
93
98
|
|
|
94
99
|
# analyze options and create new agent if not already created or set
|
|
95
|
-
|
|
96
|
-
|
|
100
|
+
# TODO: make a Factory pattern
|
|
101
|
+
def agent_instance
|
|
102
|
+
return @agent unless @agent.nil?
|
|
97
103
|
agent_type = @opt_mgr.get_option(:transfer, mandatory: true)
|
|
98
104
|
# agent plugin is loaded on demand to avoid loading unnecessary dependencies
|
|
99
|
-
require "aspera/
|
|
105
|
+
require "aspera/agent/#{agent_type}"
|
|
100
106
|
# set keys as symbols
|
|
101
107
|
agent_options = @opt_mgr.get_option(:transfer_info).symbolize_keys
|
|
102
108
|
# special cases
|
|
@@ -110,14 +116,20 @@ module Aspera
|
|
|
110
116
|
when :direct
|
|
111
117
|
# by default do not display ascp native progress bar
|
|
112
118
|
agent_options[:quiet] = true unless agent_options.key?(:quiet)
|
|
113
|
-
agent_options[:
|
|
114
|
-
agent_options[:trusted_certs] = @config.trusted_cert_locations
|
|
119
|
+
agent_options[:check_ignore_cb] = ->(host, port){@config.ignore_cert?(host, port)}
|
|
120
|
+
agent_options[:trusted_certs] = @config.trusted_cert_locations unless agent_options.key?(:trusted_certs)
|
|
121
|
+
when :httpgw
|
|
122
|
+
unless agent_options.key?(:url) || @httpgw_url_lambda.nil?
|
|
123
|
+
Log.log.debug('retrieving HTTPGW URL from webapp')
|
|
124
|
+
agent_options[:url] = @httpgw_url_lambda.call
|
|
125
|
+
end
|
|
115
126
|
end
|
|
116
127
|
agent_options[:progress] = @config.progress_bar
|
|
117
128
|
# get agent instance
|
|
118
|
-
new_agent = Kernel.const_get("Aspera::
|
|
129
|
+
new_agent = Kernel.const_get("Aspera::Agent::#{agent_type.capitalize}").new(agent_options)
|
|
119
130
|
self.agent_instance = new_agent
|
|
120
|
-
|
|
131
|
+
Log.log.debug{"transfer agent is a #{@agent.class}"}
|
|
132
|
+
return @agent
|
|
121
133
|
end
|
|
122
134
|
|
|
123
135
|
# return destination folder for transfers
|
|
@@ -131,9 +143,9 @@ module Aspera
|
|
|
131
143
|
return dest_folder unless dest_folder.nil?
|
|
132
144
|
# default: / on remote, . on local
|
|
133
145
|
case direction.to_s
|
|
134
|
-
when
|
|
135
|
-
when
|
|
136
|
-
else
|
|
146
|
+
when Transfer::Spec::DIRECTION_SEND then dest_folder = '/'
|
|
147
|
+
when Transfer::Spec::DIRECTION_RECEIVE then dest_folder = '.'
|
|
148
|
+
else Aspera.error_unexpected_value(direction)
|
|
137
149
|
end
|
|
138
150
|
return dest_folder
|
|
139
151
|
end
|
|
@@ -145,6 +157,11 @@ module Aspera
|
|
|
145
157
|
end
|
|
146
158
|
end
|
|
147
159
|
|
|
160
|
+
def httpgw_url_cb=(httpgw_url_proc)
|
|
161
|
+
Aspera.assert_type(httpgw_url_proc, Proc){'httpgw_url_cb'}
|
|
162
|
+
@httpgw_url_lambda = httpgw_url_proc
|
|
163
|
+
end
|
|
164
|
+
|
|
148
165
|
# This is how the list of files to be transferred is specified
|
|
149
166
|
# get paths suitable for transfer spec from command line
|
|
150
167
|
# @return [Hash] {source: (mandatory), destination: (optional)}
|
|
@@ -167,7 +184,7 @@ module Aspera
|
|
|
167
184
|
Log.log.debug('assume list provided in transfer spec')
|
|
168
185
|
special_case_direct_with_list =
|
|
169
186
|
@opt_mgr.get_option(:transfer, mandatory: true).eql?(:direct) &&
|
|
170
|
-
|
|
187
|
+
Transfer::Parameters.ascp_args_file_list?(@opt_mgr.get_option(:transfer_info)['ascp_args'])
|
|
171
188
|
raise Cli::BadArgument, 'transfer spec on command line must have sources' if @transfer_paths.nil? && !special_case_direct_with_list
|
|
172
189
|
# here we assume check of sources is made in transfer agent
|
|
173
190
|
return @transfer_paths
|
|
@@ -181,14 +198,15 @@ module Aspera
|
|
|
181
198
|
if !@transfer_paths.nil?
|
|
182
199
|
Log.log.warn('--sources overrides paths from --ts')
|
|
183
200
|
end
|
|
184
|
-
|
|
201
|
+
source_type = @opt_mgr.get_option(:src_type, mandatory: true)
|
|
202
|
+
case source_type
|
|
185
203
|
when :list
|
|
186
204
|
# when providing a list, just specify source
|
|
187
205
|
@transfer_paths = file_list.map{|i|{'source' => i}}
|
|
188
206
|
when :pair
|
|
189
|
-
|
|
207
|
+
Aspera.assert(file_list.length.even?, exception_class: Cli::BadArgument){"When using pair, provide an even number of paths: #{file_list.length}"}
|
|
190
208
|
@transfer_paths = file_list.each_slice(2).to_a.map{|s, d|{'source' => s, 'destination' => d}}
|
|
191
|
-
else
|
|
209
|
+
else Aspera.error_unexpected_value(source_type)
|
|
192
210
|
end
|
|
193
211
|
Log.log.debug{"paths=#{@transfer_paths}"}
|
|
194
212
|
return @transfer_paths
|
|
@@ -199,14 +217,14 @@ module Aspera
|
|
|
199
217
|
# @param rest_token [Rest] if oauth token regeneration supported
|
|
200
218
|
def start(transfer_spec, rest_token: nil)
|
|
201
219
|
# check parameters
|
|
202
|
-
|
|
220
|
+
Aspera.assert_type(transfer_spec, Hash){'transfer_spec'}
|
|
203
221
|
# process :src option
|
|
204
222
|
case transfer_spec['direction']
|
|
205
|
-
when
|
|
223
|
+
when Transfer::Spec::DIRECTION_RECEIVE
|
|
206
224
|
# init default if required in any case
|
|
207
225
|
@transfer_spec_command_line['destination_root'] ||= destination_folder(transfer_spec['direction'])
|
|
208
|
-
when
|
|
209
|
-
if transfer_spec.dig('tags',
|
|
226
|
+
when Transfer::Spec::DIRECTION_SEND
|
|
227
|
+
if transfer_spec.dig('tags', Transfer::Spec::TAG_RESERVED, 'node', 'access_key')
|
|
210
228
|
# gen4
|
|
211
229
|
@transfer_spec_command_line.delete('destination_root') if @transfer_spec_command_line.key?('destination_root_id')
|
|
212
230
|
elsif transfer_spec.key?('token')
|
|
@@ -223,12 +241,14 @@ module Aspera
|
|
|
223
241
|
@transfer_spec_command_line['paths'] = transfer_spec['paths'] || ts_source_paths
|
|
224
242
|
# updated transfer spec with command line
|
|
225
243
|
updated_ts(transfer_spec)
|
|
244
|
+
# if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
|
|
245
|
+
if transfer_spec['content_protection'].eql?('decrypt') && !transfer_spec.key?('content_protection_password')
|
|
246
|
+
transfer_spec['content_protection_password'] = @opt_mgr.prompt_user_input('content protection password', true)
|
|
247
|
+
end
|
|
226
248
|
# create transfer agent
|
|
227
|
-
|
|
228
|
-
Log.log.debug{"transfer agent is a #{@agent.class}"}
|
|
229
|
-
@agent.start_transfer(transfer_spec, token_regenerator: rest_token)
|
|
249
|
+
agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
|
|
230
250
|
# list of: :success or "error message string"
|
|
231
|
-
result =
|
|
251
|
+
result = agent_instance.wait_for_completion
|
|
232
252
|
send_email_transfer_notification(transfer_spec, result)
|
|
233
253
|
return result
|
|
234
254
|
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'aspera/log'
|
|
4
|
+
require 'aspera/assert'
|
|
4
5
|
require 'ruby-progressbar'
|
|
5
6
|
|
|
6
7
|
module Aspera
|
|
@@ -25,9 +26,7 @@ module Aspera
|
|
|
25
26
|
|
|
26
27
|
def event(session_id:, type:, info: nil)
|
|
27
28
|
Log.log.debug{"progress: #{type} #{session_id} #{info}"}
|
|
28
|
-
|
|
29
|
-
raise 'Internal error: session_id is nil'
|
|
30
|
-
end
|
|
29
|
+
Aspera.assert(!session_id.nil? || type.eql?(:pre_start)){'session_id is nil'}
|
|
31
30
|
return if @completed
|
|
32
31
|
if @progress_bar.nil?
|
|
33
32
|
@progress_bar = ProgressBar.create(
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/colors.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'aspera/log'
|
|
4
|
+
require 'aspera/assert'
|
|
3
5
|
module Aspera
|
|
4
6
|
# helper class to build command line from a parameter list (key-value hash)
|
|
5
7
|
# constructor takes hash: { 'param1':'value1', ...}
|
|
@@ -27,20 +29,20 @@ module Aspera
|
|
|
27
29
|
# Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
|
|
28
30
|
def normalize_description(full_description)
|
|
29
31
|
full_description.each do |name, options|
|
|
30
|
-
|
|
32
|
+
Aspera.assert_type(options, Hash){name}
|
|
31
33
|
unsupported_keys = options.keys - OPTIONS_KEYS
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
Aspera.assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
|
|
35
|
+
Aspera.assert(options.key?(:cli)){"Missing key: cli for #{name}"}
|
|
36
|
+
Aspera.assert_type(options[:cli], Hash){'Key: cli'}
|
|
37
|
+
Aspera.assert(options[:cli].key?(:type)){'Missing key: cli.type'}
|
|
38
|
+
Aspera.assert_values(options[:cli][:type], CLI_OPTION_TYPES){"Unsupported processing type for #{name}"}
|
|
37
39
|
# by default : optional
|
|
38
40
|
options[:mandatory] ||= false
|
|
39
41
|
options[:desc] ||= ''
|
|
40
42
|
options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
|
|
41
43
|
cli = options[:cli]
|
|
42
44
|
unsupported_cli_keys = cli.keys - CLI_KEYS
|
|
43
|
-
|
|
45
|
+
Aspera.assert(unsupported_cli_keys.empty?){"Unsupported cli keys: #{unsupported_cli_keys}"}
|
|
44
46
|
# by default : string, unless it's without arg
|
|
45
47
|
options[:accepted_types] ||= options[:cli][:type].eql?(:opt_without_arg) ? :bool : :string
|
|
46
48
|
# single type is placed in array
|
|
@@ -109,7 +111,7 @@ module Aspera
|
|
|
109
111
|
end
|
|
110
112
|
processing_type = read ? :get_value : options[:cli][:type]
|
|
111
113
|
# check mandatory parameter (nil is valid value)
|
|
112
|
-
raise
|
|
114
|
+
raise Transfer::Error, "Missing mandatory parameter: #{name}" if options[:mandatory] && !@param_hash.key?(name)
|
|
113
115
|
parameter_value = @param_hash[name]
|
|
114
116
|
# no default setting
|
|
115
117
|
# parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
|
|
@@ -121,11 +123,11 @@ module Aspera
|
|
|
121
123
|
when :hash then Hash
|
|
122
124
|
when :int then Integer
|
|
123
125
|
when :bool then [TrueClass, FalseClass]
|
|
124
|
-
else
|
|
126
|
+
else Aspera.error_unexpected_value(type_symbol)
|
|
125
127
|
end
|
|
126
128
|
end.flatten
|
|
127
129
|
# check that value is of expected type
|
|
128
|
-
raise
|
|
130
|
+
raise Transfer::Error, "#{name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, " \
|
|
129
131
|
unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
|
|
130
132
|
# special processing will be requested with type get_value
|
|
131
133
|
@used_param_names.push(name) unless processing_type.eql?(:special)
|
|
@@ -147,10 +149,10 @@ module Aspera
|
|
|
147
149
|
# :convert has name of class and encoding method
|
|
148
150
|
conversion_class, conversion_method = options[:cli][:convert].split('.')
|
|
149
151
|
converted_value = Kernel.const_get(conversion_class).send(conversion_method, parameter_value)
|
|
150
|
-
raise
|
|
152
|
+
raise Transfer::Error, "unsupported #{name}: #{parameter_value}" if converted_value.nil?
|
|
151
153
|
parameter_value = converted_value
|
|
152
154
|
when NilClass
|
|
153
|
-
else
|
|
155
|
+
else Aspera.error_unexpected_value(options[:cli][:convert].class)
|
|
154
156
|
end
|
|
155
157
|
|
|
156
158
|
case processing_type
|
|
@@ -159,14 +161,14 @@ module Aspera
|
|
|
159
161
|
when :ignore, :special # ignore this parameter or process later
|
|
160
162
|
return
|
|
161
163
|
when :envvar # set in env var
|
|
162
|
-
|
|
164
|
+
Aspera.assert(options[:cli].key?(:variable)){'missing key: cli.variable'}
|
|
163
165
|
@result[:env][options[:cli][:variable]] = parameter_value
|
|
164
166
|
when :opt_without_arg # if present and true : just add option without value
|
|
165
167
|
add_param = false
|
|
166
168
|
case parameter_value
|
|
167
169
|
when false then nil # nothing to put on command line, no creation by default
|
|
168
170
|
when true then add_param = true
|
|
169
|
-
else
|
|
171
|
+
else Aspera.error_unexpected_value(parameter_value){name}
|
|
170
172
|
end
|
|
171
173
|
add_param = !add_param if options[:add_on_false]
|
|
172
174
|
add_command_line_options([options[:cli][:switch]]) if add_param
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# coverage for tests
|
|
4
|
+
if ENV.key?('ENABLE_COVERAGE')
|
|
5
|
+
require 'simplecov'
|
|
6
|
+
require 'securerandom'
|
|
7
|
+
# compute development top folder based on this source location
|
|
8
|
+
development_root = 3.times.inject(File.realpath(__FILE__)) { |p, _| File.dirname(p) }
|
|
9
|
+
SimpleCov.root(development_root)
|
|
10
|
+
SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
|
|
11
|
+
# keep cache data for 1 day (must be longer that time to run the whole test suite)
|
|
12
|
+
SimpleCov.merge_timeout(86400)
|
|
13
|
+
SimpleCov.command_name(SecureRandom.uuid)
|
|
14
|
+
SimpleCov.at_exit do
|
|
15
|
+
original_file_descriptor = $stdout
|
|
16
|
+
$stdout.reopen(File.join(development_root, 'simplecov.log'))
|
|
17
|
+
SimpleCov.result.format!
|
|
18
|
+
$stdout.reopen(original_file_descriptor)
|
|
19
|
+
end
|
|
20
|
+
SimpleCov.start
|
|
21
|
+
end
|
|
@@ -1,15 +1,46 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'aspera/
|
|
3
|
+
require 'aspera/assert'
|
|
4
4
|
require 'singleton'
|
|
5
|
+
require 'openssl'
|
|
5
6
|
|
|
6
7
|
module Aspera
|
|
7
8
|
# a simple binary data repository
|
|
8
9
|
class DataRepository
|
|
9
10
|
include Singleton
|
|
11
|
+
# in same order as elements in folder
|
|
12
|
+
ELEMENTS = %i[dsa rsa uuid aspera.global-cli-client aspera.drive license]
|
|
13
|
+
START_INDEX = 1
|
|
14
|
+
DATA_FOLDER_NAME = 'data'
|
|
15
|
+
|
|
16
|
+
# decode data as expected as string
|
|
17
|
+
# @param name [Symbol] name of the data item
|
|
18
|
+
# @return [String] decoded data
|
|
19
|
+
def item(name)
|
|
20
|
+
index = ELEMENTS.index(name)
|
|
21
|
+
raise ArgumentError, "unknown data item #{name} (#{name.class})" unless index
|
|
22
|
+
raw_data = data(START_INDEX + index)
|
|
23
|
+
case name
|
|
24
|
+
when :dsa, :rsa
|
|
25
|
+
# generate PEM from DER
|
|
26
|
+
return OpenSSL::PKey.const_get(name.to_s.upcase).new(raw_data).to_pem
|
|
27
|
+
when :license
|
|
28
|
+
return Zlib::Inflate.inflate(raw_data)
|
|
29
|
+
when :uuid
|
|
30
|
+
return format('%08x-%04x-%04x-%04x-%04x%08x', *raw_data.unpack('NnnnnN'))
|
|
31
|
+
when :'aspera.global-cli-client', :'aspera.drive'
|
|
32
|
+
return Base64.urlsafe_encode64(raw_data)
|
|
33
|
+
else Aspera.error_unexpected_value(name)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
private_constant :START_INDEX, :DATA_FOLDER_NAME
|
|
40
|
+
|
|
10
41
|
# get binary value from data repository
|
|
11
42
|
def data(id)
|
|
12
|
-
File.read(File.join(__dir__,
|
|
43
|
+
File.read(File.join(__dir__, DATA_FOLDER_NAME, id.to_s), mode: 'rb')
|
|
13
44
|
end
|
|
14
45
|
end
|
|
15
46
|
end
|
data/lib/aspera/environment.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# cspell:ignore USERPROFILE HOMEDRIVE HOMEPATH LC_CTYPE msys aarch
|
|
4
4
|
require 'aspera/log'
|
|
5
|
+
require 'aspera/assert'
|
|
5
6
|
require 'rbconfig'
|
|
6
7
|
|
|
7
8
|
# cspell:words MEBI mswin bccwin
|
|
@@ -15,10 +16,11 @@ module Aspera
|
|
|
15
16
|
OS_AIX = :aix
|
|
16
17
|
OS_LIST = [OS_WINDOWS, OS_X, OS_LINUX, OS_AIX].freeze
|
|
17
18
|
CPU_X86_64 = :x86_64
|
|
19
|
+
CPU_ARM64 = :arm64
|
|
18
20
|
CPU_PPC64 = :ppc64
|
|
19
21
|
CPU_PPC64LE = :ppc64le
|
|
20
22
|
CPU_S390 = :s390
|
|
21
|
-
CPU_LIST = [CPU_X86_64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
|
|
23
|
+
CPU_LIST = [CPU_X86_64, CPU_ARM64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
|
|
22
24
|
|
|
23
25
|
BITS_PER_BYTE = 8
|
|
24
26
|
MEBI = 1024 * 1024
|
|
@@ -54,8 +56,7 @@ module Aspera
|
|
|
54
56
|
when /s390/
|
|
55
57
|
return CPU_S390
|
|
56
58
|
when /arm/, /aarch64/
|
|
57
|
-
|
|
58
|
-
return CPU_X86_64 if os.eql?(OS_X)
|
|
59
|
+
return CPU_ARM64
|
|
59
60
|
end
|
|
60
61
|
raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
|
|
61
62
|
end
|
|
@@ -88,7 +89,7 @@ module Aspera
|
|
|
88
89
|
|
|
89
90
|
# value is provided in block
|
|
90
91
|
def write_file_restricted(path, force: false, mode: nil)
|
|
91
|
-
|
|
92
|
+
Aspera.assert(block_given?, exception_class: Aspera::InternalError)
|
|
92
93
|
if force || !File.exist?(path)
|
|
93
94
|
# Windows may give error
|
|
94
95
|
File.unlink(path) rescue nil
|
data/lib/aspera/faspex_gw.rb
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'aspera/web_server_simple'
|
|
4
3
|
require 'aspera/log'
|
|
4
|
+
require 'aspera/assert'
|
|
5
|
+
require 'webrick'
|
|
5
6
|
require 'json'
|
|
6
7
|
|
|
7
8
|
module Aspera
|
|
8
|
-
#
|
|
9
|
+
# Simulate the Faspex 4 /send API and creates a package on Aspera on Cloud or Faspex 5
|
|
9
10
|
class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
|
|
10
|
-
# @param app_api
|
|
11
|
+
# @param app_api
|
|
11
12
|
# @param app_context [String]
|
|
12
13
|
def initialize(server, app_api, app_context)
|
|
14
|
+
Aspera.assert_values(app_api.class.name, ['Aspera::Api::AoC', 'Aspera::Rest'])
|
|
13
15
|
super(server)
|
|
14
|
-
# typed: Aspera::AoC
|
|
15
16
|
@app_api = app_api
|
|
16
17
|
@app_context = app_context
|
|
17
18
|
end
|
|
@@ -49,7 +50,7 @@ module Aspera
|
|
|
49
50
|
operation: 'POST',
|
|
50
51
|
subpath: "packages/#{package['id']}/transfer_spec/upload",
|
|
51
52
|
headers: {'Accept' => 'application/json'},
|
|
52
|
-
url_params: {transfer_type:
|
|
53
|
+
url_params: {transfer_type: Cli::Plugins::Faspex5::TRANSFER_CONNECT},
|
|
53
54
|
json_params: {paths: [{'destination'=>'/'}]}
|
|
54
55
|
)[:data]
|
|
55
56
|
transfer_spec.delete('authentication')
|
|
@@ -67,13 +68,14 @@ module Aspera
|
|
|
67
68
|
raise 'no payload' if request.body.nil?
|
|
68
69
|
faspex_pkg_parameters = JSON.parse(request.body)
|
|
69
70
|
Log.log.debug{"faspex pkg create parameters=#{faspex_pkg_parameters}"}
|
|
71
|
+
# compare string, as class is not yet known here
|
|
70
72
|
faspex_package_create_result =
|
|
71
|
-
|
|
73
|
+
case @app_api.class.name
|
|
74
|
+
when 'Aspera::Api::AoC'
|
|
72
75
|
faspex4_send_to_aoc(faspex_pkg_parameters)
|
|
73
|
-
|
|
76
|
+
when 'Aspera::Rest'
|
|
74
77
|
faspex4_send_to_faspex5(faspex_pkg_parameters)
|
|
75
|
-
else
|
|
76
|
-
raise "No such adapter: #{@app_api.class}"
|
|
78
|
+
else Aspera.error_unexpected_value(@app_api.class.name)
|
|
77
79
|
end
|
|
78
80
|
Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
|
|
79
81
|
response.status = 200
|
|
@@ -89,8 +91,8 @@ module Aspera
|
|
|
89
91
|
else
|
|
90
92
|
response.status = 400
|
|
91
93
|
response['Content-Type'] = 'application/json'
|
|
92
|
-
response.body = {error: '
|
|
94
|
+
response.body = {error: 'Unsupported endpoint'}.to_json
|
|
93
95
|
end
|
|
94
96
|
end
|
|
95
97
|
end # Faspex4GWServlet
|
|
96
|
-
end #
|
|
98
|
+
end # Aspera
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'English'
|
|
4
|
-
require 'aspera/web_server_simple'
|
|
5
|
-
require 'aspera/log'
|
|
6
3
|
require 'json'
|
|
7
4
|
require 'timeout'
|
|
5
|
+
require 'English'
|
|
6
|
+
require 'webrick'
|
|
7
|
+
require 'aspera/log'
|
|
8
|
+
require 'aspera/assert'
|
|
8
9
|
|
|
9
10
|
module Aspera
|
|
10
11
|
# this class answers the Faspex /send API and creates a package on Aspera on Cloud
|
|
11
12
|
class Faspex4PostProcServlet < WEBrick::HTTPServlet::AbstractServlet
|
|
12
13
|
ALLOWED_PARAMETERS = %i[root script_folder fail_on_error timeout_seconds].freeze
|
|
13
14
|
def initialize(server, parameters)
|
|
14
|
-
|
|
15
|
+
Aspera.assert_type(parameters, Hash)
|
|
15
16
|
@parameters = parameters.symbolize_keys
|
|
16
17
|
Log.log.debug{Log.dump(:post_proc_parameters, @parameters)}
|
|
17
18
|
raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
|
|
@@ -74,4 +75,4 @@ module Aspera
|
|
|
74
75
|
end
|
|
75
76
|
end
|
|
76
77
|
end # Faspex4PostProcServlet
|
|
77
|
-
end #
|
|
78
|
+
end # Aspera
|
data/lib/aspera/id_generator.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'aspera/assert'
|
|
3
4
|
require 'uri'
|
|
4
5
|
|
|
5
6
|
module Aspera
|
|
@@ -11,14 +12,15 @@ module Aspera
|
|
|
11
12
|
class << self
|
|
12
13
|
def from_list(object_id)
|
|
13
14
|
if object_id.is_a?(Array)
|
|
15
|
+
# compact: remove nils
|
|
14
16
|
object_id = object_id.compact.map do |i|
|
|
15
17
|
i.is_a?(String) && i.start_with?('https://') ? URI.parse(i).host : i.to_s
|
|
16
18
|
end.join(ID_SEPARATOR)
|
|
17
19
|
end
|
|
18
|
-
|
|
20
|
+
Aspera.assert_type(object_id, String)
|
|
19
21
|
return object_id
|
|
20
22
|
.gsub(WINDOWS_PROTECTED_CHAR, PROTECTED_CHAR_REPLACE) # remove windows forbidden chars
|
|
21
|
-
.gsub('.', PROTECTED_CHAR_REPLACE)
|
|
23
|
+
.gsub('.', PROTECTED_CHAR_REPLACE) # keep dot for extension only (nicer)
|
|
22
24
|
.downcase
|
|
23
25
|
end
|
|
24
26
|
end
|
data/lib/aspera/json_rpc.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
# cspell:ignore blankslate
|
|
4
4
|
|
|
5
5
|
require 'aspera/rest_error_analyzer'
|
|
6
|
+
require 'aspera/assert'
|
|
6
7
|
require 'blankslate'
|
|
7
8
|
|
|
8
9
|
Aspera::RestErrorAnalyzer.instance.add_simple_handler(name: 'JSON RPC', path: %w[error message], always: true)
|
|
@@ -34,15 +35,16 @@ module Aspera
|
|
|
34
35
|
params: args,
|
|
35
36
|
id: @request_id += 1
|
|
36
37
|
})[:data]
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
Aspera.assert_type(data, Hash){'response'}
|
|
39
|
+
Aspera.assert(data['jsonrpc'] == JSON_RPC_VERSION){'bad version in response'}
|
|
40
|
+
Aspera.assert(data.key?('id')){'missing id in response'}
|
|
41
|
+
Aspera.assert(!(data.key?('error') && data.key?('result'))){'both error and response'}
|
|
42
|
+
Aspera.assert(
|
|
42
43
|
!data.key?('error') ||
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
44
|
+
data['error'].is_a?(Hash) &&
|
|
45
|
+
data['error']['code'].is_a?(Integer) &&
|
|
46
|
+
data['error']['message'].is_a?(String)
|
|
47
|
+
){'bad error response'}
|
|
46
48
|
return data['result']
|
|
47
49
|
end
|
|
48
50
|
end
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
require 'aspera/hash_ext'
|
|
4
4
|
require 'aspera/environment'
|
|
5
|
+
require 'aspera/log'
|
|
6
|
+
require 'aspera/assert'
|
|
5
7
|
require 'symmetric_encryption/core'
|
|
6
8
|
require 'yaml'
|
|
7
9
|
|
|
@@ -9,33 +11,66 @@ module Aspera
|
|
|
9
11
|
module Keychain
|
|
10
12
|
# Manage secrets in a simple Hash
|
|
11
13
|
class EncryptedHash
|
|
12
|
-
|
|
14
|
+
LEGACY_CIPHER_NAME = 'aes-256-cbc'
|
|
15
|
+
DEFAULT_CIPHER_NAME = 'aes-256-cbc'
|
|
16
|
+
FILE_TYPE = 'encrypted_hash_vault'
|
|
13
17
|
CONTENT_KEYS = %i[label username password url description].freeze
|
|
18
|
+
FILE_KEYS = %w[version type cipher data].sort.freeze
|
|
14
19
|
def initialize(path, current_password)
|
|
20
|
+
Aspera.assert_type(path, String){'path to vault file'}
|
|
15
21
|
@path = path
|
|
22
|
+
@all_secrets = {}
|
|
23
|
+
vault_encrypted_data = nil
|
|
24
|
+
if File.exist?(@path)
|
|
25
|
+
vault_file = File.read(@path)
|
|
26
|
+
if vault_file.start_with?('---')
|
|
27
|
+
vault_info = YAML.parse(vault_file).to_ruby
|
|
28
|
+
Aspera.assert(vault_info.keys.sort == FILE_KEYS){'Invalid vault file'}
|
|
29
|
+
@cipher_name = vault_info['cipher']
|
|
30
|
+
vault_encrypted_data = vault_info['data']
|
|
31
|
+
else
|
|
32
|
+
# legacy vault file
|
|
33
|
+
@cipher_name = LEGACY_CIPHER_NAME
|
|
34
|
+
vault_encrypted_data = File.read(@path, mode: 'rb')
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
# setting password also creates the cipher
|
|
16
38
|
self.password = current_password
|
|
17
|
-
|
|
18
|
-
|
|
39
|
+
if !vault_encrypted_data.nil?
|
|
40
|
+
@all_secrets = YAML.load_stream(@cipher.decrypt(vault_encrypted_data)).first
|
|
41
|
+
end
|
|
19
42
|
end
|
|
20
43
|
|
|
44
|
+
# set the password and cipher
|
|
21
45
|
def password=(new_password)
|
|
22
46
|
# number of bits in second position
|
|
23
|
-
key_bytes =
|
|
47
|
+
key_bytes = DEFAULT_CIPHER_NAME.split('-')[1].to_i / Environment::BITS_PER_BYTE
|
|
24
48
|
# derive key from passphrase, add trailing zeros
|
|
25
49
|
key = "#{new_password}#{"\x0" * key_bytes}"[0..(key_bytes - 1)]
|
|
26
|
-
Log.log.
|
|
27
|
-
|
|
50
|
+
Log.log.trace1{"secret=[#{key}],#{key.length}"}
|
|
51
|
+
@cipher = SymmetricEncryption.cipher = SymmetricEncryption::Cipher.new(cipher_name: DEFAULT_CIPHER_NAME, key: key, encoding: :none)
|
|
28
52
|
end
|
|
29
53
|
|
|
54
|
+
# save current data to file with format
|
|
30
55
|
def save
|
|
31
|
-
|
|
56
|
+
vault_info = {
|
|
57
|
+
'version' => '1.0.0',
|
|
58
|
+
'type' => FILE_TYPE,
|
|
59
|
+
'cipher' => @cipher_name,
|
|
60
|
+
'data' => @cipher.encrypt(YAML.dump(@all_secrets))
|
|
61
|
+
}
|
|
62
|
+
File.write(@path, YAML.dump(vault_info))
|
|
32
63
|
end
|
|
33
64
|
|
|
65
|
+
# set a secret
|
|
66
|
+
# @param options [Hash] with keys :label, :username, :password, :url, :description
|
|
34
67
|
def set(options)
|
|
35
|
-
|
|
68
|
+
Aspera.assert_type(options, Hash){'options'}
|
|
36
69
|
unsupported = options.keys - CONTENT_KEYS
|
|
37
|
-
|
|
38
|
-
|
|
70
|
+
Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
|
71
|
+
options.each_pair do |k, v|
|
|
72
|
+
Aspera.assert_type(v, String){k.to_s}
|
|
73
|
+
end
|
|
39
74
|
label = options.delete(:label)
|
|
40
75
|
raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
|
|
41
76
|
@all_secrets[label] = options.symbolize_keys
|
|
@@ -59,7 +94,7 @@ module Aspera
|
|
|
59
94
|
end
|
|
60
95
|
|
|
61
96
|
def get(label:, exception: true)
|
|
62
|
-
|
|
97
|
+
Aspera.assert(@all_secrets.key?(label)){"Label not found: #{label}"} if exception
|
|
63
98
|
result = @all_secrets[label].clone
|
|
64
99
|
result[:label] = label if result.is_a?(Hash)
|
|
65
100
|
return result
|