aspera-cli 4.15.0 → 4.16.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +29 -3
- data/CHANGELOG.md +292 -228
- data/CONTRIBUTING.md +69 -18
- data/README.md +1102 -952
- data/bin/ascli +13 -31
- data/bin/asession +3 -1
- data/examples/dascli +2 -2
- data/lib/aspera/aoc.rb +28 -33
- data/lib/aspera/ascmd.rb +3 -6
- data/lib/aspera/assert.rb +45 -0
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formatter.rb +26 -13
- data/lib/aspera/cli/hints.rb +4 -3
- data/lib/aspera/cli/main.rb +16 -3
- data/lib/aspera/cli/manager.rb +45 -36
- data/lib/aspera/cli/plugin.rb +20 -13
- data/lib/aspera/cli/plugins/aoc.rb +103 -73
- data/lib/aspera/cli/plugins/ats.rb +4 -3
- data/lib/aspera/cli/plugins/config.rb +114 -119
- data/lib/aspera/cli/plugins/cos.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +23 -19
- data/lib/aspera/cli/plugins/faspex5.rb +75 -43
- data/lib/aspera/cli/plugins/node.rb +28 -15
- data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
- data/lib/aspera/cli/plugins/preview.rb +9 -7
- data/lib/aspera/cli/plugins/server.rb +6 -3
- data/lib/aspera/cli/plugins/shares.rb +30 -26
- data/lib/aspera/cli/sync_actions.rb +9 -9
- data/lib/aspera/cli/transfer_agent.rb +21 -14
- data/lib/aspera/cli/transfer_progress.rb +2 -3
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +13 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/coverage.rb +22 -0
- data/lib/aspera/data_repository.rb +33 -2
- data/lib/aspera/environment.rb +4 -2
- data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
- data/lib/aspera/fasp/agent_base.rb +17 -7
- data/lib/aspera/fasp/agent_direct.rb +88 -84
- data/lib/aspera/fasp/agent_httpgw.rb +4 -3
- data/lib/aspera/fasp/agent_node.rb +3 -2
- data/lib/aspera/fasp/agent_trsdk.rb +79 -37
- data/lib/aspera/fasp/installation.rb +51 -12
- data/lib/aspera/fasp/management.rb +11 -6
- data/lib/aspera/fasp/parameters.rb +53 -47
- data/lib/aspera/fasp/resume_policy.rb +7 -5
- data/lib/aspera/fasp/sync.rb +273 -0
- data/lib/aspera/fasp/transfer_spec.rb +10 -8
- data/lib/aspera/fasp/uri.rb +2 -2
- data/lib/aspera/faspex_gw.rb +11 -8
- data/lib/aspera/faspex_postproc.rb +6 -5
- data/lib/aspera/id_generator.rb +3 -1
- data/lib/aspera/json_rpc.rb +10 -8
- data/lib/aspera/keychain/encrypted_hash.rb +46 -11
- data/lib/aspera/keychain/macos_security.rb +15 -13
- data/lib/aspera/log.rb +4 -3
- data/lib/aspera/nagios.rb +7 -2
- data/lib/aspera/node.rb +17 -16
- data/lib/aspera/node_simulator.rb +214 -0
- data/lib/aspera/oauth.rb +22 -19
- data/lib/aspera/persistency_action_once.rb +13 -14
- data/lib/aspera/persistency_folder.rb +3 -2
- data/lib/aspera/preview/file_types.rb +53 -267
- data/lib/aspera/preview/generator.rb +7 -5
- data/lib/aspera/preview/terminal.rb +14 -5
- data/lib/aspera/preview/utils.rb +8 -7
- data/lib/aspera/proxy_auto_config.rb +6 -3
- data/lib/aspera/rest.rb +29 -13
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/rest_errors_aspera.rb +2 -0
- data/lib/aspera/secret_hider.rb +5 -2
- data/lib/aspera/ssh.rb +10 -8
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/web_server_simple.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +96 -45
- metadata.gz.sig +0 -0
- data/lib/aspera/sync.rb +0 -219
@@ -0,0 +1,273 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# cspell:words logdir bidi watchd cooloff asyncadmin
|
4
|
+
|
5
|
+
require 'aspera/command_line_builder'
|
6
|
+
require 'aspera/fasp/installation'
|
7
|
+
require 'aspera/log'
|
8
|
+
require 'aspera/assert'
|
9
|
+
require 'json'
|
10
|
+
require 'base64'
|
11
|
+
require 'open3'
|
12
|
+
require 'English'
|
13
|
+
|
14
|
+
module Aspera
|
15
|
+
module Fasp
|
16
|
+
# builds command line arg for async
|
17
|
+
module Sync
|
18
|
+
# sync direction, default is push
|
19
|
+
DIRECTIONS = %i[push pull bidi].freeze
|
20
|
+
# custom JSON for async instance command line options
|
21
|
+
PARAMS_VX_INSTANCE =
|
22
|
+
{
|
23
|
+
'alt_logdir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
24
|
+
'watchd' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
25
|
+
'apply_local_docroot' => { cli: { type: :opt_without_arg}},
|
26
|
+
'quiet' => { cli: { type: :opt_without_arg}},
|
27
|
+
'ws_connect' => { cli: { type: :opt_without_arg}}
|
28
|
+
}.freeze
|
29
|
+
|
30
|
+
# map sync session parameters to transfer spec: sync -> ts, true if same
|
31
|
+
PARAMS_VX_SESSION =
|
32
|
+
{
|
33
|
+
'name' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
34
|
+
'local_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
35
|
+
'remote_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
36
|
+
'local_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
37
|
+
'remote_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
38
|
+
'host' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_host},
|
39
|
+
'user' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_user},
|
40
|
+
'private_key_paths' => { cli: { type: :opt_with_arg, switch: '--private-key-path'}, accepted_types: :array},
|
41
|
+
'direction' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
42
|
+
'checksum' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
43
|
+
'tags' => { cli: { type: :opt_with_arg, switch: '--tags64', convert: 'Aspera::Fasp::Parameters.convert_json64'},
|
44
|
+
accepted_types: :hash, ts: true},
|
45
|
+
'tcp_port' => { cli: { type: :opt_with_arg}, accepted_types: :int, ts: :ssh_port},
|
46
|
+
'rate_policy' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
47
|
+
'target_rate' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
48
|
+
'cooloff' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
49
|
+
'pending_max' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
50
|
+
'scan_intensity' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
51
|
+
'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Fasp::Parameters.convert_remove_hyphen'}, accepted_types: :string, ts: true},
|
52
|
+
'transfer_threads' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
53
|
+
'preserve_time' => { cli: { type: :opt_without_arg}, ts: :preserve_times},
|
54
|
+
'preserve_access_time' => { cli: { type: :opt_without_arg}, ts: nil},
|
55
|
+
'preserve_modification_time' => { cli: { type: :opt_without_arg}, ts: nil},
|
56
|
+
'preserve_uid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_uid},
|
57
|
+
'preserve_gid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_gid},
|
58
|
+
'create_dir' => { cli: { type: :opt_without_arg}, ts: true},
|
59
|
+
'reset' => { cli: { type: :opt_without_arg}},
|
60
|
+
# NOTE: only one env var, but multiple sessions... could be a problem
|
61
|
+
'remote_password' => { cli: { type: :envvar, variable: 'ASPERA_SCP_PASS'}, ts: true},
|
62
|
+
'cookie' => { cli: { type: :envvar, variable: 'ASPERA_SCP_COOKIE'}, ts: true},
|
63
|
+
'token' => { cli: { type: :envvar, variable: 'ASPERA_SCP_TOKEN'}, ts: true},
|
64
|
+
'license' => { cli: { type: :envvar, variable: 'ASPERA_SCP_LICENSE'}}
|
65
|
+
}.freeze
|
66
|
+
|
67
|
+
Aspera::CommandLineBuilder.normalize_description(PARAMS_VX_INSTANCE)
|
68
|
+
Aspera::CommandLineBuilder.normalize_description(PARAMS_VX_SESSION)
|
69
|
+
|
70
|
+
PARAMS_VX_KEYS = %w[instance sessions].freeze
|
71
|
+
|
72
|
+
# Translation of transfer spec parameters to async v2 API (asyncs)
|
73
|
+
TS_TO_PARAMS_V2 = {
|
74
|
+
'remote_host' => 'remote.host',
|
75
|
+
'remote_user' => 'remote.user',
|
76
|
+
'remote_password' => 'remote.pass',
|
77
|
+
'sshfp' => 'remote.fingerprint',
|
78
|
+
'ssh_port' => 'remote.port',
|
79
|
+
'wss_port' => 'remote.ws_port',
|
80
|
+
'proxy' => 'remote.proxy',
|
81
|
+
'token' => 'remote.token',
|
82
|
+
'tags' => 'tags'
|
83
|
+
}.freeze
|
84
|
+
|
85
|
+
ASYNC_EXECUTABLE = 'async'
|
86
|
+
ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
|
87
|
+
|
88
|
+
private_constant :PARAMS_VX_INSTANCE, :PARAMS_VX_SESSION, :PARAMS_VX_KEYS, :TS_TO_PARAMS_V2, :ASYNC_EXECUTABLE, :ASYNC_ADMIN_EXECUTABLE
|
89
|
+
|
90
|
+
class << self
|
91
|
+
# Set remote_dir in sync parameters based on transfer spec
|
92
|
+
# @param params [Hash] sync parameters, old or new format
|
93
|
+
# @param remote_dir_key [String] key to update in above hash
|
94
|
+
# @param transfer_spec [Hash] transfer spec
|
95
|
+
def update_remote_dir(sync_params, remote_dir_key, transfer_spec)
|
96
|
+
if transfer_spec.dig(*%w[tags aspera node file_id])
|
97
|
+
# in AoC, use gen4
|
98
|
+
sync_params[remote_dir_key] = '/'
|
99
|
+
elsif transfer_spec['cookie']&.start_with?('aspera.shares2')
|
100
|
+
# TODO : something more generic, independent of Shares
|
101
|
+
# in Shares, the actual folder on remote end is not always the same as the name of the share
|
102
|
+
actual_remote = transfer_spec['paths']&.first&.[]('source')
|
103
|
+
sync_params[remote_dir_key] = actual_remote if actual_remote
|
104
|
+
end
|
105
|
+
nil
|
106
|
+
end
|
107
|
+
|
108
|
+
def remote_certificates(remote)
|
109
|
+
certificates_to_use = []
|
110
|
+
# use web socket secure for session ?
|
111
|
+
if remote['connect_mode']&.eql?('ws')
|
112
|
+
remote.delete('port')
|
113
|
+
remote.delete('fingerprint')
|
114
|
+
# ignore cert for wss ?
|
115
|
+
if false # @options[:check_ignore]&.call(remote['host'], remote['ws_port'])
|
116
|
+
wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
|
117
|
+
wss_url = "https://#{remote['host']}:#{remote['ws_port']}"
|
118
|
+
File.write(wss_cert_file, Rest.remote_certificates(wss_url))
|
119
|
+
certificates_to_use.push(wss_cert_file)
|
120
|
+
end
|
121
|
+
# set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
|
122
|
+
# certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs]
|
123
|
+
else
|
124
|
+
# remove unused parameter (avoid warning)
|
125
|
+
remote.delete('ws_port')
|
126
|
+
# add SSH bypass keys when authentication is token and no auth is provided
|
127
|
+
if remote.key?('token') && !remote.key?('pass')
|
128
|
+
certificates_to_use.concat(Installation.instance.aspera_token_ssh_key_paths)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
return certificates_to_use
|
132
|
+
end
|
133
|
+
|
134
|
+
# @param sync_params [Hash] sync parameters, old or new format
|
135
|
+
# @param block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
|
136
|
+
def start(sync_params, &block)
|
137
|
+
assert_type(sync_params, Hash)
|
138
|
+
env_args = {
|
139
|
+
args: [],
|
140
|
+
env: {}
|
141
|
+
}
|
142
|
+
if sync_params.key?('local')
|
143
|
+
remote = sync_params['remote']
|
144
|
+
# async native JSON format (v2)
|
145
|
+
assert_type(remote, Hash)
|
146
|
+
# get transfer spec if possible, and feed back to new structure
|
147
|
+
if block
|
148
|
+
transfer_spec = yield((sync_params['direction'] || 'push').to_sym, sync_params['local']['path'], remote['path'])
|
149
|
+
# async native JSON format
|
150
|
+
assert_type(sync_params['local'], Hash)
|
151
|
+
# translate transfer spec to async parameters
|
152
|
+
TS_TO_PARAMS_V2.each do |ts_param, sy_path|
|
153
|
+
next unless transfer_spec.key?(ts_param)
|
154
|
+
sy_dig = sy_path.split('.')
|
155
|
+
param = sy_dig.pop
|
156
|
+
hash = sy_dig.empty? ? sync_params : sync_params[sy_dig.first]
|
157
|
+
hash = sync_params[sy_dig.first] = {} if hash.nil?
|
158
|
+
hash[param] = transfer_spec[ts_param]
|
159
|
+
end
|
160
|
+
update_remote_dir(remote, 'path', transfer_spec)
|
161
|
+
end
|
162
|
+
remote['connect_mode'] ||= remote.key?('ws_port') ? 'ws' : 'ssh'
|
163
|
+
add_certificates = remote_certificates(remote)
|
164
|
+
if !add_certificates.empty?
|
165
|
+
remote['private_key_paths'] ||= []
|
166
|
+
remote['private_key_paths'].concat(add_certificates)
|
167
|
+
end
|
168
|
+
assert_type(sync_params, Hash)
|
169
|
+
env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"]
|
170
|
+
elsif sync_params.key?('sessions')
|
171
|
+
# ascli JSON format (v1)
|
172
|
+
if block
|
173
|
+
sync_params['sessions'].each do |session|
|
174
|
+
transfer_spec = yield((session['direction'] || 'push').to_sym, session['local_dir'], session['remote_dir'])
|
175
|
+
PARAMS_VX_SESSION.each do |async_param, behavior|
|
176
|
+
if behavior.key?(:ts)
|
177
|
+
tspec_param = behavior[:ts].is_a?(TrueClass) ? async_param : behavior[:ts].to_s
|
178
|
+
session[async_param] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
session['private_key_paths'] = Fasp::Installation.instance.aspera_token_ssh_key_paths if transfer_spec.key?('token')
|
182
|
+
update_remote_dir(session, 'remote_dir', transfer_spec)
|
183
|
+
end
|
184
|
+
end
|
185
|
+
raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
|
186
|
+
sync_params.keys.push('instance').uniq.sort.eql?(PARAMS_VX_KEYS)
|
187
|
+
assert_type(sync_params['sessions'], Array)
|
188
|
+
assert_type(sync_params['sessions'].first, Hash)
|
189
|
+
if sync_params.key?('instance')
|
190
|
+
assert_type(sync_params['instance'], Hash)
|
191
|
+
instance_builder = Aspera::CommandLineBuilder.new(sync_params['instance'], PARAMS_VX_INSTANCE)
|
192
|
+
instance_builder.process_params
|
193
|
+
instance_builder.add_env_args(env_args)
|
194
|
+
end
|
195
|
+
|
196
|
+
sync_params['sessions'].each do |session_params|
|
197
|
+
assert_type(session_params, Hash)
|
198
|
+
assert(session_params.key?('name')){'session must contain at least name'}
|
199
|
+
session_builder = Aspera::CommandLineBuilder.new(session_params, PARAMS_VX_SESSION)
|
200
|
+
session_builder.process_params
|
201
|
+
session_builder.add_env_args(env_args)
|
202
|
+
end
|
203
|
+
else
|
204
|
+
raise 'At least one of `local` or `sessions` must be present in async parameters'
|
205
|
+
end
|
206
|
+
Log.log.debug{Log.dump(:sync_params, sync_params)}
|
207
|
+
Log.log.debug{"execute: #{env_args[:env].map{|k, v| "#{k}=\"#{v}\""}.join(' ')} \"#{ASYNC_EXECUTABLE}\" \"#{env_args[:args].join('" "')}\""}
|
208
|
+
res = system(env_args[:env], [ASYNC_EXECUTABLE, ASYNC_EXECUTABLE], *env_args[:args])
|
209
|
+
Log.log.debug{"result=#{res}"}
|
210
|
+
case res
|
211
|
+
when true then return nil
|
212
|
+
when false then raise "failed: #{$CHILD_STATUS}"
|
213
|
+
when nil then raise "not started: #{$CHILD_STATUS}"
|
214
|
+
else error_unexpected_value(res)
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
def parse_status(stdout)
|
219
|
+
Log.log.trace1{"stdout=#{stdout}"}
|
220
|
+
result = {}
|
221
|
+
ids = nil
|
222
|
+
stdout.split("\n").each do |line|
|
223
|
+
info = line.split(':', 2).map(&:lstrip)
|
224
|
+
if info[1].eql?('')
|
225
|
+
info[1] = ids = []
|
226
|
+
elsif info[1].nil?
|
227
|
+
ids.push(info[0])
|
228
|
+
next
|
229
|
+
end
|
230
|
+
result[info[0]] = info[1]
|
231
|
+
end
|
232
|
+
return result
|
233
|
+
end
|
234
|
+
|
235
|
+
def admin_status(sync_params, session_name)
|
236
|
+
command_line = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
|
237
|
+
if sync_params.key?('local')
|
238
|
+
assert(!sync_params['name'].nil?){'Missing session name'}
|
239
|
+
assert(session_name.nil? || session_name.eql?(sync_params['name'])){'Session not found'}
|
240
|
+
command_line.push("--name=#{sync_params['name']}")
|
241
|
+
if sync_params.key?('local_db_dir')
|
242
|
+
command_line.push("--local-db-dir=#{sync_params['local_db_dir']}")
|
243
|
+
elsif sync_params.dig('local', 'path')
|
244
|
+
command_line.push("--local-dir=#{sync_params.dig('local', 'path')}")
|
245
|
+
else
|
246
|
+
raise 'Missing either local_db_dir or local.path'
|
247
|
+
end
|
248
|
+
elsif sync_params.key?('sessions')
|
249
|
+
session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{|s|s['name'].eql?(session_name)}
|
250
|
+
raise "Session #{session_name} not found in #{sync_params['sessions'].map{|s|s['name']}.join(',')}" if session.nil?
|
251
|
+
raise 'Missing session name' if session['name'].nil?
|
252
|
+
command_line.push("--name=#{session['name']}")
|
253
|
+
if session.key?('local_db_dir')
|
254
|
+
command_line.push("--local-db-dir=#{session['local_db_dir']}")
|
255
|
+
elsif session.key?('local_dir')
|
256
|
+
command_line.push("--local-dir=#{session['local_dir']}")
|
257
|
+
else
|
258
|
+
raise 'Missing either local_db_dir or local_dir'
|
259
|
+
end
|
260
|
+
else
|
261
|
+
raise 'At least one of `local` or `sessions` must be present in async parameters'
|
262
|
+
end
|
263
|
+
Log.log.debug{"execute: #{command_line.join(' ')}"}
|
264
|
+
stdout, stderr, status = Open3.capture3(*command_line)
|
265
|
+
Log.log.debug{"status=#{status}, stderr=#{stderr}"}
|
266
|
+
Log.log.trace1{"stdout=#{stdout}"}
|
267
|
+
raise "Sync failed: #{status.exitstatus} : #{stderr}" unless status.success?
|
268
|
+
return parse_status(stdout)
|
269
|
+
end
|
270
|
+
end # end self
|
271
|
+
end # end Sync
|
272
|
+
end # end Fasp
|
273
|
+
end # end Aspera
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/fasp/parameters'
|
4
|
+
require 'aspera/assert'
|
4
5
|
|
5
6
|
module Aspera
|
6
7
|
module Fasp
|
@@ -27,22 +28,23 @@ module Aspera
|
|
27
28
|
end
|
28
29
|
class << self
|
29
30
|
def action_to_direction(tspec, command)
|
30
|
-
|
31
|
+
assert_type(tspec, Hash){'transfer spec'}
|
31
32
|
tspec['direction'] = case command.to_sym
|
32
33
|
when :upload then DIRECTION_SEND
|
33
34
|
when :download then DIRECTION_RECEIVE
|
34
|
-
else
|
35
|
+
else error_unexpected_value(command.to_sym)
|
35
36
|
end
|
36
37
|
return tspec
|
37
38
|
end
|
38
39
|
|
39
40
|
def action(tspec)
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
41
|
+
assert_type(tspec, Hash){'transfer spec'}
|
42
|
+
assert_values(tspec['direction'], [DIRECTION_SEND, DIRECTION_RECEIVE]){'direction'}
|
43
|
+
case tspec['direction']
|
44
|
+
when DIRECTION_SEND then :upload
|
45
|
+
when DIRECTION_RECEIVE then :download
|
46
|
+
else error_unexpected_value(tspec['direction'])
|
47
|
+
end
|
46
48
|
end
|
47
49
|
end
|
48
50
|
end
|
data/lib/aspera/fasp/uri.rb
CHANGED
@@ -8,12 +8,12 @@ require 'aspera/command_line_builder'
|
|
8
8
|
|
9
9
|
module Aspera
|
10
10
|
module Fasp
|
11
|
-
# translates a "faspe:" URI (used in Faspex 4) into transfer spec
|
11
|
+
# translates a "faspe:" URI (used in Faspex 4) into transfer spec (Hash)
|
12
12
|
class Uri
|
13
13
|
SCHEME = 'faspe'
|
14
14
|
def initialize(fasp_link)
|
15
15
|
@fasp_uri = URI.parse(fasp_link.gsub(' ', '%20'))
|
16
|
-
# TODO: check scheme is faspe
|
16
|
+
# TODO: check scheme is 'faspe'
|
17
17
|
end
|
18
18
|
|
19
19
|
def transfer_spec
|
data/lib/aspera/faspex_gw.rb
CHANGED
@@ -1,15 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'aspera/web_server_simple'
|
4
3
|
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
5
|
+
require 'webrick'
|
5
6
|
require 'json'
|
6
7
|
|
7
8
|
module Aspera
|
8
|
-
#
|
9
|
+
# Simulate the Faspex 4 /send API and creates a package on Aspera on Cloud or Faspex 5
|
9
10
|
class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
|
10
11
|
# @param app_api [Aspera::AoC]
|
11
12
|
# @param app_context [String]
|
12
13
|
def initialize(server, app_api, app_context)
|
14
|
+
assert_values(app_api.class.name, ['Aspera::AoC', 'Aspera::Rest'])
|
13
15
|
super(server)
|
14
16
|
# typed: Aspera::AoC
|
15
17
|
@app_api = app_api
|
@@ -67,13 +69,14 @@ module Aspera
|
|
67
69
|
raise 'no payload' if request.body.nil?
|
68
70
|
faspex_pkg_parameters = JSON.parse(request.body)
|
69
71
|
Log.log.debug{"faspex pkg create parameters=#{faspex_pkg_parameters}"}
|
72
|
+
# compare string, as class is not yet known here
|
70
73
|
faspex_package_create_result =
|
71
|
-
|
74
|
+
case @app_api.class.name
|
75
|
+
when 'Aspera::AoC'
|
72
76
|
faspex4_send_to_aoc(faspex_pkg_parameters)
|
73
|
-
|
77
|
+
when 'Aspera::Rest'
|
74
78
|
faspex4_send_to_faspex5(faspex_pkg_parameters)
|
75
|
-
else
|
76
|
-
raise "No such adapter: #{@app_api.class}"
|
79
|
+
else error_unexpected_value(@app_api.class.name)
|
77
80
|
end
|
78
81
|
Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
|
79
82
|
response.status = 200
|
@@ -89,8 +92,8 @@ module Aspera
|
|
89
92
|
else
|
90
93
|
response.status = 400
|
91
94
|
response['Content-Type'] = 'application/json'
|
92
|
-
response.body = {error: '
|
95
|
+
response.body = {error: 'Unsupported endpoint'}.to_json
|
93
96
|
end
|
94
97
|
end
|
95
98
|
end # Faspex4GWServlet
|
96
|
-
end #
|
99
|
+
end # Aspera
|
@@ -1,17 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'English'
|
4
|
-
require 'aspera/web_server_simple'
|
5
|
-
require 'aspera/log'
|
6
3
|
require 'json'
|
7
4
|
require 'timeout'
|
5
|
+
require 'English'
|
6
|
+
require 'webrick'
|
7
|
+
require 'aspera/log'
|
8
|
+
require 'aspera/assert'
|
8
9
|
|
9
10
|
module Aspera
|
10
11
|
# this class answers the Faspex /send API and creates a package on Aspera on Cloud
|
11
12
|
class Faspex4PostProcServlet < WEBrick::HTTPServlet::AbstractServlet
|
12
13
|
ALLOWED_PARAMETERS = %i[root script_folder fail_on_error timeout_seconds].freeze
|
13
14
|
def initialize(server, parameters)
|
14
|
-
|
15
|
+
assert_type(parameters, Hash)
|
15
16
|
@parameters = parameters.symbolize_keys
|
16
17
|
Log.log.debug{Log.dump(:post_proc_parameters, @parameters)}
|
17
18
|
raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
|
@@ -74,4 +75,4 @@ module Aspera
|
|
74
75
|
end
|
75
76
|
end
|
76
77
|
end # Faspex4PostProcServlet
|
77
|
-
end #
|
78
|
+
end # Aspera
|
data/lib/aspera/id_generator.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/assert'
|
3
4
|
require 'uri'
|
4
5
|
|
5
6
|
module Aspera
|
@@ -11,11 +12,12 @@ module Aspera
|
|
11
12
|
class << self
|
12
13
|
def from_list(object_id)
|
13
14
|
if object_id.is_a?(Array)
|
15
|
+
# compact: remove nils
|
14
16
|
object_id = object_id.compact.map do |i|
|
15
17
|
i.is_a?(String) && i.start_with?('https://') ? URI.parse(i).host : i.to_s
|
16
18
|
end.join(ID_SEPARATOR)
|
17
19
|
end
|
18
|
-
|
20
|
+
assert_type(object_id, String)
|
19
21
|
return object_id
|
20
22
|
.gsub(WINDOWS_PROTECTED_CHAR, PROTECTED_CHAR_REPLACE) # remove windows forbidden chars
|
21
23
|
.gsub('.', PROTECTED_CHAR_REPLACE) # keep dot for extension only (nicer)
|
data/lib/aspera/json_rpc.rb
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
# cspell:ignore blankslate
|
4
4
|
|
5
5
|
require 'aspera/rest_error_analyzer'
|
6
|
+
require 'aspera/assert'
|
6
7
|
require 'blankslate'
|
7
8
|
|
8
9
|
Aspera::RestErrorAnalyzer.instance.add_simple_handler(name: 'JSON RPC', path: %w[error message], always: true)
|
@@ -34,15 +35,16 @@ module Aspera
|
|
34
35
|
params: args,
|
35
36
|
id: @request_id += 1
|
36
37
|
})[:data]
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
38
|
+
assert_type(data, Hash){'response'}
|
39
|
+
assert(data['jsonrpc'] == JSON_RPC_VERSION){'bad version in response'}
|
40
|
+
assert(data.key?('id')){'missing id in response'}
|
41
|
+
assert(!(data.key?('error') && data.key?('result'))){'both error and response'}
|
42
|
+
assert(
|
42
43
|
!data.key?('error') ||
|
43
|
-
|
44
|
-
|
45
|
-
|
44
|
+
data['error'].is_a?(Hash) &&
|
45
|
+
data['error']['code'].is_a?(Integer) &&
|
46
|
+
data['error']['message'].is_a?(String)
|
47
|
+
){'bad error response'}
|
46
48
|
return data['result']
|
47
49
|
end
|
48
50
|
end
|
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
require 'aspera/hash_ext'
|
4
4
|
require 'aspera/environment'
|
5
|
+
require 'aspera/log'
|
6
|
+
require 'aspera/assert'
|
5
7
|
require 'symmetric_encryption/core'
|
6
8
|
require 'yaml'
|
7
9
|
|
@@ -9,33 +11,66 @@ module Aspera
|
|
9
11
|
module Keychain
|
10
12
|
# Manage secrets in a simple Hash
|
11
13
|
class EncryptedHash
|
12
|
-
|
14
|
+
LEGACY_CIPHER_NAME = 'aes-256-cbc'
|
15
|
+
DEFAULT_CIPHER_NAME = 'aes-256-cbc'
|
16
|
+
FILE_TYPE = 'encrypted_hash_vault'
|
13
17
|
CONTENT_KEYS = %i[label username password url description].freeze
|
18
|
+
FILE_KEYS = %w[version type cipher data].sort.freeze
|
14
19
|
def initialize(path, current_password)
|
20
|
+
assert_type(path, String){'path to vault file'}
|
15
21
|
@path = path
|
22
|
+
@all_secrets = {}
|
23
|
+
vault_encrypted_data = nil
|
24
|
+
if File.exist?(@path)
|
25
|
+
vault_file = File.read(@path)
|
26
|
+
if vault_file.start_with?('---')
|
27
|
+
vault_info = YAML.parse(vault_file).to_ruby
|
28
|
+
assert(vault_info.keys.sort == FILE_KEYS){'Invalid vault file'}
|
29
|
+
@cipher_name = vault_info['cipher']
|
30
|
+
vault_encrypted_data = vault_info['data']
|
31
|
+
else
|
32
|
+
# legacy vault file
|
33
|
+
@cipher_name = LEGACY_CIPHER_NAME
|
34
|
+
vault_encrypted_data = File.read(@path, mode: 'rb')
|
35
|
+
end
|
36
|
+
end
|
37
|
+
# setting password also creates the cipher
|
16
38
|
self.password = current_password
|
17
|
-
|
18
|
-
|
39
|
+
if !vault_encrypted_data.nil?
|
40
|
+
@all_secrets = YAML.load_stream(@cipher.decrypt(vault_encrypted_data)).first
|
41
|
+
end
|
19
42
|
end
|
20
43
|
|
44
|
+
# set the password and cipher
|
21
45
|
def password=(new_password)
|
22
46
|
# number of bits in second position
|
23
|
-
key_bytes =
|
47
|
+
key_bytes = DEFAULT_CIPHER_NAME.split('-')[1].to_i / Environment::BITS_PER_BYTE
|
24
48
|
# derive key from passphrase, add trailing zeros
|
25
49
|
key = "#{new_password}#{"\x0" * key_bytes}"[0..(key_bytes - 1)]
|
26
|
-
Log.log.
|
27
|
-
|
50
|
+
Log.log.trace1{"secret=[#{key}],#{key.length}"}
|
51
|
+
@cipher = SymmetricEncryption.cipher = SymmetricEncryption::Cipher.new(cipher_name: DEFAULT_CIPHER_NAME, key: key, encoding: :none)
|
28
52
|
end
|
29
53
|
|
54
|
+
# save current data to file with format
|
30
55
|
def save
|
31
|
-
|
56
|
+
vault_info = {
|
57
|
+
'version' => '1.0.0',
|
58
|
+
'type' => FILE_TYPE,
|
59
|
+
'cipher' => @cipher_name,
|
60
|
+
'data' => @cipher.encrypt(YAML.dump(@all_secrets))
|
61
|
+
}
|
62
|
+
File.write(@path, YAML.dump(vault_info))
|
32
63
|
end
|
33
64
|
|
65
|
+
# set a secret
|
66
|
+
# @param options [Hash] with keys :label, :username, :password, :url, :description
|
34
67
|
def set(options)
|
35
|
-
|
68
|
+
assert_type(options, Hash){'options'}
|
36
69
|
unsupported = options.keys - CONTENT_KEYS
|
37
|
-
|
38
|
-
|
70
|
+
assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
71
|
+
options.each_pair do |k, v|
|
72
|
+
assert_type(v, String){k.to_s}
|
73
|
+
end
|
39
74
|
label = options.delete(:label)
|
40
75
|
raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
|
41
76
|
@all_secrets[label] = options.symbolize_keys
|
@@ -59,7 +94,7 @@ module Aspera
|
|
59
94
|
end
|
60
95
|
|
61
96
|
def get(label:, exception: true)
|
62
|
-
|
97
|
+
assert(@all_secrets.key?(label)){"Label not found: #{label}"} if exception
|
63
98
|
result = @all_secrets[label].clone
|
64
99
|
result[:label] = label if result.is_a?(Hash)
|
65
100
|
return result
|
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
# https://github.com/fastlane-community/security
|
4
4
|
require 'aspera/cli/info'
|
5
|
+
require 'aspera/log'
|
6
|
+
require 'aspera/assert'
|
5
7
|
|
6
8
|
# enhance the gem to support other key chains
|
7
9
|
module Aspera
|
@@ -36,7 +38,7 @@ module Aspera
|
|
36
38
|
url = options&.delete(:url)
|
37
39
|
if !url.nil?
|
38
40
|
uri = URI.parse(url)
|
39
|
-
|
41
|
+
assert(uri.scheme.eql?('https')){'only https'}
|
40
42
|
options[:protocol] = 'htps' # cspell: disable-line
|
41
43
|
raise 'host required in URL' if uri.host.nil?
|
42
44
|
options[:server] = uri.host
|
@@ -45,7 +47,7 @@ module Aspera
|
|
45
47
|
end
|
46
48
|
cmd = ['security', command]
|
47
49
|
options&.each do |k, v|
|
48
|
-
|
50
|
+
assert(supported.key?(k)){"unknown option: #{k}"}
|
49
51
|
next if v.nil?
|
50
52
|
cmd.push("-#{supported[k]}")
|
51
53
|
cmd.push(v.shellescape) unless v.empty?
|
@@ -70,7 +72,7 @@ module Aspera
|
|
70
72
|
end
|
71
73
|
|
72
74
|
def list(options={})
|
73
|
-
|
75
|
+
assert_values(options[:domain], DOMAINS, exception_class: ArgumentError){'domain'} unless options[:domain].nil?
|
74
76
|
key_chains(execute('list-key_chains', options, LIST_OPTIONS))
|
75
77
|
end
|
76
78
|
|
@@ -89,11 +91,11 @@ module Aspera
|
|
89
91
|
end
|
90
92
|
|
91
93
|
def password(operation, pass_type, options)
|
92
|
-
|
93
|
-
|
94
|
-
|
94
|
+
assert_values(operation, %i[add find delete]){'operation'}
|
95
|
+
assert_values(pass_type, %i[generic internet]){'pass_type'}
|
96
|
+
assert_type(options, Hash)
|
95
97
|
missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
|
96
|
-
|
98
|
+
assert(missing.empty?){"missing options: #{missing}"}
|
97
99
|
options[:getpass] = '' if operation.eql?(:find)
|
98
100
|
output = self.class.execute("#{operation}-#{pass_type}-password", options, ADD_PASS_OPTIONS, @path)
|
99
101
|
raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
|
@@ -127,18 +129,18 @@ module Aspera
|
|
127
129
|
end
|
128
130
|
|
129
131
|
def set(options)
|
130
|
-
|
132
|
+
assert_type(options, Hash){'options'}
|
131
133
|
unsupported = options.keys - %i[label username password url description]
|
132
|
-
|
134
|
+
assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
133
135
|
@keychain.password(
|
134
136
|
:add, :generic, service: options[:label],
|
135
137
|
account: options[:username] || 'none', password: options[:password], comment: options[:description])
|
136
138
|
end
|
137
139
|
|
138
140
|
def get(options)
|
139
|
-
|
141
|
+
assert_type(options, Hash){'options'}
|
140
142
|
unsupported = options.keys - %i[label]
|
141
|
-
|
143
|
+
assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
142
144
|
info = @keychain.password(:find, :generic, label: options[:label])
|
143
145
|
raise 'not found' if info.nil?
|
144
146
|
result = options.clone
|
@@ -153,9 +155,9 @@ module Aspera
|
|
153
155
|
end
|
154
156
|
|
155
157
|
def delete(options)
|
156
|
-
|
158
|
+
assert_type(options, Hash){'options'}
|
157
159
|
unsupported = options.keys - %i[label]
|
158
|
-
|
160
|
+
assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
159
161
|
raise 'delete not implemented, use macos keychain app'
|
160
162
|
end
|
161
163
|
end
|