aspera-cli 4.15.0 → 4.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +29 -3
- data/CHANGELOG.md +292 -228
- data/CONTRIBUTING.md +69 -18
- data/README.md +1102 -952
- data/bin/ascli +13 -31
- data/bin/asession +3 -1
- data/examples/dascli +2 -2
- data/lib/aspera/aoc.rb +28 -33
- data/lib/aspera/ascmd.rb +3 -6
- data/lib/aspera/assert.rb +45 -0
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formatter.rb +26 -13
- data/lib/aspera/cli/hints.rb +4 -3
- data/lib/aspera/cli/main.rb +16 -3
- data/lib/aspera/cli/manager.rb +45 -36
- data/lib/aspera/cli/plugin.rb +20 -13
- data/lib/aspera/cli/plugins/aoc.rb +103 -73
- data/lib/aspera/cli/plugins/ats.rb +4 -3
- data/lib/aspera/cli/plugins/config.rb +114 -119
- data/lib/aspera/cli/plugins/cos.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +23 -19
- data/lib/aspera/cli/plugins/faspex5.rb +75 -43
- data/lib/aspera/cli/plugins/node.rb +28 -15
- data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
- data/lib/aspera/cli/plugins/preview.rb +9 -7
- data/lib/aspera/cli/plugins/server.rb +6 -3
- data/lib/aspera/cli/plugins/shares.rb +30 -26
- data/lib/aspera/cli/sync_actions.rb +9 -9
- data/lib/aspera/cli/transfer_agent.rb +21 -14
- data/lib/aspera/cli/transfer_progress.rb +2 -3
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +13 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/coverage.rb +22 -0
- data/lib/aspera/data_repository.rb +33 -2
- data/lib/aspera/environment.rb +4 -2
- data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
- data/lib/aspera/fasp/agent_base.rb +17 -7
- data/lib/aspera/fasp/agent_direct.rb +88 -84
- data/lib/aspera/fasp/agent_httpgw.rb +4 -3
- data/lib/aspera/fasp/agent_node.rb +3 -2
- data/lib/aspera/fasp/agent_trsdk.rb +79 -37
- data/lib/aspera/fasp/installation.rb +51 -12
- data/lib/aspera/fasp/management.rb +11 -6
- data/lib/aspera/fasp/parameters.rb +53 -47
- data/lib/aspera/fasp/resume_policy.rb +7 -5
- data/lib/aspera/fasp/sync.rb +273 -0
- data/lib/aspera/fasp/transfer_spec.rb +10 -8
- data/lib/aspera/fasp/uri.rb +2 -2
- data/lib/aspera/faspex_gw.rb +11 -8
- data/lib/aspera/faspex_postproc.rb +6 -5
- data/lib/aspera/id_generator.rb +3 -1
- data/lib/aspera/json_rpc.rb +10 -8
- data/lib/aspera/keychain/encrypted_hash.rb +46 -11
- data/lib/aspera/keychain/macos_security.rb +15 -13
- data/lib/aspera/log.rb +4 -3
- data/lib/aspera/nagios.rb +7 -2
- data/lib/aspera/node.rb +17 -16
- data/lib/aspera/node_simulator.rb +214 -0
- data/lib/aspera/oauth.rb +22 -19
- data/lib/aspera/persistency_action_once.rb +13 -14
- data/lib/aspera/persistency_folder.rb +3 -2
- data/lib/aspera/preview/file_types.rb +53 -267
- data/lib/aspera/preview/generator.rb +7 -5
- data/lib/aspera/preview/terminal.rb +14 -5
- data/lib/aspera/preview/utils.rb +8 -7
- data/lib/aspera/proxy_auto_config.rb +6 -3
- data/lib/aspera/rest.rb +29 -13
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/rest_errors_aspera.rb +2 -0
- data/lib/aspera/secret_hider.rb +5 -2
- data/lib/aspera/ssh.rb +10 -8
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/web_server_simple.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +96 -45
- metadata.gz.sig +0 -0
- data/lib/aspera/sync.rb +0 -219
data/lib/aspera/sync.rb
DELETED
@@ -1,219 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# cspell:words logdir bidi watchd cooloff asyncadmin
|
4
|
-
|
5
|
-
require 'aspera/command_line_builder'
|
6
|
-
require 'aspera/fasp/installation'
|
7
|
-
require 'aspera/log'
|
8
|
-
require 'json'
|
9
|
-
require 'base64'
|
10
|
-
require 'open3'
|
11
|
-
require 'English'
|
12
|
-
|
13
|
-
module Aspera
|
14
|
-
# builds command line arg for async
|
15
|
-
module Sync
|
16
|
-
# sync direction, default is push
|
17
|
-
DIRECTIONS = %i[push pull bidi].freeze
|
18
|
-
PARAMS_VX_INSTANCE =
|
19
|
-
{
|
20
|
-
'alt_logdir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
21
|
-
'watchd' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
22
|
-
'apply_local_docroot' => { cli: { type: :opt_without_arg}},
|
23
|
-
'quiet' => { cli: { type: :opt_without_arg}},
|
24
|
-
'ws_connect' => { cli: { type: :opt_without_arg}}
|
25
|
-
}.freeze
|
26
|
-
|
27
|
-
# map sync session parameters to transfer spec: sync -> ts, true if same
|
28
|
-
PARAMS_VX_SESSION =
|
29
|
-
{
|
30
|
-
'name' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
31
|
-
'local_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
32
|
-
'remote_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
33
|
-
'local_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
34
|
-
'remote_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
35
|
-
'host' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_host},
|
36
|
-
'user' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_user},
|
37
|
-
'private_key_paths' => { cli: { type: :opt_with_arg, switch: '--private-key-path'}, accepted_types: :array},
|
38
|
-
'direction' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
39
|
-
'checksum' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
40
|
-
'tags' => { cli: { type: :opt_with_arg, switch: '--tags64', convert: 'Aspera::Fasp::Parameters.convert_json64'},
|
41
|
-
accepted_types: :hash, ts: true},
|
42
|
-
'tcp_port' => { cli: { type: :opt_with_arg}, accepted_types: :int, ts: :ssh_port},
|
43
|
-
'rate_policy' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
44
|
-
'target_rate' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
45
|
-
'cooloff' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
46
|
-
'pending_max' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
47
|
-
'scan_intensity' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
48
|
-
'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Fasp::Parameters.convert_remove_hyphen'}, accepted_types: :string, ts: true},
|
49
|
-
'transfer_threads' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
50
|
-
'preserve_time' => { cli: { type: :opt_without_arg}, ts: :preserve_times},
|
51
|
-
'preserve_access_time' => { cli: { type: :opt_without_arg}, ts: nil},
|
52
|
-
'preserve_modification_time' => { cli: { type: :opt_without_arg}, ts: nil},
|
53
|
-
'preserve_uid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_uid},
|
54
|
-
'preserve_gid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_gid},
|
55
|
-
'create_dir' => { cli: { type: :opt_without_arg}, ts: true},
|
56
|
-
'reset' => { cli: { type: :opt_without_arg}},
|
57
|
-
# NOTE: only one env var, but multiple sessions... could be a problem
|
58
|
-
'remote_password' => { cli: { type: :envvar, variable: 'ASPERA_SCP_PASS'}, ts: true},
|
59
|
-
'cookie' => { cli: { type: :envvar, variable: 'ASPERA_SCP_COOKIE'}, ts: true},
|
60
|
-
'token' => { cli: { type: :envvar, variable: 'ASPERA_SCP_TOKEN'}, ts: true},
|
61
|
-
'license' => { cli: { type: :envvar, variable: 'ASPERA_SCP_LICENSE'}}
|
62
|
-
}.freeze
|
63
|
-
|
64
|
-
Aspera::CommandLineBuilder.normalize_description(PARAMS_VX_INSTANCE)
|
65
|
-
Aspera::CommandLineBuilder.normalize_description(PARAMS_VX_SESSION)
|
66
|
-
|
67
|
-
PARAMS_VX_KEYS = %w[instance sessions].freeze
|
68
|
-
|
69
|
-
# new API
|
70
|
-
TS_TO_PARAMS_V2 = {
|
71
|
-
'remote_host' => 'remote.host',
|
72
|
-
'remote_user' => 'remote.user',
|
73
|
-
'remote_password' => 'remote.pass',
|
74
|
-
'sshfp' => 'remote.fingerprint',
|
75
|
-
'ssh_port' => 'remote.port',
|
76
|
-
'wss_port' => 'remote.ws_port',
|
77
|
-
'proxy' => 'remote.proxy',
|
78
|
-
'token' => 'remote.token',
|
79
|
-
'tags' => 'tags'
|
80
|
-
}.freeze
|
81
|
-
|
82
|
-
ASYNC_EXECUTABLE = 'async'
|
83
|
-
ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
|
84
|
-
|
85
|
-
private_constant :PARAMS_VX_INSTANCE, :PARAMS_VX_SESSION, :PARAMS_VX_KEYS, :TS_TO_PARAMS_V2, :ASYNC_EXECUTABLE, :ASYNC_ADMIN_EXECUTABLE
|
86
|
-
|
87
|
-
class << self
|
88
|
-
# Set remote_dir in sync parameters based on transfer spec
|
89
|
-
# @param params [Hash] sync parameters, old or new format
|
90
|
-
# @param remote_dir_key [String] key to update in above hash
|
91
|
-
# @param transfer_spec [Hash] transfer spec
|
92
|
-
def update_remote_dir(sync_params, remote_dir_key, transfer_spec)
|
93
|
-
if transfer_spec.dig(*%w[tags aspera node file_id])
|
94
|
-
# in AoC, use gen4
|
95
|
-
sync_params[remote_dir_key] = '/'
|
96
|
-
elsif transfer_spec['cookie']&.start_with?('aspera.shares2')
|
97
|
-
# TODO : something more generic, independent of Shares
|
98
|
-
# in Shares, the actual folder on remote end is not always the same as the name of the share
|
99
|
-
actual_remote = transfer_spec['paths']&.first&.[]('source')
|
100
|
-
sync_params[remote_dir_key] = actual_remote if actual_remote
|
101
|
-
end
|
102
|
-
nil
|
103
|
-
end
|
104
|
-
|
105
|
-
# @param sync_params [Hash] sync parameters, old or new format
|
106
|
-
# @param block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
|
107
|
-
def start(sync_params, &block)
|
108
|
-
raise 'Internal Error: sync_params parameter must be Hash' unless sync_params.is_a?(Hash)
|
109
|
-
env_args = {
|
110
|
-
args: [],
|
111
|
-
env: {}
|
112
|
-
}
|
113
|
-
if sync_params.key?('local')
|
114
|
-
# async native JSON format (v2)
|
115
|
-
raise StandardError, 'remote must be Hash' unless sync_params['remote'].is_a?(Hash)
|
116
|
-
if block
|
117
|
-
transfer_spec = yield((sync_params['direction'] || 'push').to_sym, sync_params['local']['path'], sync_params['remote']['path'])
|
118
|
-
# async native JSON format
|
119
|
-
raise StandardError, 'sync parameter "local" must be Hash' unless sync_params['local'].is_a?(Hash)
|
120
|
-
TS_TO_PARAMS_V2.each do |ts_param, sy_path|
|
121
|
-
next unless transfer_spec.key?(ts_param)
|
122
|
-
sy_dig = sy_path.split('.')
|
123
|
-
param = sy_dig.pop
|
124
|
-
hash = sy_dig.empty? ? sync_params : sync_params[sy_dig.first]
|
125
|
-
hash = sync_params[sy_dig.first] = {} if hash.nil?
|
126
|
-
hash[param] = transfer_spec[ts_param]
|
127
|
-
end
|
128
|
-
sync_params['remote']['connect_mode'] ||= sync_params['remote'].key?('ws_port') ? 'ws' : 'ssh'
|
129
|
-
sync_params['remote']['private_key_paths'] ||= Fasp::Installation.instance.aspera_token_ssh_key_paths if transfer_spec.key?('token')
|
130
|
-
update_remote_dir(sync_params['remote'], 'path', transfer_spec)
|
131
|
-
end
|
132
|
-
env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"]
|
133
|
-
elsif sync_params.key?('sessions')
|
134
|
-
# ascli JSON format (v1)
|
135
|
-
if block
|
136
|
-
sync_params['sessions'].each do |session|
|
137
|
-
transfer_spec = yield((session['direction'] || 'push').to_sym, session['local_dir'], session['remote_dir'])
|
138
|
-
PARAMS_VX_SESSION.each do |async_param, behavior|
|
139
|
-
if behavior.key?(:ts)
|
140
|
-
tspec_param = behavior[:ts].is_a?(TrueClass) ? async_param : behavior[:ts].to_s
|
141
|
-
session[async_param] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
|
142
|
-
end
|
143
|
-
end
|
144
|
-
session['private_key_paths'] = Fasp::Installation.instance.aspera_token_ssh_key_paths if transfer_spec.key?('token')
|
145
|
-
update_remote_dir(session, 'remote_dir', transfer_spec)
|
146
|
-
end
|
147
|
-
end
|
148
|
-
raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
|
149
|
-
sync_params.keys.push('instance').uniq.sort.eql?(PARAMS_VX_KEYS)
|
150
|
-
raise StandardError, 'sessions key must be Array' unless sync_params['sessions'].is_a?(Array)
|
151
|
-
raise StandardError, 'sessions key requires at least one Hash' unless sync_params['sessions'].first.is_a?(Hash)
|
152
|
-
|
153
|
-
if sync_params.key?('instance')
|
154
|
-
raise StandardError, 'instance key must be Hash' unless sync_params['instance'].is_a?(Hash)
|
155
|
-
instance_builder = Aspera::CommandLineBuilder.new(sync_params['instance'], PARAMS_VX_INSTANCE)
|
156
|
-
instance_builder.process_params
|
157
|
-
instance_builder.add_env_args(env_args)
|
158
|
-
end
|
159
|
-
|
160
|
-
sync_params['sessions'].each do |session_params|
|
161
|
-
raise StandardError, 'sessions must contain hashes' unless session_params.is_a?(Hash)
|
162
|
-
raise StandardError, 'session must contain at least name' unless session_params.key?('name')
|
163
|
-
session_builder = Aspera::CommandLineBuilder.new(session_params, PARAMS_VX_SESSION)
|
164
|
-
session_builder.process_params
|
165
|
-
session_builder.add_env_args(env_args)
|
166
|
-
end
|
167
|
-
else
|
168
|
-
raise 'At least one of `local` or `sessions` must be present in async parameters'
|
169
|
-
end
|
170
|
-
Log.log.debug{Log.dump(:sync_params, sync_params)}
|
171
|
-
|
172
|
-
Log.log.debug{"execute: #{env_args[:env].map{|k, v| "#{k}=\"#{v}\""}.join(' ')} \"#{ASYNC_EXECUTABLE}\" \"#{env_args[:args].join('" "')}\""}
|
173
|
-
res = system(env_args[:env], [ASYNC_EXECUTABLE, ASYNC_EXECUTABLE], *env_args[:args])
|
174
|
-
Log.log.debug{"result=#{res}"}
|
175
|
-
case res
|
176
|
-
when true then return nil
|
177
|
-
when false then raise "failed: #{$CHILD_STATUS}"
|
178
|
-
when nil then raise "not started: #{$CHILD_STATUS}"
|
179
|
-
else raise 'internal error: unspecified case'
|
180
|
-
end
|
181
|
-
end
|
182
|
-
|
183
|
-
def admin_status(sync_params, session_name)
|
184
|
-
command_line = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
|
185
|
-
if sync_params.key?('local')
|
186
|
-
raise 'Missing session name' if sync_params['name'].nil?
|
187
|
-
raise 'Session not found' unless session_name.nil? || session_name.eql?(sync_params['name'])
|
188
|
-
command_line.push("--name=#{sync_params['name']}")
|
189
|
-
if sync_params.key?('local_db_dir')
|
190
|
-
command_line.push("--local-db-dir=#{sync_params['local_db_dir']}")
|
191
|
-
elsif sync_params.dig('local', 'path')
|
192
|
-
command_line.push("--local-dir=#{sync_params.dig('local', 'path')}")
|
193
|
-
else
|
194
|
-
raise 'Missing either local_db_dir or local.path'
|
195
|
-
end
|
196
|
-
elsif sync_params.key?('sessions')
|
197
|
-
session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{|s|s['name'].eql?(session_name)}
|
198
|
-
raise "Session #{session_name} not found in #{sync_params['sessions'].map{|s|s['name']}.join(',')}" if session.nil?
|
199
|
-
raise 'Missing session name' if session['name'].nil?
|
200
|
-
command_line.push("--name=#{session['name']}")
|
201
|
-
if session.key?('local_db_dir')
|
202
|
-
command_line.push("--local-db-dir=#{session['local_db_dir']}")
|
203
|
-
elsif session.key?('local_dir')
|
204
|
-
command_line.push("--local-dir=#{session['local_dir']}")
|
205
|
-
else
|
206
|
-
raise 'Missing either local_db_dir or local_dir'
|
207
|
-
end
|
208
|
-
else
|
209
|
-
raise 'At least one of `local` or `sessions` must be present in async parameters'
|
210
|
-
end
|
211
|
-
Log.log.debug{"execute: #{command_line.join(' ')}"}
|
212
|
-
stdout, stderr, status = Open3.capture3(*command_line)
|
213
|
-
Log.log.debug{"status=#{status}, stderr=#{stderr}"}
|
214
|
-
raise "Sync failed: #{status.exitstatus} : #{stderr}" unless status.success?
|
215
|
-
return stdout.split("\n").each_with_object({}){|l, m|i = l.split(':', 2); m[i.first.lstrip] = i.last.lstrip} # rubocop:disable Style/Semicolon
|
216
|
-
end
|
217
|
-
end
|
218
|
-
end # end Sync
|
219
|
-
end # end Aspera
|