aspera-cli 4.21.2 → 4.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +1 -1
- data/CHANGELOG.md +402 -374
- data/CONTRIBUTING.md +6 -10
- data/README.md +1018 -687
- data/lib/aspera/agent/base.rb +9 -5
- data/lib/aspera/agent/connect.rb +30 -28
- data/lib/aspera/agent/desktop.rb +29 -25
- data/lib/aspera/agent/direct.rb +137 -125
- data/lib/aspera/agent/httpgw.rb +22 -26
- data/lib/aspera/agent/node.rb +14 -11
- data/lib/aspera/agent/transferd.rb +6 -2
- data/lib/aspera/api/aoc.rb +15 -18
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/httpgw.rb +15 -7
- data/lib/aspera/api/node.rb +6 -4
- data/lib/aspera/ascmd.rb +17 -9
- data/lib/aspera/ascp/installation.rb +21 -19
- data/lib/aspera/ascp/management.rb +1 -1
- data/lib/aspera/assert.rb +14 -5
- data/lib/aspera/cli/error.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +38 -19
- data/lib/aspera/cli/formatter.rb +48 -48
- data/lib/aspera/cli/hints.rb +10 -2
- data/lib/aspera/cli/main.rb +190 -168
- data/lib/aspera/cli/manager.rb +16 -16
- data/lib/aspera/cli/plugin.rb +24 -21
- data/lib/aspera/cli/plugin_factory.rb +1 -1
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +173 -126
- data/lib/aspera/cli/plugins/ats.rb +19 -17
- data/lib/aspera/cli/plugins/config.rb +87 -98
- data/lib/aspera/cli/plugins/console.rb +5 -3
- data/lib/aspera/cli/plugins/faspex.rb +39 -35
- data/lib/aspera/cli/plugins/faspex5.rb +104 -80
- data/lib/aspera/cli/plugins/faspio.rb +13 -1
- data/lib/aspera/cli/plugins/httpgw.rb +13 -1
- data/lib/aspera/cli/plugins/node.rb +336 -205
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +7 -6
- data/lib/aspera/cli/plugins/shares.rb +5 -5
- data/lib/aspera/cli/sync_actions.rb +19 -18
- data/lib/aspera/cli/transfer_agent.rb +11 -15
- data/lib/aspera/cli/transfer_progress.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +116 -95
- data/lib/aspera/coverage.rb +4 -3
- data/lib/aspera/data_repository.rb +1 -0
- data/lib/aspera/environment.rb +7 -6
- data/lib/aspera/faspex_gw.rb +14 -14
- data/lib/aspera/faspex_postproc.rb +7 -6
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +47 -34
- data/lib/aspera/keychain/factory.rb +41 -0
- data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
- data/lib/aspera/keychain/macos_security.rb +19 -11
- data/lib/aspera/log.rb +29 -34
- data/lib/aspera/nagios.rb +6 -6
- data/lib/aspera/node_simulator.rb +8 -8
- data/lib/aspera/oauth/base.rb +10 -6
- data/lib/aspera/oauth/factory.rb +6 -6
- data/lib/aspera/oauth/url_json.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +6 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +40 -33
- data/lib/aspera/preview/generator.rb +1 -1
- data/lib/aspera/preview/options.rb +16 -16
- data/lib/aspera/preview/terminal.rb +3 -3
- data/lib/aspera/preview/utils.rb +11 -13
- data/lib/aspera/products/connect.rb +2 -1
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +2 -2
- data/lib/aspera/rest.rb +70 -50
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/secret_hider.rb +5 -5
- data/lib/aspera/ssh.rb +5 -5
- data/lib/aspera/temp_file_manager.rb +1 -0
- data/lib/aspera/timer_limiter.rb +7 -5
- data/lib/aspera/transfer/async_conf.schema.yaml +716 -0
- data/lib/aspera/transfer/convert.rb +29 -0
- data/lib/aspera/transfer/error_info.rb +66 -66
- data/lib/aspera/transfer/parameters.rb +13 -68
- data/lib/aspera/transfer/spec.rb +5 -6
- data/lib/aspera/transfer/spec.schema.yaml +753 -0
- data/lib/aspera/transfer/spec_doc.rb +62 -0
- data/lib/aspera/transfer/sync.rb +37 -76
- data/lib/aspera/transfer/sync_instance.schema.yaml +20 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +86 -0
- data/lib/aspera/transfer/uri.rb +6 -6
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +53 -44
- data.tar.gz.sig +0 -0
- metadata +38 -7
- metadata.gz.sig +0 -0
- data/examples/build_package.sh +0 -28
- data/examples/dascli +0 -30
- data/examples/get_proto_file.rb +0 -8
- data/examples/proxy.pac +0 -60
- data/lib/aspera/transfer/spec.yaml +0 -718
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/agent/base'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
module Transfer
|
7
|
+
# translate transfer specification to ascp parameter list
|
8
|
+
class SpecDoc
|
9
|
+
class << self
|
10
|
+
# first letter of agent name symbol
|
11
|
+
def agent_to_short(agent_sym)
|
12
|
+
agent_sym.to_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
|
13
|
+
end
|
14
|
+
|
15
|
+
# @columns formatter [Cli::Formatter] formatter to use, methods: special_format, check_row
|
16
|
+
# @columns &block modify parameter info if needed
|
17
|
+
# @return a table suitable to display in manual
|
18
|
+
def man_table(formatter, cli: true)
|
19
|
+
col_local = agent_to_short(:direct)
|
20
|
+
Spec::SCHEMA['properties'].filter_map do |name, properties|
|
21
|
+
# manual table
|
22
|
+
columns = {
|
23
|
+
name: name,
|
24
|
+
type: properties['type'],
|
25
|
+
description: []
|
26
|
+
}
|
27
|
+
# replace "back solidus" HTML entity with its text value and split lines
|
28
|
+
columns[:description].concat(properties['description'].gsub('\', '\\').split("\n")) if properties.key?('description')
|
29
|
+
columns[:description].unshift("DEPRECATED: #{properties['x-deprecation']}") if properties.key?('x-deprecation')
|
30
|
+
# add flags for supported agents in doc
|
31
|
+
AGENT_LIST.each do |agent_info|
|
32
|
+
columns[agent_info.last] = Cli::Formatter.tick(properties['x-agents'].nil? || properties['x-agents'].include?(agent_info.first.to_s))
|
33
|
+
end
|
34
|
+
columns[col_local] = Cli::Formatter.tick(true) if properties['x-cli-option']
|
35
|
+
# only keep lines that are usable in supported agents
|
36
|
+
next false if AGENT_LIST.map(&:last).inject(true){ |memory, agent_short_sym| memory && columns[agent_short_sym].empty?}
|
37
|
+
columns[:description].push("Allowed values: #{properties['enum'].join(', ')}") if properties.key?('enum')
|
38
|
+
cli_option =
|
39
|
+
if properties['x-cli-switch']
|
40
|
+
properties['x-cli-option']
|
41
|
+
elsif properties['x-cli-special']
|
42
|
+
formatter.special_format('special')
|
43
|
+
elsif properties['x-cli-option']
|
44
|
+
arg_type = properties.key?('enum') ? '{enum}' : "{#{[properties['type']].flatten.join('|')}}"
|
45
|
+
conversion_tag = properties.key?('x-cli-convert') ? '(conversion)' : ''
|
46
|
+
sep = properties['x-cli-option'].start_with?('--') ? '=' : ' '
|
47
|
+
"#{properties['x-cli-option']}#{sep}#{conversion_tag}#{arg_type}"
|
48
|
+
end
|
49
|
+
cli_option = 'env:' + properties['x-cli-envvar'] if properties.key?('x-cli-envvar')
|
50
|
+
columns[:description].push("(#{cli_option})") if cli && !cli_option.to_s.empty?
|
51
|
+
formatter.check_row(columns)
|
52
|
+
end.sort_by{ |i| i[:name]}
|
53
|
+
end
|
54
|
+
end
|
55
|
+
# Agents shown in manual for parameters (sub list)
|
56
|
+
AGENT_LIST = Agent::Base.agent_list.map do |agent_sym|
|
57
|
+
[agent_sym, agent_sym.to_s.capitalize, agent_to_short(agent_sym)]
|
58
|
+
end.sort_by(&:last).freeze
|
59
|
+
TABLE_COLUMNS = (%i[name type] + AGENT_LIST.map(&:last) + %i[description]).freeze
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
data/lib/aspera/transfer/sync.rb
CHANGED
@@ -5,6 +5,7 @@
|
|
5
5
|
require 'aspera/command_line_builder'
|
6
6
|
require 'aspera/ascp/installation'
|
7
7
|
require 'aspera/agent/direct'
|
8
|
+
require 'aspera/transfer/convert'
|
8
9
|
require 'aspera/log'
|
9
10
|
require 'aspera/assert'
|
10
11
|
require 'json'
|
@@ -16,62 +17,18 @@ module Aspera
|
|
16
17
|
module Transfer
|
17
18
|
# builds command line arg for async and execute it
|
18
19
|
module Sync
|
19
|
-
# sync direction
|
20
|
+
# sync direction
|
20
21
|
DIRECTIONS = %i[push pull bidi].freeze
|
22
|
+
# default direction for sync
|
23
|
+
DEFAULT_DIRECTION = :push
|
21
24
|
# JSON for async instance command line options
|
22
|
-
|
23
|
-
|
24
|
-
'alt_logdir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
25
|
-
'watchd' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
26
|
-
'apply_local_docroot' => { cli: { type: :opt_without_arg}},
|
27
|
-
'quiet' => { cli: { type: :opt_without_arg}},
|
28
|
-
'ws_connect' => { cli: { type: :opt_without_arg}}
|
29
|
-
}.freeze
|
30
|
-
|
31
|
-
# map sync session parameters to transfer spec: sync -> ts, true if same
|
32
|
-
CMDLINE_PARAMS_SESSION =
|
33
|
-
{
|
34
|
-
'name' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
35
|
-
'local_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
36
|
-
'remote_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
37
|
-
'local_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
38
|
-
'remote_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
39
|
-
'host' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_host},
|
40
|
-
'user' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_user},
|
41
|
-
'private_key_paths' => { cli: { type: :opt_with_arg, switch: '--private-key-path'}, accepted_types: :array},
|
42
|
-
'direction' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
43
|
-
'checksum' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
44
|
-
'tags' => { cli: { type: :opt_with_arg, switch: '--tags64', convert: 'Aspera::Transfer::Parameters.convert_json64'},
|
45
|
-
accepted_types: :hash, ts: true},
|
46
|
-
'tcp_port' => { cli: { type: :opt_with_arg}, accepted_types: :int, ts: :ssh_port},
|
47
|
-
'rate_policy' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
48
|
-
'target_rate' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
49
|
-
'cooloff' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
50
|
-
'pending_max' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
51
|
-
'scan_intensity' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
52
|
-
'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Transfer::Parameters.convert_remove_hyphen'},
|
53
|
-
accepted_types: :string, ts: true},
|
54
|
-
'transfer_threads' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
55
|
-
'preserve_time' => { cli: { type: :opt_without_arg}, ts: :preserve_times},
|
56
|
-
'preserve_access_time' => { cli: { type: :opt_without_arg}, ts: nil},
|
57
|
-
'preserve_modification_time' => { cli: { type: :opt_without_arg}, ts: nil},
|
58
|
-
'preserve_uid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_uid},
|
59
|
-
'preserve_gid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_gid},
|
60
|
-
'create_dir' => { cli: { type: :opt_without_arg}, ts: true},
|
61
|
-
'reset' => { cli: { type: :opt_without_arg}},
|
62
|
-
# NOTE: only one env var, but multiple sessions... could be a problem
|
63
|
-
'remote_password' => { cli: { type: :envvar, variable: 'ASPERA_SCP_PASS'}, ts: true},
|
64
|
-
'cookie' => { cli: { type: :envvar, variable: 'ASPERA_SCP_COOKIE'}, ts: true},
|
65
|
-
'token' => { cli: { type: :envvar, variable: 'ASPERA_SCP_TOKEN'}, ts: true},
|
66
|
-
'license' => { cli: { type: :envvar, variable: 'ASPERA_SCP_LICENSE'}}
|
67
|
-
}.freeze
|
68
|
-
|
69
|
-
CommandLineBuilder.normalize_description(CMDLINE_PARAMS_INSTANCE)
|
70
|
-
CommandLineBuilder.normalize_description(CMDLINE_PARAMS_SESSION)
|
25
|
+
INSTANCE_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'instance')
|
26
|
+
SESSION_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'session')
|
71
27
|
|
72
28
|
CMDLINE_PARAMS_KEYS = %w[instance sessions].freeze
|
73
29
|
|
74
30
|
# Translation of transfer spec parameters to async v2 API (asyncs)
|
31
|
+
# TODO: complete this list with all transfer spec parameters or include in async json schema with x- parameters
|
75
32
|
TSPEC_TO_ASYNC_CONF = {
|
76
33
|
'remote_host' => 'remote.host',
|
77
34
|
'remote_user' => 'remote.user',
|
@@ -86,10 +43,10 @@ module Aspera
|
|
86
43
|
|
87
44
|
ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
|
88
45
|
|
89
|
-
private_constant :
|
46
|
+
private_constant :INSTANCE_SCHEMA, :SESSION_SCHEMA, :CMDLINE_PARAMS_KEYS, :TSPEC_TO_ASYNC_CONF, :ASYNC_ADMIN_EXECUTABLE
|
90
47
|
|
91
48
|
class << self
|
92
|
-
# Set remote_dir in sync parameters based on transfer spec
|
49
|
+
# Set `remote_dir` in sync parameters based on transfer spec
|
93
50
|
# @param params [Hash] sync parameters, old or new format
|
94
51
|
# @param remote_dir_key [String] key to update in above hash
|
95
52
|
# @param transfer_spec [Hash] transfer spec
|
@@ -100,7 +57,8 @@ module Aspera
|
|
100
57
|
elsif transfer_spec['cookie']&.start_with?('aspera.shares2')
|
101
58
|
# TODO : something more generic, independent of Shares
|
102
59
|
# in Shares, the actual folder on remote end is not always the same as the name of the share
|
103
|
-
|
60
|
+
remote_key = transfer_spec['direction'].eql?('send') ? 'destination' : 'source'
|
61
|
+
actual_remote = transfer_spec['paths']&.first&.[](remote_key)
|
104
62
|
sync_params[remote_dir_key] = actual_remote if actual_remote
|
105
63
|
end
|
106
64
|
nil
|
@@ -132,14 +90,19 @@ module Aspera
|
|
132
90
|
return certificates_to_use
|
133
91
|
end
|
134
92
|
|
93
|
+
# Get symbol of sync direction, defaulting to :push
|
94
|
+
# @param params [Hash] sync parameters, old or new format
|
95
|
+
# @return [Symbol] direction symbol, one of :push, :pull, :bidi
|
96
|
+
def direction_sym(params)
|
97
|
+
(params['direction'] || DEFAULT_DIRECTION).to_sym
|
98
|
+
end
|
99
|
+
|
135
100
|
# @param sync_params [Hash] sync parameters, old or new format
|
136
|
-
# @param block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
|
137
|
-
def start(
|
138
|
-
sync_params,
|
139
|
-
&block
|
140
|
-
)
|
101
|
+
# @param &block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
|
102
|
+
def start(sync_params)
|
141
103
|
Log.log.debug{Log.dump(:sync_params_initial, sync_params)}
|
142
104
|
Aspera.assert_type(sync_params, Hash)
|
105
|
+
Aspera.assert(%w[local sessions].any?{ |k| sync_params.key?(k)}){'At least one of `local` or `sessions` must be present in async parameters'}
|
143
106
|
env_args = {
|
144
107
|
args: [],
|
145
108
|
env: {}
|
@@ -151,8 +114,8 @@ module Aspera
|
|
151
114
|
Aspera.assert_type(remote, Hash){'remote'}
|
152
115
|
Aspera.assert_type(remote['path'], String){'remote path'}
|
153
116
|
# get transfer spec if possible, and feed back to new structure
|
154
|
-
if
|
155
|
-
transfer_spec = yield((sync_params
|
117
|
+
if block_given?
|
118
|
+
transfer_spec = yield(direction_sym(sync_params), sync_params['local']['path'], remote['path'])
|
156
119
|
# translate transfer spec to async parameters
|
157
120
|
TSPEC_TO_ASYNC_CONF.each do |ts_param, sy_path|
|
158
121
|
next unless transfer_spec.key?(ts_param)
|
@@ -172,24 +135,25 @@ module Aspera
|
|
172
135
|
end
|
173
136
|
# '--exclusive-mgmt-port=12345', '--arg-err-path=-',
|
174
137
|
env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"]
|
175
|
-
Log.log.debug{Log.dump(:
|
138
|
+
Log.log.debug{Log.dump(:sync_conf, sync_params)}
|
176
139
|
agent = Agent::Direct.new
|
177
140
|
agent.start_and_monitor_process(session: {}, name: :async, **env_args)
|
178
|
-
|
141
|
+
else
|
142
|
+
# key 'sessions' is present
|
179
143
|
# ascli JSON format (cmdline)
|
180
144
|
raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
|
181
145
|
sync_params.keys.push('instance').uniq.sort.eql?(CMDLINE_PARAMS_KEYS)
|
182
146
|
Aspera.assert_type(sync_params['sessions'], Array)
|
183
147
|
Aspera.assert_type(sync_params['sessions'].first, Hash)
|
184
|
-
if
|
148
|
+
if block_given?
|
185
149
|
sync_params['sessions'].each do |session|
|
186
150
|
Aspera.assert_type(session['local_dir'], String){'local_dir'}
|
187
151
|
Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
|
188
|
-
transfer_spec = yield((session
|
189
|
-
|
190
|
-
if
|
191
|
-
tspec_param =
|
192
|
-
session[
|
152
|
+
transfer_spec = yield(direction_sym(session), session['local_dir'], session['remote_dir'])
|
153
|
+
SESSION_SCHEMA['properties'].each do |name, properties|
|
154
|
+
if properties.key?('x-tspec')
|
155
|
+
tspec_param = properties['x-tspec'].is_a?(TrueClass) ? name : properties['x-tspec'].to_s
|
156
|
+
session[name] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
|
193
157
|
end
|
194
158
|
end
|
195
159
|
session['private_key_paths'] = Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa) if transfer_spec.key?('token')
|
@@ -198,21 +162,18 @@ module Aspera
|
|
198
162
|
end
|
199
163
|
if sync_params.key?('instance')
|
200
164
|
Aspera.assert_type(sync_params['instance'], Hash)
|
201
|
-
instance_builder = CommandLineBuilder.new(sync_params['instance'],
|
165
|
+
instance_builder = CommandLineBuilder.new(sync_params['instance'], INSTANCE_SCHEMA, Convert)
|
202
166
|
instance_builder.process_params
|
203
167
|
instance_builder.add_env_args(env_args)
|
204
168
|
end
|
205
|
-
|
206
169
|
sync_params['sessions'].each do |session_params|
|
207
170
|
Aspera.assert_type(session_params, Hash)
|
208
|
-
Aspera.assert(session_params.key?('name')){'session must contain at least name'}
|
209
|
-
session_builder = CommandLineBuilder.new(session_params,
|
171
|
+
Aspera.assert(session_params.key?('name')){'session must contain at least: name'}
|
172
|
+
session_builder = CommandLineBuilder.new(session_params, SESSION_SCHEMA, Convert)
|
210
173
|
session_builder.process_params
|
211
174
|
session_builder.add_env_args(env_args)
|
212
175
|
end
|
213
176
|
Environment.secure_execute(exec: Ascp::Installation.instance.path(:async), **env_args)
|
214
|
-
else
|
215
|
-
raise 'At least one of `local` or `sessions` must be present in async parameters'
|
216
177
|
end
|
217
178
|
return nil
|
218
179
|
end
|
@@ -248,8 +209,8 @@ module Aspera
|
|
248
209
|
raise 'Missing either local_db_dir or local.path'
|
249
210
|
end
|
250
211
|
elsif sync_params.key?('sessions')
|
251
|
-
session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{|s|s['name'].eql?(session_name)}
|
252
|
-
raise "Session #{session_name} not found in #{sync_params['sessions'].map{|s|s['name']}.join(',')}" if session.nil?
|
212
|
+
session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{ |s| s['name'].eql?(session_name)}
|
213
|
+
raise "Session #{session_name} not found in #{sync_params['sessions'].map{ |s| s['name']}.join(',')}" if session.nil?
|
253
214
|
raise 'Missing session name' if session['name'].nil?
|
254
215
|
arguments.push("--name=#{session['name']}")
|
255
216
|
if session.key?('local_db_dir')
|
@@ -0,0 +1,20 @@
|
|
1
|
+
$schema: https://json-schema.org/draft/2020-12/schema
|
2
|
+
$id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/transfer/sync_instance.schema.yaml
|
3
|
+
$comment: >-
|
4
|
+
`x-` fields documented in command_line_builder.rb
|
5
|
+
This spec is specific to ascli.
|
6
|
+
The native async conf format is now preferred.
|
7
|
+
title: SyncInstanceSpec
|
8
|
+
description: Global parameters for async.
|
9
|
+
type: object
|
10
|
+
properties:
|
11
|
+
alt_logdir:
|
12
|
+
type: string
|
13
|
+
watchd:
|
14
|
+
type: string
|
15
|
+
apply_local_docroot:
|
16
|
+
x-cli-switch: true
|
17
|
+
quiet:
|
18
|
+
x-cli-switch: true
|
19
|
+
ws_connect:
|
20
|
+
x-cli-switch: true
|
@@ -0,0 +1,86 @@
|
|
1
|
+
$schema: https://json-schema.org/draft/2020-12/schema
|
2
|
+
$id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/transfer/sync_instance.schema.yaml
|
3
|
+
$comment: >-
|
4
|
+
`x-` fields documented in command_line_builder.rb
|
5
|
+
This spec is specific to ascli.
|
6
|
+
The native async conf format is now preferred.
|
7
|
+
title: SyncSessionSpec
|
8
|
+
description: Sync session parameters for async.
|
9
|
+
type: object
|
10
|
+
properties:
|
11
|
+
name:
|
12
|
+
type: string
|
13
|
+
local_dir:
|
14
|
+
type: string
|
15
|
+
remote_dir:
|
16
|
+
type: string
|
17
|
+
local_db_dir:
|
18
|
+
type: string
|
19
|
+
remote_db_dir:
|
20
|
+
type: string
|
21
|
+
host:
|
22
|
+
type: string
|
23
|
+
x-tspec: remote_host
|
24
|
+
user:
|
25
|
+
type: string
|
26
|
+
x-tspec: remote_user
|
27
|
+
private_key_paths:
|
28
|
+
type: array
|
29
|
+
x-cli-option: "--private-key-path"
|
30
|
+
direction:
|
31
|
+
type: string
|
32
|
+
checksum:
|
33
|
+
type: string
|
34
|
+
tags:
|
35
|
+
type: object
|
36
|
+
x-cli-option: "--tags64"
|
37
|
+
x-cli-convert: json64
|
38
|
+
x-tspec: true
|
39
|
+
tcp_port:
|
40
|
+
type: integer
|
41
|
+
x-tspec: ssh_port
|
42
|
+
rate_policy:
|
43
|
+
type: string
|
44
|
+
target_rate:
|
45
|
+
type: string
|
46
|
+
cooloff:
|
47
|
+
type: integer
|
48
|
+
pending_max:
|
49
|
+
type: integer
|
50
|
+
scan_intensity:
|
51
|
+
type: string
|
52
|
+
cipher:
|
53
|
+
type: string
|
54
|
+
x-cli-convert: remove_hyphen
|
55
|
+
x-tspec: true
|
56
|
+
transfer_threads:
|
57
|
+
type: integer
|
58
|
+
preserve_time:
|
59
|
+
x-cli-switch: true
|
60
|
+
x-tspec: preserve_times
|
61
|
+
preserve_access_time:
|
62
|
+
x-cli-switch: true
|
63
|
+
preserve_modification_time:
|
64
|
+
x-cli-switch: true
|
65
|
+
preserve_uid:
|
66
|
+
x-cli-switch: true
|
67
|
+
x-tspec: preserve_file_owner_uid
|
68
|
+
preserve_gid:
|
69
|
+
x-cli-switch: true
|
70
|
+
x-tspec: preserve_file_owner_gid
|
71
|
+
create_dir:
|
72
|
+
x-cli-switch: true
|
73
|
+
x-tspec: true
|
74
|
+
reset:
|
75
|
+
x-cli-switch: true
|
76
|
+
remote_password:
|
77
|
+
x-cli-envvar: ASPERA_SCP_PASS
|
78
|
+
x-tspec: true
|
79
|
+
cookie:
|
80
|
+
x-cli-envvar: ASPERA_SCP_COOKIE
|
81
|
+
x-tspec: true
|
82
|
+
token:
|
83
|
+
x-cli-envvar: ASPERA_SCP_TOKEN
|
84
|
+
x-tspec: true
|
85
|
+
license:
|
86
|
+
x-cli-envvar: ASPERA_SCP_LICENSE
|
data/lib/aspera/transfer/uri.rb
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
require 'aspera/log'
|
6
6
|
require 'aspera/rest'
|
7
|
-
require 'aspera/
|
7
|
+
require 'aspera/transfer/convert'
|
8
8
|
|
9
9
|
module Aspera
|
10
10
|
module Transfer
|
@@ -24,7 +24,7 @@ module Aspera
|
|
24
24
|
result_ts['ssh_port'] = @fasp_uri.port
|
25
25
|
result_ts['paths'] = [{'source' => URI.decode_www_form_component(@fasp_uri.path)}]
|
26
26
|
# faspex 4 does not encode trailing base64 padding, fix that to be able to decode properly
|
27
|
-
fixed_query = @fasp_uri.query.gsub(/(=+)$/){|trail_equals|'%3D' * trail_equals.length}
|
27
|
+
fixed_query = @fasp_uri.query.gsub(/(=+)$/){ |trail_equals| '%3D' * trail_equals.length}
|
28
28
|
|
29
29
|
Rest.query_to_h(fixed_query).each do |name, value|
|
30
30
|
case name
|
@@ -39,10 +39,10 @@ module Aspera
|
|
39
39
|
when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
|
40
40
|
when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub('--', '-')
|
41
41
|
when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
|
42
|
-
when 'createpath' then result_ts['create_dir'] =
|
43
|
-
when 'fallback' then result_ts['http_fallback'] =
|
44
|
-
when 'lockpolicy' then result_ts['lock_rate_policy'] =
|
45
|
-
when 'lockminrate' then result_ts['lock_min_rate'] =
|
42
|
+
when 'createpath' then result_ts['create_dir'] = Convert.yes_to_true(value)
|
43
|
+
when 'fallback' then result_ts['http_fallback'] = Convert.yes_to_true(value)
|
44
|
+
when 'lockpolicy' then result_ts['lock_rate_policy'] = Convert.yes_to_true(value)
|
45
|
+
when 'lockminrate' then result_ts['lock_min_rate'] = Convert.yes_to_true(value)
|
46
46
|
when 'auth' then Log.log.debug{"ignoring #{name}=#{value}"} # Not used (yes/no)
|
47
47
|
when 'v' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch -- Not used (shall be 2)
|
48
48
|
when 'protect' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch -- TODO: what is this ?
|
data/lib/aspera/uri_reader.rb
CHANGED
@@ -22,7 +22,7 @@ module Aspera
|
|
22
22
|
raise 'URL shall have a path, check syntax' if local_file_path.nil?
|
23
23
|
local_file_path = File.expand_path(local_file_path.gsub(%r{^/}, '')) if %r{^/(~|.|..)/}.match?(local_file_path)
|
24
24
|
return File.read(local_file_path)
|
25
|
-
else Aspera.error_unexpected_value(uri.scheme)
|
25
|
+
else Aspera.error_unexpected_value(uri.scheme){"scheme for [#{uri_to_read}]"}
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
data/lib/aspera/web_auth.rb
CHANGED
@@ -182,7 +182,7 @@ module Aspera
|
|
182
182
|
# last argument (self) is provided to constructor of servlet
|
183
183
|
mount(@expected_path, WebAuthServlet, self)
|
184
184
|
# server runs in thread
|
185
|
-
Thread.new
|
185
|
+
Thread.new{start}
|
186
186
|
end
|
187
187
|
|
188
188
|
# Called by web server thread on received request
|
@@ -10,19 +10,23 @@ require 'openssl'
|
|
10
10
|
module Aspera
|
11
11
|
# Simple WEBrick server with HTTPS support
|
12
12
|
class WebServerSimple < WEBrick::HTTPServer
|
13
|
-
|
13
|
+
PARAMS = %i[cert key chain].freeze
|
14
|
+
DEFAULT_URL = 'http://localhost:8080'
|
14
15
|
GENERIC_ISSUER = '/C=FR/O=Test/OU=Test/CN=Test'
|
15
16
|
ONE_YEAR_SECONDS = 365 * 24 * 60 * 60
|
17
|
+
PKCS12_EXT = %w[p12 pfx].map{ |i| ".#{i}"}.freeze
|
18
|
+
CLOCK_SKEW_OFFSET_SEC = 5
|
16
19
|
|
17
|
-
private_constant :
|
20
|
+
private_constant :GENERIC_ISSUER, :ONE_YEAR_SECONDS, :PKCS12_EXT, :CLOCK_SKEW_OFFSET_SEC
|
18
21
|
|
19
22
|
class << self
|
20
|
-
#
|
21
|
-
def
|
23
|
+
# Generate or fill and self sign certificate
|
24
|
+
def self_signed_cert(private_key, digest: 'SHA256')
|
25
|
+
cert = OpenSSL::X509::Certificate.new
|
22
26
|
cert.subject = cert.issuer = OpenSSL::X509::Name.parse(GENERIC_ISSUER)
|
23
|
-
cert.not_before =
|
24
|
-
cert.not_after
|
25
|
-
cert.public_key =
|
27
|
+
cert.not_before = Time.now - CLOCK_SKEW_OFFSET_SEC
|
28
|
+
cert.not_after = cert.not_before + ONE_YEAR_SECONDS
|
29
|
+
cert.public_key = private_key.public_key
|
26
30
|
cert.serial = 0x0
|
27
31
|
cert.version = 2
|
28
32
|
ef = OpenSSL::X509::ExtensionFactory.new
|
@@ -34,68 +38,73 @@ module Aspera
|
|
34
38
|
# ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
|
35
39
|
]
|
36
40
|
cert.add_extension(ef.create_extension('authorityKeyIdentifier', 'keyid:always,issuer:always'))
|
37
|
-
cert.sign(
|
41
|
+
cert.sign(private_key, OpenSSL::Digest.new(digest))
|
42
|
+
cert
|
43
|
+
end
|
44
|
+
|
45
|
+
# @return a list of Certificates from chain file
|
46
|
+
def read_chain_file(chain)
|
47
|
+
File.read(chain).scan(/-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----/m).map{ |i| OpenSSL::X509::Certificate.new(i)}
|
38
48
|
end
|
39
49
|
end
|
40
50
|
|
41
|
-
# @param
|
42
|
-
|
43
|
-
|
51
|
+
# @param url [URI] Local address where server will listen (use scheme, host and port only)
|
52
|
+
# @param cert [String] Path to certificate file, either with extension .p12 or .pfx, else assumed PEM
|
53
|
+
# @param key [String] Path to key file (PEM) or passphrase (pkcs12)
|
54
|
+
# @param chain [String] Path to certificate chain file (PEM only)
|
55
|
+
def initialize(uri, cert: nil, key: nil, chain: nil)
|
56
|
+
Aspera.assert_type(uri, URI)
|
57
|
+
@uri = uri
|
44
58
|
# see https://www.rubydoc.info/stdlib/webrick/WEBrick/Config
|
45
59
|
webrick_options = {
|
46
|
-
BindAddress: uri.host,
|
47
|
-
Port: uri.port,
|
60
|
+
BindAddress: @uri.host,
|
61
|
+
Port: @uri.port,
|
48
62
|
Logger: Log.log,
|
49
63
|
AccessLog: [[self, WEBrick::AccessLog::COMMON_LOG_FORMAT]] # replace default access log to call local method "<<" below
|
50
64
|
}
|
51
|
-
case uri.scheme
|
65
|
+
case @uri.scheme
|
52
66
|
when 'http'
|
53
67
|
Log.log.debug('HTTP mode')
|
54
68
|
when 'https'
|
55
69
|
webrick_options[:SSLEnable] = true
|
56
|
-
if
|
70
|
+
if cert.nil? && key.nil?
|
57
71
|
webrick_options[:SSLCertName] = [['CN', WEBrick::Utils.getservername]]
|
72
|
+
elsif cert && PKCS12_EXT.include?(File.extname(cert).downcase)
|
73
|
+
# PKCS12
|
74
|
+
Log.log.debug('Using PKCS12 certificate')
|
75
|
+
raise 'PKCS12 requires a key (password)' if key.nil?
|
76
|
+
pkcs12 = OpenSSL::PKCS12.new(File.read(cert), key)
|
77
|
+
webrick_options[:SSLCertificate] = pkcs12.certificate
|
78
|
+
webrick_options[:SSLPrivateKey] = pkcs12.key
|
79
|
+
webrick_options[:SSLExtraChainCert] = pkcs12.ca_certs
|
58
80
|
else
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
webrick_options[:
|
67
|
-
webrick_options[:SSLPrivateKey] = pkcs12.key
|
68
|
-
webrick_options[:SSLExtraChainCert] = pkcs12.ca_certs
|
81
|
+
Log.log.debug('Using PEM certificate')
|
82
|
+
webrick_options[:SSLPrivateKey] = if key.nil?
|
83
|
+
OpenSSL::PKey::RSA.new(4096)
|
84
|
+
else
|
85
|
+
OpenSSL::PKey::RSA.new(File.read(key))
|
86
|
+
end
|
87
|
+
webrick_options[:SSLCertificate] = if cert.nil?
|
88
|
+
self.class.self_signed_cert(webrick_options[:SSLPrivateKey])
|
69
89
|
else
|
70
|
-
|
71
|
-
webrick_options[:SSLPrivateKey] = if certificate.key?(:key)
|
72
|
-
OpenSSL::PKey::RSA.new(File.read(certificate[:key]))
|
73
|
-
else
|
74
|
-
OpenSSL::PKey::RSA.new(4096)
|
75
|
-
end
|
76
|
-
if certificate.key?(:cert)
|
77
|
-
webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read(certificate[:cert]))
|
78
|
-
else
|
79
|
-
webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new
|
80
|
-
self.class.fill_self_signed_cert(webrick_options[:SSLCertificate], webrick_options[:SSLPrivateKey])
|
81
|
-
end
|
82
|
-
if certificate.key?(:chain)
|
83
|
-
webrick_options[:SSLExtraChainCert] = [OpenSSL::X509::Certificate.new(File.read(certificate[:chain]))]
|
84
|
-
end
|
90
|
+
OpenSSL::X509::Certificate.new(File.read(cert))
|
85
91
|
end
|
92
|
+
webrick_options[:SSLExtraChainCert] = read_chain_file(chain) unless chain.nil?
|
93
|
+
raise 'key and cert do not match' unless webrick_options[:SSLCertificate].public_key.to_der == webrick_options[:SSLPrivateKey].public_key.to_der
|
86
94
|
end
|
87
95
|
end
|
88
96
|
# call constructor of parent class, but capture STDERR
|
89
97
|
# self signed certificate generates characters on STDERR
|
90
98
|
# see create_self_signed_cert in webrick/ssl.rb
|
91
|
-
Log.capture_stderr
|
99
|
+
Log.capture_stderr{super(webrick_options)}
|
92
100
|
end
|
93
101
|
|
94
102
|
# blocking
|
95
103
|
def start
|
96
|
-
Log.log.info{"Listening on #{@
|
97
|
-
# kill -
|
98
|
-
|
104
|
+
Log.log.info{"Listening on #{@uri}"}
|
105
|
+
# kill (-TERM) for graceful shutdown
|
106
|
+
handler = proc{shutdown}
|
107
|
+
%i{INT TERM}.each{ |sig| trap(sig, &handler)}
|
99
108
|
super
|
100
109
|
end
|
101
110
|
|
data.tar.gz.sig
CHANGED
Binary file
|