aspera-cli 4.21.1 → 4.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +1 -1
- data/CHANGELOG.md +52 -22
- data/CONTRIBUTING.md +69 -148
- data/README.md +929 -668
- data/bin/ascli +5 -14
- data/bin/asession +1 -3
- data/examples/get_proto_file.rb +4 -3
- data/examples/proxy.pac +20 -20
- data/lib/aspera/agent/base.rb +11 -5
- data/lib/aspera/agent/connect.rb +30 -28
- data/lib/aspera/agent/{alpha.rb → desktop.rb} +35 -31
- data/lib/aspera/agent/direct.rb +141 -121
- data/lib/aspera/agent/httpgw.rb +22 -26
- data/lib/aspera/agent/node.rb +14 -11
- data/lib/aspera/agent/transferd.rb +30 -19
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +6 -6
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/httpgw.rb +7 -3
- data/lib/aspera/api/node.rb +10 -8
- data/lib/aspera/ascmd.rb +3 -3
- data/lib/aspera/ascp/installation.rb +53 -72
- data/lib/aspera/ascp/management.rb +1 -1
- data/lib/aspera/assert.rb +11 -2
- data/lib/aspera/cli/error.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +46 -21
- data/lib/aspera/cli/formatter.rb +55 -48
- data/lib/aspera/cli/hints.rb +1 -1
- data/lib/aspera/cli/info.rb +1 -0
- data/lib/aspera/cli/main.rb +192 -170
- data/lib/aspera/cli/manager.rb +18 -18
- data/lib/aspera/cli/plugin.rb +23 -20
- data/lib/aspera/cli/plugin_factory.rb +1 -1
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +247 -159
- data/lib/aspera/cli/plugins/ats.rb +19 -17
- data/lib/aspera/cli/plugins/config.rb +76 -113
- data/lib/aspera/cli/plugins/console.rb +5 -3
- data/lib/aspera/cli/plugins/faspex.rb +39 -35
- data/lib/aspera/cli/plugins/faspex5.rb +111 -84
- data/lib/aspera/cli/plugins/faspio.rb +13 -1
- data/lib/aspera/cli/plugins/httpgw.rb +13 -1
- data/lib/aspera/cli/plugins/node.rb +312 -182
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +6 -6
- data/lib/aspera/cli/plugins/shares.rb +5 -5
- data/lib/aspera/cli/sync_actions.rb +19 -18
- data/lib/aspera/cli/transfer_agent.rb +5 -5
- data/lib/aspera/cli/transfer_progress.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +116 -95
- data/lib/aspera/coverage.rb +8 -5
- data/lib/aspera/environment.rb +26 -17
- data/lib/aspera/faspex_gw.rb +14 -14
- data/lib/aspera/faspex_postproc.rb +10 -11
- data/lib/aspera/hash_ext.rb +4 -14
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +47 -34
- data/lib/aspera/keychain/factory.rb +41 -0
- data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
- data/lib/aspera/keychain/macos_security.rb +19 -11
- data/lib/aspera/log.rb +28 -34
- data/lib/aspera/nagios.rb +6 -6
- data/lib/aspera/node_simulator.rb +8 -8
- data/lib/aspera/oauth/base.rb +14 -7
- data/lib/aspera/oauth/factory.rb +5 -6
- data/lib/aspera/oauth/url_json.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +6 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +13 -10
- data/lib/aspera/preview/options.rb +16 -16
- data/lib/aspera/preview/terminal.rb +4 -4
- data/lib/aspera/preview/utils.rb +15 -17
- data/lib/aspera/products/connect.rb +35 -1
- data/lib/aspera/products/{alpha.rb → desktop.rb} +3 -3
- data/lib/aspera/products/transferd.rb +9 -2
- data/lib/aspera/proxy_auto_config.rb +2 -2
- data/lib/aspera/rest.rb +56 -47
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/secret_hider.rb +12 -5
- data/lib/aspera/ssh.rb +4 -4
- data/lib/aspera/temp_file_manager.rb +5 -4
- data/lib/aspera/transfer/convert.rb +29 -0
- data/lib/aspera/transfer/error_info.rb +66 -66
- data/lib/aspera/transfer/parameters.rb +13 -68
- data/lib/aspera/transfer/spec.rb +5 -6
- data/lib/aspera/transfer/spec.schema.yaml +753 -0
- data/lib/aspera/transfer/spec_doc.rb +62 -0
- data/lib/aspera/transfer/sync.rb +23 -72
- data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
- data/lib/aspera/transfer/uri.rb +6 -6
- data/lib/aspera/uri_reader.rb +18 -1
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +53 -44
- data.tar.gz.sig +0 -0
- metadata +28 -165
- metadata.gz.sig +0 -0
- data/examples/build_exec +0 -74
- data/examples/build_exec_rubyc +0 -40
- data/examples/build_package.sh +0 -28
- data/lib/aspera/transfer/spec.yaml +0 -718
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/agent/base'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
module Transfer
|
7
|
+
# translate transfer specification to ascp parameter list
|
8
|
+
class SpecDoc
|
9
|
+
class << self
|
10
|
+
# first letter of agent name symbol
|
11
|
+
def agent_to_short(agent_sym)
|
12
|
+
agent_sym.to_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
|
13
|
+
end
|
14
|
+
|
15
|
+
# @columns formatter [Cli::Formatter] formatter to use, methods: special_format, check_row
|
16
|
+
# @columns &block modify parameter info if needed
|
17
|
+
# @return a table suitable to display in manual
|
18
|
+
def man_table(formatter, cli: true)
|
19
|
+
col_local = agent_to_short(:direct)
|
20
|
+
Spec::SCHEMA['properties'].filter_map do |name, properties|
|
21
|
+
# manual table
|
22
|
+
columns = {
|
23
|
+
name: name,
|
24
|
+
type: properties['type'],
|
25
|
+
description: []
|
26
|
+
}
|
27
|
+
# replace "back solidus" HTML entity with its text value and split lines
|
28
|
+
columns[:description].concat(properties['description'].gsub('\', '\\').split("\n")) if properties.key?('description')
|
29
|
+
columns[:description].unshift("DEPRECATED: #{properties['x-deprecation']}") if properties.key?('x-deprecation')
|
30
|
+
# add flags for supported agents in doc
|
31
|
+
AGENT_LIST.each do |agent_info|
|
32
|
+
columns[agent_info.last] = Cli::Formatter.tick(properties['x-agents'].nil? || properties['x-agents'].include?(agent_info.first.to_s))
|
33
|
+
end
|
34
|
+
columns[col_local] = Cli::Formatter.tick(true) if properties['x-cli-option']
|
35
|
+
# only keep lines that are usable in supported agents
|
36
|
+
next false if AGENT_LIST.map(&:last).inject(true){ |memory, agent_short_sym| memory && columns[agent_short_sym].empty?}
|
37
|
+
columns[:description].push("Allowed values: #{properties['enum'].join(', ')}") if properties.key?('enum')
|
38
|
+
cli_option =
|
39
|
+
if properties['x-cli-switch']
|
40
|
+
properties['x-cli-option']
|
41
|
+
elsif properties['x-cli-special']
|
42
|
+
formatter.special_format('special')
|
43
|
+
elsif properties['x-cli-option']
|
44
|
+
arg_type = properties.key?('enum') ? '{enum}' : "{#{[properties['type']].flatten.join('|')}}"
|
45
|
+
conversion_tag = properties.key?('x-cli-convert') ? '(conversion)' : ''
|
46
|
+
sep = properties['x-cli-option'].start_with?('--') ? '=' : ' '
|
47
|
+
"#{properties['x-cli-option']}#{sep}#{conversion_tag}#{arg_type}"
|
48
|
+
end
|
49
|
+
cli_option = 'env:' + properties['x-cli-envvar'] if properties.key?('x-cli-envvar')
|
50
|
+
columns[:description].push("(#{cli_option})") if cli && !cli_option.to_s.empty?
|
51
|
+
formatter.check_row(columns)
|
52
|
+
end.sort_by{ |i| i[:name]}
|
53
|
+
end
|
54
|
+
end
|
55
|
+
# Agents shown in manual for parameters (sub list)
|
56
|
+
AGENT_LIST = Agent::Base.agent_list.map do |agent_sym|
|
57
|
+
[agent_sym, agent_sym.to_s.capitalize, agent_to_short(agent_sym)]
|
58
|
+
end.sort_by(&:last).freeze
|
59
|
+
TABLE_COLUMNS = (%i[name type] + AGENT_LIST.map(&:last) + %i[description]).freeze
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
data/lib/aspera/transfer/sync.rb
CHANGED
@@ -5,6 +5,7 @@
|
|
5
5
|
require 'aspera/command_line_builder'
|
6
6
|
require 'aspera/ascp/installation'
|
7
7
|
require 'aspera/agent/direct'
|
8
|
+
require 'aspera/transfer/convert'
|
8
9
|
require 'aspera/log'
|
9
10
|
require 'aspera/assert'
|
10
11
|
require 'json'
|
@@ -19,55 +20,8 @@ module Aspera
|
|
19
20
|
# sync direction, default is push
|
20
21
|
DIRECTIONS = %i[push pull bidi].freeze
|
21
22
|
# JSON for async instance command line options
|
22
|
-
|
23
|
-
|
24
|
-
'alt_logdir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
25
|
-
'watchd' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
26
|
-
'apply_local_docroot' => { cli: { type: :opt_without_arg}},
|
27
|
-
'quiet' => { cli: { type: :opt_without_arg}},
|
28
|
-
'ws_connect' => { cli: { type: :opt_without_arg}}
|
29
|
-
}.freeze
|
30
|
-
|
31
|
-
# map sync session parameters to transfer spec: sync -> ts, true if same
|
32
|
-
CMDLINE_PARAMS_SESSION =
|
33
|
-
{
|
34
|
-
'name' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
35
|
-
'local_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
36
|
-
'remote_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
37
|
-
'local_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
38
|
-
'remote_db_dir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
39
|
-
'host' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_host},
|
40
|
-
'user' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: :remote_user},
|
41
|
-
'private_key_paths' => { cli: { type: :opt_with_arg, switch: '--private-key-path'}, accepted_types: :array},
|
42
|
-
'direction' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
43
|
-
'checksum' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
44
|
-
'tags' => { cli: { type: :opt_with_arg, switch: '--tags64', convert: 'Aspera::Transfer::Parameters.convert_json64'},
|
45
|
-
accepted_types: :hash, ts: true},
|
46
|
-
'tcp_port' => { cli: { type: :opt_with_arg}, accepted_types: :int, ts: :ssh_port},
|
47
|
-
'rate_policy' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
48
|
-
'target_rate' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
49
|
-
'cooloff' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
50
|
-
'pending_max' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
51
|
-
'scan_intensity' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
52
|
-
'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Transfer::Parameters.convert_remove_hyphen'},
|
53
|
-
accepted_types: :string, ts: true},
|
54
|
-
'transfer_threads' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
55
|
-
'preserve_time' => { cli: { type: :opt_without_arg}, ts: :preserve_times},
|
56
|
-
'preserve_access_time' => { cli: { type: :opt_without_arg}, ts: nil},
|
57
|
-
'preserve_modification_time' => { cli: { type: :opt_without_arg}, ts: nil},
|
58
|
-
'preserve_uid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_uid},
|
59
|
-
'preserve_gid' => { cli: { type: :opt_without_arg}, ts: :preserve_file_owner_gid},
|
60
|
-
'create_dir' => { cli: { type: :opt_without_arg}, ts: true},
|
61
|
-
'reset' => { cli: { type: :opt_without_arg}},
|
62
|
-
# NOTE: only one env var, but multiple sessions... could be a problem
|
63
|
-
'remote_password' => { cli: { type: :envvar, variable: 'ASPERA_SCP_PASS'}, ts: true},
|
64
|
-
'cookie' => { cli: { type: :envvar, variable: 'ASPERA_SCP_COOKIE'}, ts: true},
|
65
|
-
'token' => { cli: { type: :envvar, variable: 'ASPERA_SCP_TOKEN'}, ts: true},
|
66
|
-
'license' => { cli: { type: :envvar, variable: 'ASPERA_SCP_LICENSE'}}
|
67
|
-
}.freeze
|
68
|
-
|
69
|
-
CommandLineBuilder.normalize_description(CMDLINE_PARAMS_INSTANCE)
|
70
|
-
CommandLineBuilder.normalize_description(CMDLINE_PARAMS_SESSION)
|
23
|
+
INSTANCE_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'instance')
|
24
|
+
SESSION_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'session')
|
71
25
|
|
72
26
|
CMDLINE_PARAMS_KEYS = %w[instance sessions].freeze
|
73
27
|
|
@@ -86,10 +40,10 @@ module Aspera
|
|
86
40
|
|
87
41
|
ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
|
88
42
|
|
89
|
-
private_constant :
|
43
|
+
private_constant :INSTANCE_SCHEMA, :SESSION_SCHEMA, :CMDLINE_PARAMS_KEYS, :TSPEC_TO_ASYNC_CONF, :ASYNC_ADMIN_EXECUTABLE
|
90
44
|
|
91
45
|
class << self
|
92
|
-
# Set remote_dir in sync parameters based on transfer spec
|
46
|
+
# Set `remote_dir` in sync parameters based on transfer spec
|
93
47
|
# @param params [Hash] sync parameters, old or new format
|
94
48
|
# @param remote_dir_key [String] key to update in above hash
|
95
49
|
# @param transfer_spec [Hash] transfer spec
|
@@ -100,7 +54,8 @@ module Aspera
|
|
100
54
|
elsif transfer_spec['cookie']&.start_with?('aspera.shares2')
|
101
55
|
# TODO : something more generic, independent of Shares
|
102
56
|
# in Shares, the actual folder on remote end is not always the same as the name of the share
|
103
|
-
|
57
|
+
remote_key = transfer_spec['direction'].eql?('send') ? 'destination' : 'source'
|
58
|
+
actual_remote = transfer_spec['paths']&.first&.[](remote_key)
|
104
59
|
sync_params[remote_dir_key] = actual_remote if actual_remote
|
105
60
|
end
|
106
61
|
nil
|
@@ -133,13 +88,11 @@ module Aspera
|
|
133
88
|
end
|
134
89
|
|
135
90
|
# @param sync_params [Hash] sync parameters, old or new format
|
136
|
-
# @param block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
|
137
|
-
def start(
|
138
|
-
sync_params,
|
139
|
-
&block
|
140
|
-
)
|
91
|
+
# @param &block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
|
92
|
+
def start(sync_params)
|
141
93
|
Log.log.debug{Log.dump(:sync_params_initial, sync_params)}
|
142
94
|
Aspera.assert_type(sync_params, Hash)
|
95
|
+
Aspera.assert(%w[local sessions].any?{ |k| sync_params.key?(k)}){'At least one of `local` or `sessions` must be present in async parameters'}
|
143
96
|
env_args = {
|
144
97
|
args: [],
|
145
98
|
env: {}
|
@@ -151,7 +104,7 @@ module Aspera
|
|
151
104
|
Aspera.assert_type(remote, Hash){'remote'}
|
152
105
|
Aspera.assert_type(remote['path'], String){'remote path'}
|
153
106
|
# get transfer spec if possible, and feed back to new structure
|
154
|
-
if
|
107
|
+
if block_given?
|
155
108
|
transfer_spec = yield((sync_params['direction'] || 'push').to_sym, sync_params['local']['path'], remote['path'])
|
156
109
|
# translate transfer spec to async parameters
|
157
110
|
TSPEC_TO_ASYNC_CONF.each do |ts_param, sy_path|
|
@@ -172,24 +125,25 @@ module Aspera
|
|
172
125
|
end
|
173
126
|
# '--exclusive-mgmt-port=12345', '--arg-err-path=-',
|
174
127
|
env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"]
|
175
|
-
Log.log.debug{Log.dump(:
|
128
|
+
Log.log.debug{Log.dump(:sync_conf, sync_params)}
|
176
129
|
agent = Agent::Direct.new
|
177
130
|
agent.start_and_monitor_process(session: {}, name: :async, **env_args)
|
178
|
-
|
131
|
+
else
|
132
|
+
# key 'sessions' is present
|
179
133
|
# ascli JSON format (cmdline)
|
180
134
|
raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
|
181
135
|
sync_params.keys.push('instance').uniq.sort.eql?(CMDLINE_PARAMS_KEYS)
|
182
136
|
Aspera.assert_type(sync_params['sessions'], Array)
|
183
137
|
Aspera.assert_type(sync_params['sessions'].first, Hash)
|
184
|
-
if
|
138
|
+
if block_given?
|
185
139
|
sync_params['sessions'].each do |session|
|
186
140
|
Aspera.assert_type(session['local_dir'], String){'local_dir'}
|
187
141
|
Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
|
188
142
|
transfer_spec = yield((session['direction'] || 'push').to_sym, session['local_dir'], session['remote_dir'])
|
189
|
-
|
190
|
-
if
|
191
|
-
tspec_param =
|
192
|
-
session[
|
143
|
+
SESSION_SCHEMA['properties'].each do |name, properties|
|
144
|
+
if properties.key?('x-tspec')
|
145
|
+
tspec_param = properties['x-tspec'].is_a?(TrueClass) ? name : properties['x-tspec'].to_s
|
146
|
+
session[name] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
|
193
147
|
end
|
194
148
|
end
|
195
149
|
session['private_key_paths'] = Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa) if transfer_spec.key?('token')
|
@@ -198,21 +152,18 @@ module Aspera
|
|
198
152
|
end
|
199
153
|
if sync_params.key?('instance')
|
200
154
|
Aspera.assert_type(sync_params['instance'], Hash)
|
201
|
-
instance_builder = CommandLineBuilder.new(sync_params['instance'],
|
155
|
+
instance_builder = CommandLineBuilder.new(sync_params['instance'], INSTANCE_SCHEMA, Convert)
|
202
156
|
instance_builder.process_params
|
203
157
|
instance_builder.add_env_args(env_args)
|
204
158
|
end
|
205
|
-
|
206
159
|
sync_params['sessions'].each do |session_params|
|
207
160
|
Aspera.assert_type(session_params, Hash)
|
208
161
|
Aspera.assert(session_params.key?('name')){'session must contain at least name'}
|
209
|
-
session_builder = CommandLineBuilder.new(session_params,
|
162
|
+
session_builder = CommandLineBuilder.new(session_params, SESSION_SCHEMA, Convert)
|
210
163
|
session_builder.process_params
|
211
164
|
session_builder.add_env_args(env_args)
|
212
165
|
end
|
213
166
|
Environment.secure_execute(exec: Ascp::Installation.instance.path(:async), **env_args)
|
214
|
-
else
|
215
|
-
raise 'At least one of `local` or `sessions` must be present in async parameters'
|
216
167
|
end
|
217
168
|
return nil
|
218
169
|
end
|
@@ -248,8 +199,8 @@ module Aspera
|
|
248
199
|
raise 'Missing either local_db_dir or local.path'
|
249
200
|
end
|
250
201
|
elsif sync_params.key?('sessions')
|
251
|
-
session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{|s|s['name'].eql?(session_name)}
|
252
|
-
raise "Session #{session_name} not found in #{sync_params['sessions'].map{|s|s['name']}.join(',')}" if session.nil?
|
202
|
+
session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{ |s| s['name'].eql?(session_name)}
|
203
|
+
raise "Session #{session_name} not found in #{sync_params['sessions'].map{ |s| s['name']}.join(',')}" if session.nil?
|
253
204
|
raise 'Missing session name' if session['name'].nil?
|
254
205
|
arguments.push("--name=#{session['name']}")
|
255
206
|
if session.key?('local_db_dir')
|
@@ -0,0 +1,79 @@
|
|
1
|
+
title: SyncSessionSpec
|
2
|
+
type: object
|
3
|
+
properties:
|
4
|
+
name:
|
5
|
+
type: string
|
6
|
+
local_dir:
|
7
|
+
type: string
|
8
|
+
remote_dir:
|
9
|
+
type: string
|
10
|
+
local_db_dir:
|
11
|
+
type: string
|
12
|
+
remote_db_dir:
|
13
|
+
type: string
|
14
|
+
host:
|
15
|
+
type: string
|
16
|
+
x-tspec: remote_host
|
17
|
+
user:
|
18
|
+
type: string
|
19
|
+
x-tspec: remote_user
|
20
|
+
private_key_paths:
|
21
|
+
type: array
|
22
|
+
x-cli-option: "--private-key-path"
|
23
|
+
direction:
|
24
|
+
type: string
|
25
|
+
checksum:
|
26
|
+
type: string
|
27
|
+
tags:
|
28
|
+
type: object
|
29
|
+
x-cli-option: "--tags64"
|
30
|
+
x-cli-convert: json64
|
31
|
+
x-tspec: true
|
32
|
+
tcp_port:
|
33
|
+
type: integer
|
34
|
+
x-tspec: ssh_port
|
35
|
+
rate_policy:
|
36
|
+
type: string
|
37
|
+
target_rate:
|
38
|
+
type: string
|
39
|
+
cooloff:
|
40
|
+
type: integer
|
41
|
+
pending_max:
|
42
|
+
type: integer
|
43
|
+
scan_intensity:
|
44
|
+
type: string
|
45
|
+
cipher:
|
46
|
+
type: string
|
47
|
+
x-cli-convert: remove_hyphen
|
48
|
+
x-tspec: true
|
49
|
+
transfer_threads:
|
50
|
+
type: integer
|
51
|
+
preserve_time:
|
52
|
+
x-cli-switch: true
|
53
|
+
x-tspec: preserve_times
|
54
|
+
preserve_access_time:
|
55
|
+
x-cli-switch: true
|
56
|
+
preserve_modification_time:
|
57
|
+
x-cli-switch: true
|
58
|
+
preserve_uid:
|
59
|
+
x-cli-switch: true
|
60
|
+
x-tspec: preserve_file_owner_uid
|
61
|
+
preserve_gid:
|
62
|
+
x-cli-switch: true
|
63
|
+
x-tspec: preserve_file_owner_gid
|
64
|
+
create_dir:
|
65
|
+
x-cli-switch: true
|
66
|
+
x-tspec: true
|
67
|
+
reset:
|
68
|
+
x-cli-switch: true
|
69
|
+
remote_password:
|
70
|
+
x-cli-envvar: ASPERA_SCP_PASS
|
71
|
+
x-tspec: true
|
72
|
+
cookie:
|
73
|
+
x-cli-envvar: ASPERA_SCP_COOKIE
|
74
|
+
x-tspec: true
|
75
|
+
token:
|
76
|
+
x-cli-envvar: ASPERA_SCP_TOKEN
|
77
|
+
x-tspec: true
|
78
|
+
license:
|
79
|
+
x-cli-envvar: ASPERA_SCP_LICENSE
|
data/lib/aspera/transfer/uri.rb
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
require 'aspera/log'
|
6
6
|
require 'aspera/rest'
|
7
|
-
require 'aspera/
|
7
|
+
require 'aspera/transfer/convert'
|
8
8
|
|
9
9
|
module Aspera
|
10
10
|
module Transfer
|
@@ -24,7 +24,7 @@ module Aspera
|
|
24
24
|
result_ts['ssh_port'] = @fasp_uri.port
|
25
25
|
result_ts['paths'] = [{'source' => URI.decode_www_form_component(@fasp_uri.path)}]
|
26
26
|
# faspex 4 does not encode trailing base64 padding, fix that to be able to decode properly
|
27
|
-
fixed_query = @fasp_uri.query.gsub(/(=+)$/){|trail_equals|'%3D' * trail_equals.length}
|
27
|
+
fixed_query = @fasp_uri.query.gsub(/(=+)$/){ |trail_equals| '%3D' * trail_equals.length}
|
28
28
|
|
29
29
|
Rest.query_to_h(fixed_query).each do |name, value|
|
30
30
|
case name
|
@@ -39,10 +39,10 @@ module Aspera
|
|
39
39
|
when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
|
40
40
|
when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub('--', '-')
|
41
41
|
when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
|
42
|
-
when 'createpath' then result_ts['create_dir'] =
|
43
|
-
when 'fallback' then result_ts['http_fallback'] =
|
44
|
-
when 'lockpolicy' then result_ts['lock_rate_policy'] =
|
45
|
-
when 'lockminrate' then result_ts['lock_min_rate'] =
|
42
|
+
when 'createpath' then result_ts['create_dir'] = Convert.yes_to_true(value)
|
43
|
+
when 'fallback' then result_ts['http_fallback'] = Convert.yes_to_true(value)
|
44
|
+
when 'lockpolicy' then result_ts['lock_rate_policy'] = Convert.yes_to_true(value)
|
45
|
+
when 'lockminrate' then result_ts['lock_min_rate'] = Convert.yes_to_true(value)
|
46
46
|
when 'auth' then Log.log.debug{"ignoring #{name}=#{value}"} # Not used (yes/no)
|
47
47
|
when 'v' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch -- Not used (shall be 2)
|
48
48
|
when 'protect' then Log.log.debug{"ignoring #{name}=#{value}"} # rubocop:disable Lint/DuplicateBranch -- TODO: what is this ?
|
data/lib/aspera/uri_reader.rb
CHANGED
@@ -1,11 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'uri'
|
4
|
+
require 'aspera/assert'
|
4
5
|
require 'aspera/rest'
|
6
|
+
require 'aspera/temp_file_manager'
|
5
7
|
|
6
8
|
module Aspera
|
7
9
|
# read some content from some URI, support file: , http: and https: schemes
|
8
10
|
module UriReader
|
11
|
+
FILE_SCHEME_PREFIX = 'file:///'
|
12
|
+
private_constant :FILE_SCHEME_PREFIX
|
9
13
|
class << self
|
10
14
|
# read some content from some URI, support file: , http: and https: schemes
|
11
15
|
def read(uri_to_read)
|
@@ -18,8 +22,21 @@ module Aspera
|
|
18
22
|
raise 'URL shall have a path, check syntax' if local_file_path.nil?
|
19
23
|
local_file_path = File.expand_path(local_file_path.gsub(%r{^/}, '')) if %r{^/(~|.|..)/}.match?(local_file_path)
|
20
24
|
return File.read(local_file_path)
|
25
|
+
else Aspera.error_unexpected_value(uri.scheme){"scheme for [#{uri_to_read}]"}
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# @return path to file with content at URL
|
30
|
+
def read_as_file(url)
|
31
|
+
if url.start_with?('file:')
|
32
|
+
# require specific file scheme: the path part is "relative", or absolute if there are 4 slash
|
33
|
+
raise "use format: #{FILE_SCHEME_PREFIX}<path>" unless url.start_with?(FILE_SCHEME_PREFIX)
|
34
|
+
return File.expand_path(url[FILE_SCHEME_PREFIX.length..-1])
|
21
35
|
else
|
22
|
-
|
36
|
+
# autodelete on exit
|
37
|
+
sdk_archive_path = TempFileManager.instance.new_file_path_global(suffix: File.basename(url))
|
38
|
+
Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to_file: sdk_archive_path)
|
39
|
+
return sdk_archive_path
|
23
40
|
end
|
24
41
|
end
|
25
42
|
end
|
data/lib/aspera/web_auth.rb
CHANGED
@@ -182,7 +182,7 @@ module Aspera
|
|
182
182
|
# last argument (self) is provided to constructor of servlet
|
183
183
|
mount(@expected_path, WebAuthServlet, self)
|
184
184
|
# server runs in thread
|
185
|
-
Thread.new
|
185
|
+
Thread.new{start}
|
186
186
|
end
|
187
187
|
|
188
188
|
# Called by web server thread on received request
|
@@ -10,19 +10,23 @@ require 'openssl'
|
|
10
10
|
module Aspera
|
11
11
|
# Simple WEBrick server with HTTPS support
|
12
12
|
class WebServerSimple < WEBrick::HTTPServer
|
13
|
-
|
13
|
+
PARAMS = %i[cert key chain].freeze
|
14
|
+
DEFAULT_URL = 'http://localhost:8080'
|
14
15
|
GENERIC_ISSUER = '/C=FR/O=Test/OU=Test/CN=Test'
|
15
16
|
ONE_YEAR_SECONDS = 365 * 24 * 60 * 60
|
17
|
+
PKCS12_EXT = %w[p12 pfx].map{ |i| ".#{i}"}.freeze
|
18
|
+
CLOCK_SKEW_OFFSET_SEC = 5
|
16
19
|
|
17
|
-
private_constant :
|
20
|
+
private_constant :GENERIC_ISSUER, :ONE_YEAR_SECONDS, :PKCS12_EXT, :CLOCK_SKEW_OFFSET_SEC
|
18
21
|
|
19
22
|
class << self
|
20
|
-
#
|
21
|
-
def
|
23
|
+
# Generate or fill and self sign certificate
|
24
|
+
def self_signed_cert(private_key, digest: 'SHA256')
|
25
|
+
cert = OpenSSL::X509::Certificate.new
|
22
26
|
cert.subject = cert.issuer = OpenSSL::X509::Name.parse(GENERIC_ISSUER)
|
23
|
-
cert.not_before =
|
24
|
-
cert.not_after
|
25
|
-
cert.public_key =
|
27
|
+
cert.not_before = Time.now - CLOCK_SKEW_OFFSET_SEC
|
28
|
+
cert.not_after = cert.not_before + ONE_YEAR_SECONDS
|
29
|
+
cert.public_key = private_key.public_key
|
26
30
|
cert.serial = 0x0
|
27
31
|
cert.version = 2
|
28
32
|
ef = OpenSSL::X509::ExtensionFactory.new
|
@@ -34,68 +38,73 @@ module Aspera
|
|
34
38
|
# ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
|
35
39
|
]
|
36
40
|
cert.add_extension(ef.create_extension('authorityKeyIdentifier', 'keyid:always,issuer:always'))
|
37
|
-
cert.sign(
|
41
|
+
cert.sign(private_key, OpenSSL::Digest.new(digest))
|
42
|
+
cert
|
43
|
+
end
|
44
|
+
|
45
|
+
# @return a list of Certificates from chain file
|
46
|
+
def read_chain_file(chain)
|
47
|
+
File.read(chain).scan(/-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----/m).map{ |i| OpenSSL::X509::Certificate.new(i)}
|
38
48
|
end
|
39
49
|
end
|
40
50
|
|
41
|
-
# @param
|
42
|
-
|
43
|
-
|
51
|
+
# @param url [URI] Local address where server will listen (use scheme, host and port only)
|
52
|
+
# @param cert [String] Path to certificate file, either with extension .p12 or .pfx, else assumed PEM
|
53
|
+
# @param key [String] Path to key file (PEM) or passphrase (pkcs12)
|
54
|
+
# @param chain [String] Path to certificate chain file (PEM only)
|
55
|
+
def initialize(uri, cert: nil, key: nil, chain: nil)
|
56
|
+
Aspera.assert_type(uri, URI)
|
57
|
+
@uri = uri
|
44
58
|
# see https://www.rubydoc.info/stdlib/webrick/WEBrick/Config
|
45
59
|
webrick_options = {
|
46
|
-
BindAddress: uri.host,
|
47
|
-
Port: uri.port,
|
60
|
+
BindAddress: @uri.host,
|
61
|
+
Port: @uri.port,
|
48
62
|
Logger: Log.log,
|
49
63
|
AccessLog: [[self, WEBrick::AccessLog::COMMON_LOG_FORMAT]] # replace default access log to call local method "<<" below
|
50
64
|
}
|
51
|
-
case uri.scheme
|
65
|
+
case @uri.scheme
|
52
66
|
when 'http'
|
53
67
|
Log.log.debug('HTTP mode')
|
54
68
|
when 'https'
|
55
69
|
webrick_options[:SSLEnable] = true
|
56
|
-
if
|
70
|
+
if cert.nil? && key.nil?
|
57
71
|
webrick_options[:SSLCertName] = [['CN', WEBrick::Utils.getservername]]
|
72
|
+
elsif cert && PKCS12_EXT.include?(File.extname(cert).downcase)
|
73
|
+
# PKCS12
|
74
|
+
Log.log.debug('Using PKCS12 certificate')
|
75
|
+
raise 'PKCS12 requires a key (password)' if key.nil?
|
76
|
+
pkcs12 = OpenSSL::PKCS12.new(File.read(cert), key)
|
77
|
+
webrick_options[:SSLCertificate] = pkcs12.certificate
|
78
|
+
webrick_options[:SSLPrivateKey] = pkcs12.key
|
79
|
+
webrick_options[:SSLExtraChainCert] = pkcs12.ca_certs
|
58
80
|
else
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
webrick_options[:
|
67
|
-
webrick_options[:SSLPrivateKey] = pkcs12.key
|
68
|
-
webrick_options[:SSLExtraChainCert] = pkcs12.ca_certs
|
81
|
+
Log.log.debug('Using PEM certificate')
|
82
|
+
webrick_options[:SSLPrivateKey] = if key.nil?
|
83
|
+
OpenSSL::PKey::RSA.new(4096)
|
84
|
+
else
|
85
|
+
OpenSSL::PKey::RSA.new(File.read(key))
|
86
|
+
end
|
87
|
+
webrick_options[:SSLCertificate] = if cert.nil?
|
88
|
+
self.class.self_signed_cert(webrick_options[:SSLPrivateKey])
|
69
89
|
else
|
70
|
-
|
71
|
-
webrick_options[:SSLPrivateKey] = if certificate.key?(:key)
|
72
|
-
OpenSSL::PKey::RSA.new(File.read(certificate[:key]))
|
73
|
-
else
|
74
|
-
OpenSSL::PKey::RSA.new(4096)
|
75
|
-
end
|
76
|
-
if certificate.key?(:cert)
|
77
|
-
webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read(certificate[:cert]))
|
78
|
-
else
|
79
|
-
webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new
|
80
|
-
self.class.fill_self_signed_cert(webrick_options[:SSLCertificate], webrick_options[:SSLPrivateKey])
|
81
|
-
end
|
82
|
-
if certificate.key?(:chain)
|
83
|
-
webrick_options[:SSLExtraChainCert] = [OpenSSL::X509::Certificate.new(File.read(certificate[:chain]))]
|
84
|
-
end
|
90
|
+
OpenSSL::X509::Certificate.new(File.read(cert))
|
85
91
|
end
|
92
|
+
webrick_options[:SSLExtraChainCert] = read_chain_file(chain) unless chain.nil?
|
93
|
+
raise 'key and cert do not match' unless webrick_options[:SSLCertificate].public_key.to_der == webrick_options[:SSLPrivateKey].public_key.to_der
|
86
94
|
end
|
87
95
|
end
|
88
96
|
# call constructor of parent class, but capture STDERR
|
89
97
|
# self signed certificate generates characters on STDERR
|
90
98
|
# see create_self_signed_cert in webrick/ssl.rb
|
91
|
-
Log.capture_stderr
|
99
|
+
Log.capture_stderr{super(webrick_options)}
|
92
100
|
end
|
93
101
|
|
94
102
|
# blocking
|
95
103
|
def start
|
96
|
-
Log.log.info{"Listening on #{@
|
97
|
-
# kill -
|
98
|
-
|
104
|
+
Log.log.info{"Listening on #{@uri}"}
|
105
|
+
# kill (-TERM) for graceful shutdown
|
106
|
+
handler = proc{shutdown}
|
107
|
+
%i{INT TERM}.each{ |sig| trap(sig, &handler)}
|
99
108
|
super
|
100
109
|
end
|
101
110
|
|
data.tar.gz.sig
CHANGED
Binary file
|