aspera-cli 4.24.1 → 4.25.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1064 -745
- data/CONTRIBUTING.md +43 -100
- data/README.md +1281 -720
- data/bin/ascli +20 -1
- data/bin/asession +23 -27
- data/lib/aspera/agent/base.rb +10 -21
- data/lib/aspera/agent/connect.rb +2 -3
- data/lib/aspera/agent/desktop.rb +2 -2
- data/lib/aspera/agent/direct.rb +49 -32
- data/lib/aspera/agent/factory.rb +31 -0
- data/lib/aspera/api/aoc.rb +134 -76
- data/lib/aspera/api/cos_node.rb +3 -2
- data/lib/aspera/api/faspex.rb +213 -0
- data/lib/aspera/api/node.rb +107 -94
- data/lib/aspera/ascmd.rb +1 -2
- data/lib/aspera/ascp/installation.rb +73 -58
- data/lib/aspera/ascp/management.rb +119 -23
- data/lib/aspera/assert.rb +39 -11
- data/lib/aspera/cli/error.rb +4 -2
- data/lib/aspera/cli/extended_value.rb +91 -67
- data/lib/aspera/cli/formatter.rb +62 -27
- data/lib/aspera/cli/hints.rb +8 -0
- data/lib/aspera/cli/info.rb +4 -4
- data/lib/aspera/cli/main.rb +76 -84
- data/lib/aspera/cli/manager.rb +352 -248
- data/lib/aspera/cli/plugins/alee.rb +5 -4
- data/lib/aspera/cli/plugins/aoc.rb +175 -195
- data/lib/aspera/cli/plugins/ats.rb +4 -4
- data/lib/aspera/cli/plugins/base.rb +343 -0
- data/lib/aspera/cli/plugins/basic_auth.rb +45 -0
- data/lib/aspera/cli/plugins/config.rb +283 -269
- data/lib/aspera/cli/plugins/console.rb +27 -22
- data/lib/aspera/cli/plugins/cos.rb +3 -3
- data/lib/aspera/cli/plugins/factory.rb +78 -0
- data/lib/aspera/cli/plugins/faspex.rb +49 -46
- data/lib/aspera/cli/plugins/faspex5.rb +113 -225
- data/lib/aspera/cli/plugins/faspio.rb +19 -18
- data/lib/aspera/cli/plugins/httpgw.rb +14 -13
- data/lib/aspera/cli/plugins/node.rb +162 -149
- data/lib/aspera/cli/plugins/oauth.rb +48 -0
- data/lib/aspera/cli/plugins/orchestrator.rb +129 -45
- data/lib/aspera/cli/plugins/preview.rb +30 -50
- data/lib/aspera/cli/plugins/server.rb +21 -21
- data/lib/aspera/cli/plugins/shares.rb +45 -47
- data/lib/aspera/cli/sync_actions.rb +50 -39
- data/lib/aspera/cli/transfer_agent.rb +35 -49
- data/lib/aspera/cli/transfer_progress.rb +6 -6
- data/lib/aspera/cli/version.rb +3 -3
- data/lib/aspera/cli/wizard.rb +70 -55
- data/lib/aspera/colors.rb +6 -0
- data/lib/aspera/command_line_builder.rb +59 -61
- data/lib/aspera/command_line_converter.rb +2 -1
- data/lib/aspera/coverage.rb +2 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +51 -41
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/keychain/factory.rb +1 -2
- data/lib/aspera/keychain/macos_security.rb +1 -1
- data/lib/aspera/log.rb +37 -9
- data/lib/aspera/markdown.rb +31 -0
- data/lib/aspera/nagios.rb +7 -6
- data/lib/aspera/oauth/base.rb +25 -28
- data/lib/aspera/oauth/factory.rb +9 -9
- data/lib/aspera/oauth/url_json.rb +2 -1
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/preview/file_types.rb +23 -37
- data/lib/aspera/products/connect.rb +7 -6
- data/lib/aspera/products/desktop.rb +1 -4
- data/lib/aspera/products/other.rb +9 -1
- data/lib/aspera/products/transferd.rb +0 -1
- data/lib/aspera/rest.rb +168 -113
- data/lib/aspera/rest_error_analyzer.rb +4 -4
- data/lib/aspera/ssh.rb +7 -4
- data/lib/aspera/ssl.rb +41 -0
- data/lib/aspera/sync/args.schema.yaml +46 -3
- data/lib/aspera/sync/conf.schema.yaml +307 -123
- data/lib/aspera/sync/database.rb +2 -1
- data/lib/aspera/sync/operations.rb +135 -79
- data/lib/aspera/temp_file_manager.rb +17 -5
- data/lib/aspera/transfer/error.rb +16 -7
- data/lib/aspera/transfer/parameters.rb +35 -22
- data/lib/aspera/transfer/resumer.rb +74 -0
- data/lib/aspera/transfer/spec.rb +5 -5
- data/lib/aspera/transfer/spec.schema.yaml +170 -59
- data/lib/aspera/transfer/spec_doc.rb +49 -43
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +6 -6
- data/lib/transferd_pb.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +26 -11
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +0 -43
- data/lib/aspera/cli/plugin.rb +0 -333
- data/lib/aspera/cli/plugin_factory.rb +0 -81
- data/lib/aspera/resumer.rb +0 -77
- data/lib/aspera/transfer/error_info.rb +0 -91
data/lib/aspera/cli/wizard.rb
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'aspera/oauth/jwt'
|
|
4
|
+
require 'aspera/assert'
|
|
5
|
+
require 'aspera/cli/plugins/factory'
|
|
4
6
|
|
|
5
7
|
module Aspera
|
|
6
8
|
module Cli
|
|
9
|
+
# The wizard detects applications and generates a config
|
|
7
10
|
class Wizard
|
|
8
11
|
WIZARD_RESULT_KEYS = %i[preset_value test_args].freeze
|
|
9
12
|
DEFAULT_PRIV_KEY_FILENAME = 'my_private_key.pem' # pragma: allowlist secret
|
|
@@ -13,13 +16,17 @@ module Aspera
|
|
|
13
16
|
def initialize(parent, main_folder)
|
|
14
17
|
@parent = parent
|
|
15
18
|
@main_folder = main_folder
|
|
16
|
-
#
|
|
17
|
-
options.declare(:override, 'Wizard: override existing value',
|
|
18
|
-
options.declare(:default, 'Wizard: set as default configuration for specified plugin (also: update)',
|
|
19
|
-
options.declare(:test_mode, 'Wizard: skip private key check step', values: :bool, default: false)
|
|
19
|
+
# Wizard options
|
|
20
|
+
options.declare(:override, 'Wizard: override existing value', allowed: Allowed::TYPES_BOOLEAN, default: false)
|
|
21
|
+
options.declare(:default, 'Wizard: set as default configuration for specified plugin (also: update)', allowed: Allowed::TYPES_BOOLEAN, default: true)
|
|
20
22
|
options.declare(:key_path, 'Wizard: path to private key for JWT')
|
|
21
23
|
end
|
|
22
24
|
|
|
25
|
+
# @return false if in test mode to avoid interactive input
|
|
26
|
+
def required
|
|
27
|
+
!ENV['ASCLI_WIZ_TEST']
|
|
28
|
+
end
|
|
29
|
+
|
|
23
30
|
def options
|
|
24
31
|
@parent.options
|
|
25
32
|
end
|
|
@@ -32,6 +39,10 @@ module Aspera
|
|
|
32
39
|
@parent.config
|
|
33
40
|
end
|
|
34
41
|
|
|
42
|
+
def check_email(email)
|
|
43
|
+
Aspera.assert(email =~ /\A[\w+\-.]+@[a-z\d\-.]+\.[a-z]+\z/i, type: ParameterError){"Username shall be an email: #{email}"}
|
|
44
|
+
end
|
|
45
|
+
|
|
35
46
|
# Find a plugin, and issue the "require"
|
|
36
47
|
# @return [Hash] plugin info: { product:, name:, url:, version: }
|
|
37
48
|
def identify_plugins_for_url
|
|
@@ -40,19 +51,19 @@ module Aspera
|
|
|
40
51
|
check_only = check_only.to_sym unless check_only.nil?
|
|
41
52
|
found_apps = []
|
|
42
53
|
my_self_plugin_sym = self.class.name.split('::').last.downcase.to_sym
|
|
43
|
-
|
|
44
|
-
#
|
|
54
|
+
Plugins::Factory.instance.plugin_list.each do |plugin_name_sym|
|
|
55
|
+
# No detection for internal plugin
|
|
45
56
|
next if plugin_name_sym.eql?(my_self_plugin_sym)
|
|
46
57
|
next if check_only && !check_only.eql?(plugin_name_sym)
|
|
47
|
-
#
|
|
48
|
-
|
|
49
|
-
#
|
|
50
|
-
next unless
|
|
58
|
+
# Load plugin class
|
|
59
|
+
plugin_klass = Plugins::Factory.instance.plugin_class(plugin_name_sym)
|
|
60
|
+
# Requires detection method
|
|
61
|
+
next unless plugin_klass.respond_to?(:detect)
|
|
51
62
|
detection_info = nil
|
|
52
63
|
begin
|
|
53
64
|
Log.log.debug{"detecting #{plugin_name_sym} at #{app_url}"}
|
|
54
65
|
formatter.long_operation_running("#{plugin_name_sym}\r")
|
|
55
|
-
detection_info =
|
|
66
|
+
detection_info = plugin_klass.detect(app_url)
|
|
56
67
|
rescue OpenSSL::SSL::SSLError => e
|
|
57
68
|
Log.log.warn(e.message)
|
|
58
69
|
Log.log.warn('Use option --insecure=yes to allow unchecked certificate') if e.message.include?('cert')
|
|
@@ -63,8 +74,8 @@ module Aspera
|
|
|
63
74
|
next if detection_info.nil?
|
|
64
75
|
Aspera.assert_type(detection_info, Hash)
|
|
65
76
|
Aspera.assert_type(detection_info[:url], String) if detection_info.key?(:url)
|
|
66
|
-
app_name =
|
|
67
|
-
#
|
|
77
|
+
app_name = plugin_klass.respond_to?(:application_name) ? plugin_klass.application_name : plugin_klass.name.split('::').last
|
|
78
|
+
# If there is a redirect, then the detector can override the url.
|
|
68
79
|
found_apps.push({product: plugin_name_sym, name: app_name, url: app_url, version: 'unknown'}.merge(detection_info))
|
|
69
80
|
end
|
|
70
81
|
raise "No known application found at #{app_url}" if found_apps.empty?
|
|
@@ -72,6 +83,42 @@ module Aspera
|
|
|
72
83
|
return found_apps
|
|
73
84
|
end
|
|
74
85
|
|
|
86
|
+
# To be called in public wizard method to get private key
|
|
87
|
+
# @param user [String] User's email
|
|
88
|
+
# @param url [String] Instance URL
|
|
89
|
+
# @param page [String] URL of page to enter pub key
|
|
90
|
+
# @return [String] Private key path (can contain ~ for home)
|
|
91
|
+
def ask_private_key(user:, url:, page:)
|
|
92
|
+
# Lets see if path to priv key is provided
|
|
93
|
+
private_key_path = options.get_option(:key_path)
|
|
94
|
+
# Give a chance to provide
|
|
95
|
+
if private_key_path.nil?
|
|
96
|
+
formatter.display_status('Path to private RSA key (leave empty to generate):')
|
|
97
|
+
private_key_path = options.get_option(:key_path, mandatory: true).to_s
|
|
98
|
+
end
|
|
99
|
+
# Else generate path
|
|
100
|
+
private_key_path = File.join(@main_folder, DEFAULT_PRIV_KEY_FILENAME) if private_key_path.empty?
|
|
101
|
+
if File.exist?(File.expand_path(private_key_path))
|
|
102
|
+
formatter.display_status('Using existing key:')
|
|
103
|
+
else
|
|
104
|
+
formatter.display_status("Generating #{OAuth::Jwt::DEFAULT_PRIV_KEY_LENGTH} bit RSA key...")
|
|
105
|
+
OAuth::Jwt.generate_rsa_private_key(path: private_key_path)
|
|
106
|
+
formatter.display_status('Created key:')
|
|
107
|
+
end
|
|
108
|
+
formatter.display_status(private_key_path)
|
|
109
|
+
private_key_pem = File.read(File.expand_path(private_key_path))
|
|
110
|
+
pub_key_pem = OpenSSL::PKey::RSA.new(private_key_pem).public_key.to_s
|
|
111
|
+
options.set_option(:private_key, private_key_pem)
|
|
112
|
+
formatter.display_status("Please Log in as user #{user.red} at: #{url.red}")
|
|
113
|
+
formatter.display_status("Navigate to: #{page}")
|
|
114
|
+
formatter.display_status("Check or update the value to (#{'including BEGIN/END lines'.red}):".blink)
|
|
115
|
+
formatter.display_status(pub_key_pem, hide_secrets: false)
|
|
116
|
+
formatter.display_status('Once updated or validated, press [Enter].')
|
|
117
|
+
Environment.instance.open_uri(url)
|
|
118
|
+
$stdin.gets if required
|
|
119
|
+
private_key_path
|
|
120
|
+
end
|
|
121
|
+
|
|
75
122
|
# Wizard function, creates configuration
|
|
76
123
|
# @param apps [Array] list of detected apps
|
|
77
124
|
def find(apps)
|
|
@@ -88,49 +135,18 @@ module Aspera
|
|
|
88
135
|
Log.dump(:identification, identification)
|
|
89
136
|
wiz_url = identification[:url]
|
|
90
137
|
formatter.display_status("Using: #{identification[:name]} at #{wiz_url}".bold)
|
|
91
|
-
#
|
|
138
|
+
# Set url for instantiation of plugin
|
|
92
139
|
options.add_option_preset({url: wiz_url}, 'wizard')
|
|
93
|
-
#
|
|
94
|
-
|
|
95
|
-
Aspera.assert(
|
|
140
|
+
# Instantiate plugin: command line options will be known, e.g. private_key, and wizard can be called
|
|
141
|
+
plugin_instance = Plugins::Factory.instance.plugin_class(identification[:product]).new(context: @parent.context)
|
|
142
|
+
Aspera.assert(plugin_instance.respond_to?(:wizard), type: Cli::BadArgument) do
|
|
96
143
|
"Detected: #{identification[:product]}, but this application has no wizard"
|
|
97
144
|
end
|
|
98
|
-
#
|
|
99
|
-
|
|
100
|
-
wiz_params = {
|
|
101
|
-
object: plugin_instance
|
|
102
|
-
}
|
|
103
|
-
# is private key needed ?
|
|
104
|
-
if options.known_options.key?(:private_key) &&
|
|
105
|
-
(!wiz_plugin_class.respond_to?(:private_key_required?) || wiz_plugin_class.private_key_required?(wiz_url))
|
|
106
|
-
# lets see if path to priv key is provided
|
|
107
|
-
private_key_path = options.get_option(:key_path)
|
|
108
|
-
# give a chance to provide
|
|
109
|
-
if private_key_path.nil?
|
|
110
|
-
formatter.display_status('Path to private RSA key (leave empty to generate):')
|
|
111
|
-
private_key_path = options.get_option(:key_path, mandatory: true).to_s
|
|
112
|
-
end
|
|
113
|
-
# else generate path
|
|
114
|
-
private_key_path = File.join(@main_folder, DEFAULT_PRIV_KEY_FILENAME) if private_key_path.empty?
|
|
115
|
-
if File.exist?(private_key_path)
|
|
116
|
-
formatter.display_status('Using existing key:')
|
|
117
|
-
else
|
|
118
|
-
formatter.display_status("Generating #{OAuth::Jwt::DEFAULT_PRIV_KEY_LENGTH} bit RSA key...")
|
|
119
|
-
OAuth::Jwt.generate_rsa_private_key(path: private_key_path)
|
|
120
|
-
formatter.display_status('Created key:')
|
|
121
|
-
end
|
|
122
|
-
formatter.display_status(private_key_path)
|
|
123
|
-
private_key_pem = File.read(private_key_path)
|
|
124
|
-
options.set_option(:private_key, private_key_pem)
|
|
125
|
-
wiz_params[:private_key_path] = private_key_path
|
|
126
|
-
wiz_params[:pub_key_pem] = OpenSSL::PKey::RSA.new(private_key_pem).public_key.to_s
|
|
127
|
-
end
|
|
128
|
-
Log.dump(:wiz_params, wiz_params)
|
|
129
|
-
# finally, call the wizard
|
|
130
|
-
wizard_result = wiz_plugin_class.wizard(**wiz_params)
|
|
145
|
+
# Call the wizard
|
|
146
|
+
wizard_result = plugin_instance.wizard(self, wiz_url)
|
|
131
147
|
Log.log.debug{"wizard result: #{wizard_result}"}
|
|
132
148
|
Aspera.assert(WIZARD_RESULT_KEYS.eql?(wizard_result.keys.sort)){"missing or extra keys in wizard result: #{wizard_result.keys}"}
|
|
133
|
-
#
|
|
149
|
+
# Get preset name from user or default
|
|
134
150
|
if wiz_preset_name.empty?
|
|
135
151
|
elements = [
|
|
136
152
|
identification[:product],
|
|
@@ -139,18 +155,17 @@ module Aspera
|
|
|
139
155
|
elements.push(options.get_option(:username, mandatory: true)) unless wizard_result[:preset_value].key?(:link) rescue nil
|
|
140
156
|
wiz_preset_name = elements.join('_').strip.downcase.gsub(/[^a-z0-9]/, '_').squeeze('_')
|
|
141
157
|
end
|
|
142
|
-
# test mode does not change conf file
|
|
143
|
-
return Main.result_single_object(wizard_result) if options.get_option(:test_mode)
|
|
144
158
|
# Write configuration file
|
|
145
159
|
formatter.display_status("Preparing preset: #{wiz_preset_name}")
|
|
146
|
-
#
|
|
160
|
+
# Init defaults if necessary
|
|
147
161
|
option_override = options.get_option(:override, mandatory: true)
|
|
148
162
|
option_default = options.get_option(:default, mandatory: true)
|
|
149
163
|
config.defaults_set(identification[:product], wiz_preset_name, wizard_result[:preset_value].stringify_keys, option_default, option_override)
|
|
150
164
|
test_args = wizard_result[:test_args]
|
|
151
165
|
test_args = "-P#{wiz_preset_name} #{test_args}" unless option_default
|
|
152
166
|
# TODO: actually test the command
|
|
153
|
-
|
|
167
|
+
test_cmd = "#{Info::CMD_NAME} #{identification[:product]} #{test_args}"
|
|
168
|
+
return Main.result_status("You can test with:\n#{test_cmd.red}")
|
|
154
169
|
end
|
|
155
170
|
end
|
|
156
171
|
end
|
data/lib/aspera/colors.rb
CHANGED
|
@@ -57,10 +57,16 @@ class String
|
|
|
57
57
|
define_method(name){self}
|
|
58
58
|
end
|
|
59
59
|
end
|
|
60
|
+
|
|
60
61
|
# Transform capitalized to snake case
|
|
61
62
|
def capital_to_snake
|
|
62
63
|
return gsub(/([a-z\d])([A-Z])/, '\1_\2')
|
|
63
64
|
.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
|
|
64
65
|
.downcase
|
|
65
66
|
end
|
|
67
|
+
|
|
68
|
+
# Transform snake case to capitalized
|
|
69
|
+
def snake_to_capital
|
|
70
|
+
split('_').map(&:capitalize).join
|
|
71
|
+
end
|
|
66
72
|
end
|
|
@@ -4,72 +4,69 @@ require 'aspera/log'
|
|
|
4
4
|
require 'aspera/assert'
|
|
5
5
|
require 'yaml'
|
|
6
6
|
module Aspera
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
# process_param is called repeatedly with all known parameters
|
|
10
|
-
# add_env_args is called to get resulting param list and env var (also checks that all params were used)
|
|
7
|
+
# Helper class to build command line from a parameter list (key-value hash)
|
|
8
|
+
# Constructor takes hash: `{ 'param1':'value1', ...}`
|
|
9
|
+
# `process_param` is called repeatedly with all known parameters
|
|
10
|
+
# `add_env_args` is called to get resulting param list and env var (also checks that all params were used)
|
|
11
11
|
class CommandLineBuilder
|
|
12
|
-
#
|
|
13
|
-
|
|
14
|
-
#
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
#
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
items
|
|
33
|
-
properties
|
|
34
|
-
required
|
|
35
|
-
$comment
|
|
36
|
-
x-cli-envvar
|
|
37
|
-
x-cli-option
|
|
38
|
-
x-cli-switch
|
|
39
|
-
x-cli-special
|
|
40
|
-
x-cli-convert
|
|
41
|
-
x-agents
|
|
42
|
-
x-ts-name
|
|
43
|
-
x-deprecation
|
|
12
|
+
# Supported keys in JSON schema
|
|
13
|
+
PROPERTY_KEYS = [
|
|
14
|
+
'description', # [String] Description
|
|
15
|
+
'type', # [String,Array] Accepted type(s) for non-enum
|
|
16
|
+
'default', # [String] Default value if not specified
|
|
17
|
+
'enum', # [Array] Set with list of values for enum types accepted in transfer spec
|
|
18
|
+
'items', # [Array]
|
|
19
|
+
'properties', # [Array]
|
|
20
|
+
'required', # [Array]
|
|
21
|
+
'$comment', # [String]
|
|
22
|
+
'x-cli-envvar', # [String] Name of env var
|
|
23
|
+
'x-cli-option', # [String] Command line option (starts with "-")
|
|
24
|
+
'x-cli-short', # [String] Command line option (starts with "-")
|
|
25
|
+
'x-cli-switch', # [Bool] `true` if option has no arg, else by default option has a value
|
|
26
|
+
'x-cli-special', # [Bool] `true` if special handling (deferred)
|
|
27
|
+
'x-cli-convert', # [String,Hash] Method name for Convert object or Conversion for enum ts to arg
|
|
28
|
+
'x-agents', # [Array] Supported agents (for doc only), if not specified: all
|
|
29
|
+
'x-ts-name', # [Bool,String] (async) true if same name in transfer spec, else real name in transfer spec, else ignored
|
|
30
|
+
'x-ts-convert', # [String] (async) Name of methods to convert value from transfer spec to `conf` API.
|
|
31
|
+
'x-deprecation' # [String] Deprecation message for doc
|
|
44
32
|
].freeze
|
|
45
33
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
private_constant :PROPERTY_KEYS, :CLI_AGENT
|
|
34
|
+
private_constant :PROPERTY_KEYS
|
|
49
35
|
|
|
50
36
|
class << self
|
|
51
|
-
#
|
|
37
|
+
# Called by provider of definition before constructor of this class so that schema has all mandatory fields
|
|
38
|
+
def read_schema(folder, name, ascp: false)
|
|
39
|
+
schema = YAML.load_file(File.join(folder, "#{name}.schema.yaml"))
|
|
40
|
+
validate_schema(schema, ascp: ascp)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# @param agent [Symbol] Transfer agent name
|
|
44
|
+
# @param properties [Hash] Transfer spec parameter information
|
|
45
|
+
# @return [Boolean] `true` if given agent supports that field
|
|
52
46
|
def supported_by_agent(agent, properties)
|
|
53
|
-
!properties.key?('x-agents') || properties['x-agents'].include?(agent)
|
|
47
|
+
!properties.key?('x-agents') || properties['x-agents'].include?(agent.to_s)
|
|
54
48
|
end
|
|
55
49
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
# Fill default values for some fields in the schema
|
|
53
|
+
# @param schema [Hash] The JSON schema
|
|
54
|
+
# @param ascp [Bool] `true` if ascp
|
|
55
|
+
def validate_schema(schema, ascp: false)
|
|
56
|
+
direct_props = %w[x-cli-option x-cli-envvar x-cli-special].freeze
|
|
57
|
+
schema['properties'].each do |name, info|
|
|
59
58
|
Aspera.assert_type(info, Hash){"#{info.class} for #{name}"}
|
|
60
59
|
unsupported_keys = info.keys - PROPERTY_KEYS
|
|
61
60
|
Aspera.assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
|
|
62
|
-
|
|
63
|
-
info['type']
|
|
64
|
-
|
|
65
|
-
|
|
61
|
+
Aspera.assert(info.key?('type') || info.key?('enum')){"Missing type for #{name} in #{schema['description']}"}
|
|
62
|
+
Aspera.assert(info['type'].eql?('boolean')){"switch must be bool: #{name}"} if info['x-cli-switch'] && !info['x-cli-special']
|
|
63
|
+
info['x-cli-option'] = "--#{name.to_s.tr('_', '-')}" if info['x-cli-option'].eql?(true) || (info['x-cli-switch'].eql?(true) && !info.key?('x-cli-option'))
|
|
64
|
+
Aspera.assert(direct_props.any?{ |i| info.key?(i)}, type: :warn){name} if ascp && supported_by_agent(:direct, info)
|
|
66
65
|
info.freeze
|
|
66
|
+
validate_schema(info, ascp: ascp) if info['type'].eql?('object') && info['properties']
|
|
67
|
+
validate_schema(info['items'], ascp: ascp) if info['type'].eql?('array') && info['items'] && info['items']['properties']
|
|
67
68
|
end
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
# Called by provider of definition before constructor of this class so that schema has all mandatory fields
|
|
71
|
-
def read_schema(source_path, name)
|
|
72
|
-
YAML.load_file(File.join(File.dirname(source_path), "#{name}.schema.yaml"))
|
|
69
|
+
schema
|
|
73
70
|
end
|
|
74
71
|
end
|
|
75
72
|
|
|
@@ -98,10 +95,10 @@ module Aspera
|
|
|
98
95
|
# Add processed parameters to env and args, warns about unused parameters
|
|
99
96
|
# @param [Hash] env_args with :env and :args
|
|
100
97
|
def add_env_args(env_args)
|
|
101
|
-
Log.
|
|
98
|
+
Log.dump(:env_args, @result)
|
|
102
99
|
# warn about non translated arguments
|
|
103
100
|
@object.each_pair do |name, value|
|
|
104
|
-
Log.log.warn{
|
|
101
|
+
Log.log.warn{"Unknown transfer spec parameter: #{name} = \"#{value}\""} unless @processed_parameters.include?(name)
|
|
105
102
|
end
|
|
106
103
|
# set result
|
|
107
104
|
env_args[:env].merge!(@result[:env])
|
|
@@ -109,9 +106,10 @@ module Aspera
|
|
|
109
106
|
return
|
|
110
107
|
end
|
|
111
108
|
|
|
112
|
-
#
|
|
113
|
-
def add_command_line_options(options)
|
|
114
|
-
|
|
109
|
+
# Add options directly to command line
|
|
110
|
+
def add_command_line_options(*options)
|
|
111
|
+
options = options.first if options.first.is_a?(Array) && options.length.eql?(1)
|
|
112
|
+
Aspera.assert_type(options, Array)
|
|
115
113
|
options.each{ |o| @result[:args].push(o.to_s)}
|
|
116
114
|
end
|
|
117
115
|
|
|
@@ -178,7 +176,7 @@ module Aspera
|
|
|
178
176
|
parameter_value = converted_value
|
|
179
177
|
end
|
|
180
178
|
|
|
181
|
-
return unless self.class.supported_by_agent(
|
|
179
|
+
return unless self.class.supported_by_agent(:direct, properties)
|
|
182
180
|
|
|
183
181
|
if read
|
|
184
182
|
# just get value (deferred)
|
|
@@ -198,13 +196,13 @@ module Aspera
|
|
|
198
196
|
else Aspera.error_unexpected_value(parameter_value){name}
|
|
199
197
|
end
|
|
200
198
|
# add_param = !add_param if properties[:add_on_false]
|
|
201
|
-
add_command_line_options(
|
|
199
|
+
add_command_line_options(properties['x-cli-option']) if add_param
|
|
202
200
|
else
|
|
203
201
|
# transform into command line option with value
|
|
204
202
|
# parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
|
|
205
203
|
parameter_value = [parameter_value] unless parameter_value.is_a?(Array)
|
|
206
204
|
# if transfer_spec value is an array, applies option many times
|
|
207
|
-
parameter_value.each{ |v| add_command_line_options(
|
|
205
|
+
parameter_value.each{ |v| add_command_line_options(properties['x-cli-option'], v)}
|
|
208
206
|
end
|
|
209
207
|
end
|
|
210
208
|
end
|
data/lib/aspera/coverage.rb
CHANGED
|
@@ -5,10 +5,10 @@ if ENV.key?('ENABLE_COVERAGE')
|
|
|
5
5
|
require 'simplecov'
|
|
6
6
|
require 'securerandom'
|
|
7
7
|
# compute development top folder based on this source location
|
|
8
|
-
development_root =
|
|
8
|
+
development_root = File.dirname(File.realpath(__FILE__), 3)
|
|
9
9
|
SimpleCov.root(development_root)
|
|
10
10
|
SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
|
|
11
|
-
# keep cache data for 1 day (must be longer
|
|
11
|
+
# keep cache data for 1 day (must be longer than time to run the whole test suite)
|
|
12
12
|
SimpleCov.merge_timeout(86400)
|
|
13
13
|
SimpleCov.command_name(SecureRandom.uuid)
|
|
14
14
|
SimpleCov.at_exit do
|
|
@@ -19,7 +19,7 @@ module Aspera
|
|
|
19
19
|
# @return [String] decoded data
|
|
20
20
|
def item(name)
|
|
21
21
|
index = ELEMENTS.index(name)
|
|
22
|
-
raise
|
|
22
|
+
raise ParameterError, "Unknown data item #{name} (#{name.class})" unless index
|
|
23
23
|
raw_data = data(START_INDEX + index)
|
|
24
24
|
case name
|
|
25
25
|
when :dsa, :rsa
|
data/lib/aspera/environment.rb
CHANGED
|
@@ -5,7 +5,9 @@ require 'aspera/log'
|
|
|
5
5
|
require 'aspera/assert'
|
|
6
6
|
require 'rbconfig'
|
|
7
7
|
require 'singleton'
|
|
8
|
+
require 'open3'
|
|
8
9
|
require 'English'
|
|
10
|
+
require 'shellwords'
|
|
9
11
|
|
|
10
12
|
# cspell:words MEBI mswin bccwin
|
|
11
13
|
|
|
@@ -38,6 +40,8 @@ module Aspera
|
|
|
38
40
|
WINDOWS_FILENAME_INVALID_CHARACTERS = '<>:"/\\|?*'
|
|
39
41
|
REPLACE_CHARACTER = '_'
|
|
40
42
|
|
|
43
|
+
RB_EXT = '.rb'
|
|
44
|
+
|
|
41
45
|
class << self
|
|
42
46
|
def ruby_version
|
|
43
47
|
return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
|
|
@@ -54,9 +58,9 @@ module Aspera
|
|
|
54
58
|
end
|
|
55
59
|
|
|
56
60
|
# Generate log line for external program with arguments
|
|
57
|
-
# @param
|
|
58
|
-
# @param
|
|
59
|
-
# @param
|
|
61
|
+
# @param exec [String] Path to executable
|
|
62
|
+
# @param args [Array, nil] Arguments
|
|
63
|
+
# @param env [Hash, nil] Environment variables
|
|
60
64
|
# @return [String] log line with environment, program and arguments
|
|
61
65
|
def log_spawn(exec:, args: nil, env: nil)
|
|
62
66
|
[
|
|
@@ -69,65 +73,70 @@ module Aspera
|
|
|
69
73
|
|
|
70
74
|
# Start process in background
|
|
71
75
|
# caller can call Process.wait on returned value
|
|
72
|
-
# @param exec [String]
|
|
73
|
-
# @param args [Array, nil]
|
|
74
|
-
# @param env [Hash, nil]
|
|
75
|
-
# @param
|
|
76
|
+
# @param exec [String] Path to executable
|
|
77
|
+
# @param args [Array, nil] Arguments for executable
|
|
78
|
+
# @param env [Hash, nil] Environment variables
|
|
79
|
+
# @param kwargs [Hash] Options for `Process.spawn`
|
|
76
80
|
# @return [String] PID of process
|
|
77
81
|
# @raise [Exception] if problem
|
|
78
|
-
def secure_spawn(exec:, args: nil, env: nil, **
|
|
82
|
+
def secure_spawn(exec:, args: nil, env: nil, **kwargs)
|
|
79
83
|
Aspera.assert_type(exec, String)
|
|
80
84
|
Aspera.assert_type(args, Array, NilClass)
|
|
81
85
|
Aspera.assert_type(env, Hash, NilClass)
|
|
82
|
-
Aspera.assert_type(options, Hash, NilClass)
|
|
83
86
|
Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
|
|
84
|
-
# start ascp in separate process
|
|
85
87
|
spawn_args = []
|
|
86
88
|
spawn_args.push(env) unless env.nil?
|
|
87
89
|
spawn_args.push([exec, exec])
|
|
88
90
|
spawn_args.concat(args) unless args.nil?
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
Log.
|
|
93
|
-
return
|
|
91
|
+
kwargs[:close_others] = true unless kwargs.key?(:close_others)
|
|
92
|
+
# Start separate process in background
|
|
93
|
+
pid = Process.spawn(*spawn_args, **kwargs)
|
|
94
|
+
Log.dump(:pid, pid)
|
|
95
|
+
return pid
|
|
94
96
|
end
|
|
95
97
|
|
|
96
|
-
#
|
|
97
|
-
#
|
|
98
|
-
# @param exec
|
|
99
|
-
# @param args
|
|
100
|
-
# @
|
|
101
|
-
|
|
98
|
+
# Start process (not in shell) and wait for completion.
|
|
99
|
+
# By default, sets `exception: true` in `kwargs`
|
|
100
|
+
# @param exec [String] Path to executable
|
|
101
|
+
# @param args [Array, nil] Arguments
|
|
102
|
+
# @param env [Hash, nil] Environment variables
|
|
103
|
+
# @param kwargs [Hash] Arguments for `Kernel.system`
|
|
104
|
+
# @return `nil`
|
|
105
|
+
# @raise [RuntimeError] if problem
|
|
106
|
+
def secure_execute(exec:, args: nil, env: nil, **kwargs)
|
|
102
107
|
Aspera.assert_type(exec, String)
|
|
103
108
|
Aspera.assert_type(args, Array, NilClass)
|
|
104
109
|
Aspera.assert_type(env, Hash, NilClass)
|
|
105
110
|
Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
|
|
106
|
-
|
|
111
|
+
Log.dump(:kwargs, kwargs, level: :trace1)
|
|
107
112
|
spawn_args = []
|
|
108
113
|
spawn_args.push(env) unless env.nil?
|
|
109
|
-
#
|
|
114
|
+
# Ensure no shell expansion
|
|
110
115
|
spawn_args.push([exec, exec])
|
|
111
116
|
spawn_args.concat(args) unless args.nil?
|
|
112
|
-
|
|
113
|
-
kwargs.
|
|
117
|
+
# By default: exception on error
|
|
118
|
+
kwargs[:exception] = true unless kwargs.key?(:exception)
|
|
119
|
+
# Start in separate process
|
|
114
120
|
Kernel.system(*spawn_args, **kwargs)
|
|
115
121
|
nil
|
|
116
122
|
end
|
|
117
123
|
|
|
118
124
|
# Execute process and capture stdout
|
|
119
|
-
# @param exec
|
|
120
|
-
# @param args
|
|
121
|
-
# @param
|
|
125
|
+
# @param exec [String] path to executable
|
|
126
|
+
# @param args [Array] arguments to executable
|
|
127
|
+
# @param kwargs [Hash] options to Open3.capture3
|
|
122
128
|
# @return stdout of executable or raise exception
|
|
123
|
-
def secure_capture(exec:, args: [], exception: true, **
|
|
129
|
+
def secure_capture(exec:, args: [], env: nil, exception: true, **kwargs)
|
|
124
130
|
Aspera.assert_type(exec, String)
|
|
125
131
|
Aspera.assert_type(args, Array)
|
|
126
|
-
|
|
127
|
-
Log.
|
|
128
|
-
Log.dump(:
|
|
129
|
-
|
|
130
|
-
|
|
132
|
+
Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
|
|
133
|
+
Log.dump(:kwargs, kwargs, level: :trace2)
|
|
134
|
+
# Log.dump(:ENV, ENV.to_h, level: :trace1)
|
|
135
|
+
capture_args = []
|
|
136
|
+
capture_args.push(env) unless env.nil?
|
|
137
|
+
capture_args.push(exec)
|
|
138
|
+
capture_args.concat(args)
|
|
139
|
+
stdout, stderr, status = Open3.capture3(*capture_args, **kwargs)
|
|
131
140
|
Log.log.debug{"status=#{status}, stderr=#{stderr}"}
|
|
132
141
|
Log.log.trace1{"stdout=#{stdout}"}
|
|
133
142
|
raise "process failed: #{status.exitstatus} (#{stderr})" if !status.success? && exception
|
|
@@ -186,7 +195,7 @@ module Aspera
|
|
|
186
195
|
end
|
|
187
196
|
end
|
|
188
197
|
attr_accessor :url_method, :file_illegal_characters
|
|
189
|
-
attr_reader :os, :cpu, :
|
|
198
|
+
attr_reader :os, :cpu, :default_gui_mode
|
|
190
199
|
|
|
191
200
|
def initialize
|
|
192
201
|
initialize_fields
|
|
@@ -218,7 +227,7 @@ module Aspera
|
|
|
218
227
|
CPU_ARM64
|
|
219
228
|
else Aspera.error_unexpected_value(RbConfig::CONFIG['host_cpu']){'host_cpu'}
|
|
220
229
|
end
|
|
221
|
-
@executable_extension = @os.eql?(OS_WINDOWS) ? 'exe' : nil
|
|
230
|
+
@executable_extension = @os.eql?(OS_WINDOWS) ? '.exe' : nil
|
|
222
231
|
# :text or :graphical depending on the environment
|
|
223
232
|
@default_gui_mode =
|
|
224
233
|
if [Environment::OS_WINDOWS, Environment::OS_MACOS].include?(os) ||
|
|
@@ -239,8 +248,10 @@ module Aspera
|
|
|
239
248
|
"#{@os}-#{@cpu}"
|
|
240
249
|
end
|
|
241
250
|
|
|
242
|
-
# executable file extension for current OS
|
|
243
|
-
|
|
251
|
+
# Add executable file extension (e.g. ".exe") for current OS
|
|
252
|
+
# @param name [String,nil] Path or file name
|
|
253
|
+
# @return [String] Executable name with extension
|
|
254
|
+
def exe_file(name = nil)
|
|
244
255
|
return name unless @executable_extension
|
|
245
256
|
return "#{name}#{@executable_extension}"
|
|
246
257
|
end
|
|
@@ -314,9 +325,8 @@ module Aspera
|
|
|
314
325
|
# Windows does not allow file name:
|
|
315
326
|
# - with control characters anywhere
|
|
316
327
|
# - ending with space or dot
|
|
317
|
-
filename = filename
|
|
318
|
-
|
|
319
|
-
.sub(/[. ]+\z/, safe_char)
|
|
328
|
+
filename = filename.gsub(/[\x00-\x1F\x7F]/, safe_char)
|
|
329
|
+
filename = filename.chop while filename.end_with?(' ', '.')
|
|
320
330
|
if @file_illegal_characters&.size.to_i >= 2
|
|
321
331
|
# replace all illegal characters with safe_char
|
|
322
332
|
filename = filename.tr(@file_illegal_characters[1..-1], safe_char)
|
data/lib/aspera/faspex_gw.rb
CHANGED
|
@@ -8,10 +8,12 @@ require 'json'
|
|
|
8
8
|
module Aspera
|
|
9
9
|
# Simulate the Faspex 4 /send API and creates a package on Aspera on Cloud or Faspex 5
|
|
10
10
|
class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
|
|
11
|
+
AOC_API = 'Aspera::Api::AoC'
|
|
12
|
+
FX_API = 'Aspera::Api::Faspex'
|
|
11
13
|
# @param app_api [Rest] API object
|
|
12
14
|
# @param app_context [String] workspace id (aoc only)
|
|
13
15
|
def initialize(server, app_api, app_context)
|
|
14
|
-
Aspera.assert_values(app_api.class.name, [
|
|
16
|
+
Aspera.assert_values(app_api.class.name, [AOC_API, FX_API])
|
|
15
17
|
super(server)
|
|
16
18
|
@app_api = app_api
|
|
17
19
|
@app_context = app_context
|
|
@@ -49,11 +51,11 @@ module Aspera
|
|
|
49
51
|
transfer_spec = @app_api.call(
|
|
50
52
|
operation: 'POST',
|
|
51
53
|
subpath: "packages/#{package['id']}/transfer_spec/upload",
|
|
52
|
-
query: {transfer_type:
|
|
54
|
+
query: {transfer_type: Api::Faspex::TRANSFER_CONNECT},
|
|
53
55
|
content_type: Rest::MIME_JSON,
|
|
54
56
|
body: {paths: [{'destination'=>'/'}]},
|
|
55
57
|
headers: {'Accept' => Rest::MIME_JSON}
|
|
56
|
-
)
|
|
58
|
+
)
|
|
57
59
|
transfer_spec.delete('authentication')
|
|
58
60
|
# but we place it in a Faspex package creation response
|
|
59
61
|
return {
|
|
@@ -72,9 +74,9 @@ module Aspera
|
|
|
72
74
|
# compare string, as class is not yet known here
|
|
73
75
|
faspex_package_create_result =
|
|
74
76
|
case @app_api.class.name
|
|
75
|
-
when
|
|
77
|
+
when AOC_API
|
|
76
78
|
faspex4_send_to_aoc(faspex_pkg_parameters)
|
|
77
|
-
when
|
|
79
|
+
when FX_API
|
|
78
80
|
faspex4_send_to_faspex5(faspex_pkg_parameters)
|
|
79
81
|
else Aspera.error_unexpected_value(@app_api.class.name)
|
|
80
82
|
end
|