aspera-cli 4.21.2 → 4.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +1 -1
- data/CHANGELOG.md +34 -16
- data/CONTRIBUTING.md +6 -10
- data/README.md +805 -574
- data/examples/get_proto_file.rb +1 -1
- data/lib/aspera/agent/base.rb +9 -5
- data/lib/aspera/agent/connect.rb +30 -28
- data/lib/aspera/agent/desktop.rb +29 -25
- data/lib/aspera/agent/direct.rb +137 -125
- data/lib/aspera/agent/httpgw.rb +22 -26
- data/lib/aspera/agent/node.rb +14 -11
- data/lib/aspera/agent/transferd.rb +6 -2
- data/lib/aspera/api/aoc.rb +6 -6
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/httpgw.rb +7 -3
- data/lib/aspera/api/node.rb +6 -4
- data/lib/aspera/ascmd.rb +3 -3
- data/lib/aspera/ascp/installation.rb +15 -16
- data/lib/aspera/ascp/management.rb +1 -1
- data/lib/aspera/assert.rb +11 -2
- data/lib/aspera/cli/error.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +38 -19
- data/lib/aspera/cli/formatter.rb +48 -48
- data/lib/aspera/cli/hints.rb +1 -1
- data/lib/aspera/cli/main.rb +190 -168
- data/lib/aspera/cli/manager.rb +15 -15
- data/lib/aspera/cli/plugin.rb +23 -20
- data/lib/aspera/cli/plugin_factory.rb +1 -1
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +144 -107
- data/lib/aspera/cli/plugins/ats.rb +19 -17
- data/lib/aspera/cli/plugins/config.rb +67 -83
- data/lib/aspera/cli/plugins/console.rb +5 -3
- data/lib/aspera/cli/plugins/faspex.rb +39 -35
- data/lib/aspera/cli/plugins/faspex5.rb +104 -80
- data/lib/aspera/cli/plugins/faspio.rb +13 -1
- data/lib/aspera/cli/plugins/httpgw.rb +13 -1
- data/lib/aspera/cli/plugins/node.rb +306 -179
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +6 -6
- data/lib/aspera/cli/plugins/shares.rb +5 -5
- data/lib/aspera/cli/sync_actions.rb +19 -18
- data/lib/aspera/cli/transfer_agent.rb +5 -5
- data/lib/aspera/cli/transfer_progress.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +116 -95
- data/lib/aspera/coverage.rb +4 -3
- data/lib/aspera/environment.rb +6 -6
- data/lib/aspera/faspex_gw.rb +14 -14
- data/lib/aspera/faspex_postproc.rb +7 -6
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +47 -34
- data/lib/aspera/keychain/factory.rb +41 -0
- data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
- data/lib/aspera/keychain/macos_security.rb +19 -11
- data/lib/aspera/log.rb +28 -34
- data/lib/aspera/nagios.rb +6 -6
- data/lib/aspera/node_simulator.rb +8 -8
- data/lib/aspera/oauth/base.rb +8 -6
- data/lib/aspera/oauth/factory.rb +5 -6
- data/lib/aspera/oauth/url_json.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +6 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +1 -1
- data/lib/aspera/preview/options.rb +16 -16
- data/lib/aspera/preview/terminal.rb +3 -3
- data/lib/aspera/preview/utils.rb +11 -13
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +2 -2
- data/lib/aspera/rest.rb +52 -43
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/secret_hider.rb +5 -5
- data/lib/aspera/ssh.rb +4 -4
- data/lib/aspera/transfer/convert.rb +29 -0
- data/lib/aspera/transfer/error_info.rb +66 -66
- data/lib/aspera/transfer/parameters.rb +13 -68
- data/lib/aspera/transfer/spec.rb +5 -6
- data/lib/aspera/transfer/spec.schema.yaml +753 -0
- data/lib/aspera/transfer/spec_doc.rb +62 -0
- data/lib/aspera/transfer/sync.rb +23 -72
- data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
- data/lib/aspera/transfer/uri.rb +6 -6
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +53 -44
- data.tar.gz.sig +1 -2
- metadata +37 -4
- metadata.gz.sig +0 -0
- data/examples/build_package.sh +0 -28
- data/lib/aspera/transfer/spec.yaml +0 -718
@@ -6,9 +6,9 @@ require 'aspera/command_line_builder'
|
|
6
6
|
require 'aspera/temp_file_manager'
|
7
7
|
require 'aspera/transfer/error'
|
8
8
|
require 'aspera/transfer/spec'
|
9
|
+
require 'aspera/transfer/convert'
|
9
10
|
require 'aspera/ascp/installation'
|
10
11
|
require 'aspera/cli/formatter'
|
11
|
-
require 'aspera/agent/base'
|
12
12
|
require 'aspera/rest'
|
13
13
|
require 'securerandom'
|
14
14
|
require 'base64'
|
@@ -20,15 +20,10 @@ module Aspera
|
|
20
20
|
module Transfer
|
21
21
|
# translate transfer specification to ascp parameter list
|
22
22
|
class Parameters
|
23
|
-
# Agents shown in manual for parameters (sub list)
|
24
|
-
SUPPORTED_AGENTS = Agent::Base.agent_list.freeze
|
25
23
|
FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
|
26
|
-
|
27
|
-
SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|agent_sym|agent_sym.to_s[0].to_sym}
|
24
|
+
private_constant :FILE_LIST_OPTIONS
|
28
25
|
HTTP_FALLBACK_ACTIVATION_VALUES = ['1', 1, true, 'force'].freeze
|
29
26
|
|
30
|
-
private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
|
31
|
-
|
32
27
|
class << self
|
33
28
|
# temp file list files are created here
|
34
29
|
def file_list_folder=(value)
|
@@ -46,64 +41,14 @@ module Aspera
|
|
46
41
|
@file_list_folder ||= TempFileManager.instance.new_file_path_global('asession_filelists')
|
47
42
|
end
|
48
43
|
|
49
|
-
# @param formatter [Cli::Formatter] formatter to use
|
50
|
-
# @return a table suitable to display in manual
|
51
|
-
def man_table(formatter)
|
52
|
-
result = []
|
53
|
-
Spec::DESCRIPTION.each do |name, options|
|
54
|
-
param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
|
55
|
-
# add flags for supported agents in doc
|
56
|
-
SUPPORTED_AGENTS.each do |agent_sym|
|
57
|
-
param[agent_sym.to_s[0].to_sym] = Cli::Formatter.tick(options[:agents].nil? || options[:agents].include?(agent_sym))
|
58
|
-
end
|
59
|
-
# only keep lines that are usable in supported agents
|
60
|
-
next if SUPPORTED_AGENTS_SHORT.inject(true){|memory, agent_short_sym|memory && param[agent_short_sym].empty?}
|
61
|
-
param[:cli] =
|
62
|
-
case options[:cli][:type]
|
63
|
-
when :envvar then 'env:' + options[:cli][:variable]
|
64
|
-
when :opt_without_arg then options[:cli][:switch]
|
65
|
-
when :opt_with_arg
|
66
|
-
values = if options.key?(:enum)
|
67
|
-
['enum']
|
68
|
-
elsif options[:accepted_types].is_a?(Array)
|
69
|
-
options[:accepted_types]
|
70
|
-
elsif !options[:accepted_types].nil?
|
71
|
-
[options[:accepted_types]]
|
72
|
-
else
|
73
|
-
raise "error: #{param}"
|
74
|
-
end.map{|n|"{#{n}}"}.join('|')
|
75
|
-
conversion_tag = options[:cli].key?(:convert) ? '(conversion)' : ''
|
76
|
-
"#{options[:cli][:switch]} #{conversion_tag}#{values}"
|
77
|
-
when :special then formatter.special_format('special')
|
78
|
-
when :ignore then formatter.special_format('ignored')
|
79
|
-
else
|
80
|
-
param[:d].eql?(tick_yes) ? '' : 'n/a'
|
81
|
-
end
|
82
|
-
param[:description] += "\nAllowed values: #{options[:enum].join(', ')}" if options.key?(:enum)
|
83
|
-
# replace "solidus" HTML entity with its text value
|
84
|
-
param[:description] = param[:description].gsub('/', '\\')
|
85
|
-
result.push(param)
|
86
|
-
end
|
87
|
-
return result.sort_by { |parameter_info| parameter_info[:name] }
|
88
|
-
end
|
89
|
-
|
90
|
-
# special encoding methods used in YAML (key: :convert)
|
91
|
-
def convert_remove_hyphen(value); value.tr('-', ''); end
|
92
|
-
|
93
|
-
# special encoding methods used in YAML (key: :convert)
|
94
|
-
def convert_json64(value); Base64.strict_encode64(JSON.generate(value)); end
|
95
|
-
|
96
|
-
# special encoding methods used in YAML (key: :convert)
|
97
|
-
def convert_base64(value); Base64.strict_encode64(value); end
|
98
|
-
|
99
44
|
# file list is provided directly with ascp arguments
|
100
|
-
# @
|
45
|
+
# @columns ascp_args [Array,NilClass] ascp arguments
|
101
46
|
def ascp_args_file_list?(ascp_args)
|
102
|
-
ascp_args&.any?{|i|FILE_LIST_OPTIONS.include?(i)}
|
47
|
+
ascp_args&.any?{ |i| FILE_LIST_OPTIONS.include?(i)}
|
103
48
|
end
|
104
49
|
end
|
105
50
|
|
106
|
-
# @
|
51
|
+
# @columns options [Hash] key: :wss: bool, :ascp_args: array of strings
|
107
52
|
def initialize(
|
108
53
|
job_spec,
|
109
54
|
ascp_args: nil,
|
@@ -125,7 +70,7 @@ module Aspera
|
|
125
70
|
Aspera.assert(@ascp_args.all?(String)){'all ascp arguments must be String'}
|
126
71
|
Aspera.assert_type(@trusted_certs, Array){'trusted_certs'}
|
127
72
|
Aspera.assert_values(@client_ssh_key, Ascp::Installation::CLIENT_SSH_KEY_OPTIONS)
|
128
|
-
@builder = CommandLineBuilder.new(@job_spec, Spec::
|
73
|
+
@builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, Convert)
|
129
74
|
end
|
130
75
|
|
131
76
|
# either place source files on command line, or add file list file
|
@@ -133,29 +78,29 @@ module Aspera
|
|
133
78
|
# is the file list provided through ascp parameters?
|
134
79
|
ascp_file_list_provided = self.class.ascp_args_file_list?(@ascp_args)
|
135
80
|
# set if paths is mandatory in ts
|
136
|
-
@builder.
|
81
|
+
@builder.required('paths', !@job_spec.key?('keepalive') && !ascp_file_list_provided) # cspell:words keepalive
|
137
82
|
# get paths in transfer spec (after setting if it is mandatory)
|
138
83
|
ts_paths_array = @builder.read_param('paths')
|
139
84
|
file_list_option = nil
|
140
85
|
# transfer spec contains paths ?
|
141
86
|
if !ts_paths_array.nil?
|
142
87
|
Aspera.assert(!ascp_file_list_provided){'file list provided both in transfer spec and ascp file list. Remove one of them.'}
|
143
|
-
Aspera.assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
|
144
|
-
is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
|
145
|
-
raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
|
88
|
+
Aspera.assert(ts_paths_array.all?{ |i| i.key?('source')}){"All elements of paths must have a 'source' key"}
|
89
|
+
is_pair_list = ts_paths_array.any?{ |i| i.key?('destination')}
|
90
|
+
raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{ |i| i.key?('destination')}
|
146
91
|
if self.class.file_list_folder.nil?
|
147
92
|
Aspera.assert(!is_pair_list){'file pair list is not supported when file list folder is not set'}
|
148
93
|
# not safe for special characters ? (maybe not, depends on OS)
|
149
94
|
Log.log.debug('placing source file list on command line (no file list file)')
|
150
|
-
@builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
|
95
|
+
@builder.add_command_line_options(ts_paths_array.map{ |i| i['source']})
|
151
96
|
else
|
152
97
|
# safer option: generate a file list file if there is storage defined for it
|
153
98
|
if is_pair_list
|
154
99
|
file_list_option = '--file-pair-list'
|
155
|
-
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'])
|
100
|
+
lines = ts_paths_array.each_with_object([]){ |e, m| m.push(e['source'], e['destination'])}
|
156
101
|
else
|
157
102
|
file_list_option = '--file-list'
|
158
|
-
lines = ts_paths_array.map{|i|i['source']}
|
103
|
+
lines = ts_paths_array.map{ |i| i['source']}
|
159
104
|
end
|
160
105
|
file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
161
106
|
Log.log.debug{Log.dump(:file_list, lines)}
|
data/lib/aspera/transfer/spec.rb
CHANGED
@@ -2,7 +2,6 @@
|
|
2
2
|
|
3
3
|
require 'aspera/command_line_builder'
|
4
4
|
require 'aspera/assert'
|
5
|
-
require 'yaml'
|
6
5
|
|
7
6
|
module Aspera
|
8
7
|
module Transfer
|
@@ -36,12 +35,12 @@ module Aspera
|
|
36
35
|
XFER_DIR_TO_TYPE.fetch(direction)
|
37
36
|
end
|
38
37
|
end
|
39
|
-
|
38
|
+
SCHEMA = CommandLineBuilder.read_schema(__FILE__)
|
40
39
|
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
|
41
|
-
|
42
|
-
next unless description[
|
43
|
-
const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description[
|
44
|
-
description[
|
40
|
+
SCHEMA['properties'].each do |name, description|
|
41
|
+
next unless description['enum'].is_a?(Array)
|
42
|
+
const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description['enum'])
|
43
|
+
description['enum'].each do |enum|
|
45
44
|
const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
46
45
|
end
|
47
46
|
end
|