aspera-cli 4.24.2 → 4.25.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1064 -758
- data/CONTRIBUTING.md +43 -100
- data/README.md +671 -419
- data/lib/aspera/api/aoc.rb +71 -43
- data/lib/aspera/api/cos_node.rb +3 -2
- data/lib/aspera/api/faspex.rb +6 -5
- data/lib/aspera/api/node.rb +10 -12
- data/lib/aspera/ascmd.rb +1 -2
- data/lib/aspera/ascp/installation.rb +53 -39
- data/lib/aspera/assert.rb +25 -3
- data/lib/aspera/cli/error.rb +4 -2
- data/lib/aspera/cli/extended_value.rb +84 -60
- data/lib/aspera/cli/formatter.rb +55 -22
- data/lib/aspera/cli/main.rb +21 -14
- data/lib/aspera/cli/manager.rb +348 -247
- data/lib/aspera/cli/plugins/alee.rb +3 -3
- data/lib/aspera/cli/plugins/aoc.rb +70 -14
- data/lib/aspera/cli/plugins/base.rb +57 -49
- data/lib/aspera/cli/plugins/config.rb +69 -84
- data/lib/aspera/cli/plugins/console.rb +13 -8
- data/lib/aspera/cli/plugins/cos.rb +1 -1
- data/lib/aspera/cli/plugins/faspex.rb +32 -26
- data/lib/aspera/cli/plugins/faspex5.rb +45 -43
- data/lib/aspera/cli/plugins/faspio.rb +5 -5
- data/lib/aspera/cli/plugins/httpgw.rb +1 -1
- data/lib/aspera/cli/plugins/node.rb +131 -120
- data/lib/aspera/cli/plugins/oauth.rb +1 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +114 -32
- data/lib/aspera/cli/plugins/preview.rb +26 -46
- data/lib/aspera/cli/plugins/server.rb +6 -8
- data/lib/aspera/cli/plugins/shares.rb +27 -32
- data/lib/aspera/cli/sync_actions.rb +49 -38
- data/lib/aspera/cli/transfer_agent.rb +16 -34
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +8 -5
- data/lib/aspera/command_line_builder.rb +20 -17
- data/lib/aspera/coverage.rb +1 -1
- data/lib/aspera/environment.rb +41 -34
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/keychain/factory.rb +1 -2
- data/lib/aspera/markdown.rb +31 -0
- data/lib/aspera/nagios.rb +6 -5
- data/lib/aspera/oauth/base.rb +17 -27
- data/lib/aspera/oauth/factory.rb +1 -1
- data/lib/aspera/oauth/url_json.rb +2 -1
- data/lib/aspera/preview/file_types.rb +23 -37
- data/lib/aspera/products/connect.rb +3 -3
- data/lib/aspera/rest.rb +51 -39
- data/lib/aspera/rest_error_analyzer.rb +4 -4
- data/lib/aspera/ssh.rb +5 -2
- data/lib/aspera/ssl.rb +41 -0
- data/lib/aspera/sync/conf.schema.yaml +182 -34
- data/lib/aspera/sync/database.rb +2 -1
- data/lib/aspera/sync/operations.rb +125 -69
- data/lib/aspera/transfer/parameters.rb +3 -4
- data/lib/aspera/transfer/spec.rb +2 -3
- data/lib/aspera/transfer/spec.schema.yaml +48 -18
- data/lib/aspera/transfer/spec_doc.rb +14 -14
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/transferd_pb.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +19 -6
- metadata.gz.sig +3 -2
|
@@ -7,7 +7,7 @@ require 'pathname'
|
|
|
7
7
|
|
|
8
8
|
module Aspera
|
|
9
9
|
module Cli
|
|
10
|
-
# Manage command line arguments to provide to Sync::
|
|
10
|
+
# Manage command line arguments to provide to Sync::Operations and Sync::Database
|
|
11
11
|
module SyncActions
|
|
12
12
|
# Translate state id (int) to string
|
|
13
13
|
STATE_STR = (['Nil'] +
|
|
@@ -19,15 +19,19 @@ module Aspera
|
|
|
19
19
|
end
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
# Read command line arguments
|
|
23
|
-
#
|
|
22
|
+
# Read 1 or 2 command line arguments and converts to `sync_info` format
|
|
23
|
+
# The resulting sync_info has `args` format only if it contains one of the `sessions` or `instance` keys.
|
|
24
|
+
# It has the `conf` format (default) otherwise.
|
|
25
|
+
# If the `conf` format is detected, then both `local` and `remote` keys are set.
|
|
26
|
+
# @param direction [Symbol,NilClass] One of directions, or `nil` if only for admin command
|
|
24
27
|
# @return [Hash] sync info
|
|
25
28
|
def async_info_from_args(direction: nil)
|
|
26
29
|
path = options.get_next_argument('path')
|
|
27
30
|
sync_info = options.get_next_argument('sync info', mandatory: false, validation: Hash, default: {})
|
|
31
|
+
# is the positional path a remote path ?
|
|
28
32
|
path_is_remote = direction.eql?(:pull)
|
|
29
33
|
if sync_info.key?('sessions') || sync_info.key?('instance')
|
|
30
|
-
#
|
|
34
|
+
# `args`
|
|
31
35
|
sync_info['sessions'] ||= [{}]
|
|
32
36
|
Aspera.assert(sync_info['sessions'].length == 1){'Only one session is supported'}
|
|
33
37
|
session = sync_info['sessions'].first
|
|
@@ -41,7 +45,7 @@ module Aspera
|
|
|
41
45
|
local_remote = %w[local remote].map{ |i| session["#{i}_dir"]}
|
|
42
46
|
end
|
|
43
47
|
else
|
|
44
|
-
#
|
|
48
|
+
# `conf`
|
|
45
49
|
session = sync_info
|
|
46
50
|
dir_key = path_is_remote ? 'remote' : 'local'
|
|
47
51
|
session[dir_key] ||= {}
|
|
@@ -54,7 +58,7 @@ module Aspera
|
|
|
54
58
|
session[dir_key]['path'] = transfer.destination_folder(path_is_remote ? Transfer::Spec::DIRECTION_RECEIVE : Transfer::Spec::DIRECTION_SEND)
|
|
55
59
|
local_remote = %w[local remote].map{ |i| session[i]['path']}
|
|
56
60
|
end
|
|
57
|
-
#
|
|
61
|
+
# `conf` is quiet by default
|
|
58
62
|
session['quiet'] = false if !session.key?('quiet') && Environment.terminal?
|
|
59
63
|
end
|
|
60
64
|
if direction
|
|
@@ -62,17 +66,20 @@ module Aspera
|
|
|
62
66
|
session['direction'] = direction.to_s
|
|
63
67
|
# generate name if not provided by user
|
|
64
68
|
if !session.key?('name')
|
|
69
|
+
safe_char = Environment.instance.safe_filename_character
|
|
70
|
+
# from async man page:
|
|
71
|
+
# -N : can contain only ASCII alphanumeric, hyphen, and underscore characters
|
|
65
72
|
session['name'] = Environment.instance.sanitized_filename(
|
|
66
73
|
([direction.to_s] + local_remote).map do |value|
|
|
67
|
-
Pathname(value).each_filename.to_a.last(2).join(
|
|
68
|
-
end.join(
|
|
74
|
+
Pathname(value).each_filename.to_a.last(2).join(safe_char)
|
|
75
|
+
end.join(safe_char).gsub(/[^A-Za-z0-9_-]/, safe_char)
|
|
69
76
|
)
|
|
70
77
|
end
|
|
71
78
|
end
|
|
72
79
|
sync_info
|
|
73
80
|
end
|
|
74
81
|
|
|
75
|
-
#
|
|
82
|
+
# Provide database object from command line arguments for admin ops
|
|
76
83
|
def db_from_args
|
|
77
84
|
sync_info = async_info_from_args
|
|
78
85
|
session = sync_info.key?('sessions') ? sync_info['sessions'].first : sync_info
|
|
@@ -86,43 +93,47 @@ module Aspera
|
|
|
86
93
|
Sync::Database.new(Sync::Operations.session_db_file(sync_info))
|
|
87
94
|
end
|
|
88
95
|
|
|
96
|
+
def execute_sync_admin
|
|
97
|
+
command2 = options.get_next_command(%i[status find meta counters file_info overview])
|
|
98
|
+
require 'aspera/sync/database' unless command2.eql?(:status)
|
|
99
|
+
case command2
|
|
100
|
+
when :status
|
|
101
|
+
return Main.result_single_object(Sync::Operations.admin_status(async_info_from_args))
|
|
102
|
+
when :find
|
|
103
|
+
folder = options.get_next_argument('path')
|
|
104
|
+
dbs = Sync::Operations.list_db_files(folder)
|
|
105
|
+
return Main.result_object_list(dbs.keys.map{ |n| {name: n, path: dbs[n]}})
|
|
106
|
+
when :meta, :counters
|
|
107
|
+
return Main.result_single_object(db_from_args.send(command2))
|
|
108
|
+
when :file_info
|
|
109
|
+
result = db_from_args.send(command2)
|
|
110
|
+
result.each do |r|
|
|
111
|
+
r['sstate'] = SyncActions::STATE_STR[r['state']] if r['state']
|
|
112
|
+
end
|
|
113
|
+
return Main.result_object_list(
|
|
114
|
+
result,
|
|
115
|
+
fields: %w[sstate record_id f_meta_path message]
|
|
116
|
+
)
|
|
117
|
+
when :overview
|
|
118
|
+
return Main.result_object_list(
|
|
119
|
+
db_from_args.overview,
|
|
120
|
+
fields: %w[table name type]
|
|
121
|
+
)
|
|
122
|
+
else Aspera.error_unexpected_value(command2)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
89
126
|
# Execute sync action
|
|
90
|
-
# @param &block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir
|
|
127
|
+
# @param &block [nil, Proc] block to generate transfer spec, takes: `direction` (one of DIRECTIONS), `local_dir`, `remote_dir`
|
|
91
128
|
def execute_sync_action(&block)
|
|
92
129
|
command = options.get_next_command(%i[admin] + Sync::Operations::DIRECTIONS)
|
|
93
130
|
# try to get 3 arguments as simple arguments
|
|
94
131
|
case command
|
|
95
132
|
when *Sync::Operations::DIRECTIONS
|
|
96
|
-
Sync::Operations.start(async_info_from_args(direction: command), transfer.
|
|
133
|
+
Sync::Operations.start(async_info_from_args(direction: command), transfer.user_transfer_spec, &block)
|
|
97
134
|
return Main.result_success
|
|
98
135
|
when :admin
|
|
99
|
-
|
|
100
|
-
require 'aspera/sync/database' unless command2.eql?(:status)
|
|
101
|
-
case command2
|
|
102
|
-
when :status
|
|
103
|
-
return Main.result_single_object(Sync::Operations.admin_status(async_info_from_args))
|
|
104
|
-
when :find
|
|
105
|
-
folder = options.get_next_argument('path')
|
|
106
|
-
dbs = Sync::Operations.list_db_files(folder)
|
|
107
|
-
return Main.result_object_list(dbs.keys.map{ |n| {name: n, path: dbs[n]}})
|
|
108
|
-
when :meta, :counters
|
|
109
|
-
return Main.result_single_object(db_from_args.send(command2))
|
|
110
|
-
when :file_info
|
|
111
|
-
result = db_from_args.send(command2)
|
|
112
|
-
result.each do |r|
|
|
113
|
-
r['sstate'] = SyncActions::STATE_STR[r['state']] if r['state']
|
|
114
|
-
end
|
|
115
|
-
return Main.result_object_list(
|
|
116
|
-
result,
|
|
117
|
-
fields: %w[sstate record_id f_meta_path message]
|
|
118
|
-
)
|
|
119
|
-
when :overview
|
|
120
|
-
return Main.result_object_list(
|
|
121
|
-
db_from_args.overview,
|
|
122
|
-
fields: %w[table name type]
|
|
123
|
-
)
|
|
124
|
-
else Aspera.error_unexpected_value(command2)
|
|
125
|
-
end
|
|
136
|
+
return execute_sync_admin
|
|
126
137
|
else Aspera.error_unexpected_value(command)
|
|
127
138
|
end
|
|
128
139
|
end
|
|
@@ -48,8 +48,8 @@ module Aspera
|
|
|
48
48
|
def initialize(opt_mgr, config_plugin)
|
|
49
49
|
@opt_mgr = opt_mgr
|
|
50
50
|
@config = config_plugin
|
|
51
|
-
#
|
|
52
|
-
@
|
|
51
|
+
# Command line can override transfer spec
|
|
52
|
+
@user_transfer_spec = {
|
|
53
53
|
'create_dir' => true,
|
|
54
54
|
'resume_policy' => 'sparse_csum'
|
|
55
55
|
}
|
|
@@ -61,12 +61,12 @@ module Aspera
|
|
|
61
61
|
@transfer_paths = nil
|
|
62
62
|
# HTTPGW URL provided by webapp
|
|
63
63
|
@httpgw_url_lambda = nil
|
|
64
|
-
@opt_mgr.declare(:ts, 'Override transfer spec values',
|
|
64
|
+
@opt_mgr.declare(:ts, 'Override transfer spec values', allowed: Hash, handler: {o: self, m: :user_transfer_spec})
|
|
65
65
|
@opt_mgr.declare(:to_folder, 'Destination folder for transferred files')
|
|
66
66
|
@opt_mgr.declare(:sources, "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})", default: FILE_LIST_FROM_ARGS)
|
|
67
|
-
@opt_mgr.declare(:src_type, 'Type of file list',
|
|
68
|
-
@opt_mgr.declare(:transfer, 'Type of transfer agent',
|
|
69
|
-
@opt_mgr.declare(:transfer_info, 'Parameters for transfer agent',
|
|
67
|
+
@opt_mgr.declare(:src_type, 'Type of file list', allowed: %i[list pair], default: :list)
|
|
68
|
+
@opt_mgr.declare(:transfer, 'Type of transfer agent', allowed: TRANSFER_AGENTS, default: :direct)
|
|
69
|
+
@opt_mgr.declare(:transfer_info, 'Parameters for transfer agent', allowed: Hash, handler: {o: self, m: :transfer_info})
|
|
70
70
|
@opt_mgr.parse_options!
|
|
71
71
|
@notification_cb = nil
|
|
72
72
|
if !@opt_mgr.get_option(:notify_to).nil?
|
|
@@ -80,25 +80,7 @@ module Aspera
|
|
|
80
80
|
end
|
|
81
81
|
end
|
|
82
82
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
# Multiple option are merged
|
|
86
|
-
# @param value [Hash] Transfer spec
|
|
87
|
-
def option_transfer_spec=(value)
|
|
88
|
-
Aspera.assert_type(value, Hash){'ts'}
|
|
89
|
-
@transfer_spec_command_line.deep_merge!(value)
|
|
90
|
-
end
|
|
91
|
-
|
|
92
|
-
# Add other transfer spec parameters
|
|
93
|
-
def option_transfer_spec_deep_merge(value); @transfer_spec_command_line.deep_merge!(value); end
|
|
94
|
-
|
|
95
|
-
attr_reader :transfer_info
|
|
96
|
-
|
|
97
|
-
# Multiple option are merged
|
|
98
|
-
# @param value [Hash]
|
|
99
|
-
def transfer_info=(value)
|
|
100
|
-
@transfer_info.deep_merge!(value)
|
|
101
|
-
end
|
|
83
|
+
attr_accessor :user_transfer_spec, :transfer_info
|
|
102
84
|
|
|
103
85
|
def agent_instance=(instance)
|
|
104
86
|
@agent = instance
|
|
@@ -145,7 +127,7 @@ module Aspera
|
|
|
145
127
|
dest_folder = @opt_mgr.get_option(:to_folder)
|
|
146
128
|
# do not expand path, if user wants to expand path: user @path:
|
|
147
129
|
return dest_folder unless dest_folder.nil?
|
|
148
|
-
dest_folder = @
|
|
130
|
+
dest_folder = @user_transfer_spec['destination_root']
|
|
149
131
|
return dest_folder unless dest_folder.nil?
|
|
150
132
|
# default: / on remote, . on local
|
|
151
133
|
case direction.to_s
|
|
@@ -193,7 +175,7 @@ module Aspera
|
|
|
193
175
|
# return cache if set
|
|
194
176
|
return @transfer_paths unless @transfer_paths.nil?
|
|
195
177
|
# start with lower priority : get paths from transfer spec on command line
|
|
196
|
-
@transfer_paths = @
|
|
178
|
+
@transfer_paths = @user_transfer_spec['paths'] if @user_transfer_spec.key?('paths')
|
|
197
179
|
# is there a source list option ?
|
|
198
180
|
sources = @opt_mgr.get_option(:sources)
|
|
199
181
|
@transfer_paths =
|
|
@@ -216,7 +198,7 @@ module Aspera
|
|
|
216
198
|
@transfer_paths
|
|
217
199
|
when Array
|
|
218
200
|
Log.log.debug('getting file list as extended value')
|
|
219
|
-
Aspera.
|
|
201
|
+
Aspera.assert_array_all(sources, String, type: Cli::BadArgument){'sources must be a Array of String'}
|
|
220
202
|
list_to_paths(sources)
|
|
221
203
|
else Aspera.error_unexpected_value(sources){'sources'}
|
|
222
204
|
end
|
|
@@ -235,25 +217,25 @@ module Aspera
|
|
|
235
217
|
case transfer_spec['direction']
|
|
236
218
|
when Transfer::Spec::DIRECTION_RECEIVE
|
|
237
219
|
# init default if required in any case
|
|
238
|
-
@
|
|
220
|
+
@user_transfer_spec['destination_root'] ||= destination_folder(transfer_spec['direction'])
|
|
239
221
|
when Transfer::Spec::DIRECTION_SEND
|
|
240
222
|
if transfer_spec.dig('tags', Transfer::Spec::TAG_RESERVED, 'node', 'access_key')
|
|
241
223
|
# gen4
|
|
242
|
-
@
|
|
224
|
+
@user_transfer_spec.delete('destination_root') if @user_transfer_spec.key?('destination_root_id')
|
|
243
225
|
elsif transfer_spec.key?('token')
|
|
244
226
|
# gen3
|
|
245
227
|
# in that case, destination is set in return by application (API/upload_setup)
|
|
246
228
|
# but to_folder was used in initial API call
|
|
247
|
-
@
|
|
229
|
+
@user_transfer_spec.delete('destination_root')
|
|
248
230
|
else
|
|
249
231
|
# init default if required
|
|
250
|
-
@
|
|
232
|
+
@user_transfer_spec['destination_root'] ||= destination_folder(transfer_spec['direction'])
|
|
251
233
|
end
|
|
252
234
|
end
|
|
253
235
|
# update command line paths, unless destination already has one
|
|
254
|
-
@
|
|
236
|
+
@user_transfer_spec['paths'] = transfer_spec['paths'] || ts_source_paths
|
|
255
237
|
# updated transfer spec with command line
|
|
256
|
-
transfer_spec.deep_merge!(@
|
|
238
|
+
transfer_spec.deep_merge!(@user_transfer_spec)
|
|
257
239
|
# recursively remove values that are nil (user wants to delete)
|
|
258
240
|
transfer_spec.deep_do{ |hash, key, value, _unused| hash.delete(key) if value.nil?}
|
|
259
241
|
# if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/cli/wizard.rb
CHANGED
|
@@ -17,8 +17,8 @@ module Aspera
|
|
|
17
17
|
@parent = parent
|
|
18
18
|
@main_folder = main_folder
|
|
19
19
|
# Wizard options
|
|
20
|
-
options.declare(:override, 'Wizard: override existing value',
|
|
21
|
-
options.declare(:default, 'Wizard: set as default configuration for specified plugin (also: update)',
|
|
20
|
+
options.declare(:override, 'Wizard: override existing value', allowed: Allowed::TYPES_BOOLEAN, default: false)
|
|
21
|
+
options.declare(:default, 'Wizard: set as default configuration for specified plugin (also: update)', allowed: Allowed::TYPES_BOOLEAN, default: true)
|
|
22
22
|
options.declare(:key_path, 'Wizard: path to private key for JWT')
|
|
23
23
|
end
|
|
24
24
|
|
|
@@ -84,7 +84,10 @@ module Aspera
|
|
|
84
84
|
end
|
|
85
85
|
|
|
86
86
|
# To be called in public wizard method to get private key
|
|
87
|
-
# @
|
|
87
|
+
# @param user [String] User's email
|
|
88
|
+
# @param url [String] Instance URL
|
|
89
|
+
# @param page [String] URL of page to enter pub key
|
|
90
|
+
# @return [String] Private key path (can contain ~ for home)
|
|
88
91
|
def ask_private_key(user:, url:, page:)
|
|
89
92
|
# Lets see if path to priv key is provided
|
|
90
93
|
private_key_path = options.get_option(:key_path)
|
|
@@ -95,7 +98,7 @@ module Aspera
|
|
|
95
98
|
end
|
|
96
99
|
# Else generate path
|
|
97
100
|
private_key_path = File.join(@main_folder, DEFAULT_PRIV_KEY_FILENAME) if private_key_path.empty?
|
|
98
|
-
if File.exist?(private_key_path)
|
|
101
|
+
if File.exist?(File.expand_path(private_key_path))
|
|
99
102
|
formatter.display_status('Using existing key:')
|
|
100
103
|
else
|
|
101
104
|
formatter.display_status("Generating #{OAuth::Jwt::DEFAULT_PRIV_KEY_LENGTH} bit RSA key...")
|
|
@@ -103,7 +106,7 @@ module Aspera
|
|
|
103
106
|
formatter.display_status('Created key:')
|
|
104
107
|
end
|
|
105
108
|
formatter.display_status(private_key_path)
|
|
106
|
-
private_key_pem = File.read(private_key_path)
|
|
109
|
+
private_key_pem = File.read(File.expand_path(private_key_path))
|
|
107
110
|
pub_key_pem = OpenSSL::PKey::RSA.new(private_key_pem).public_key.to_s
|
|
108
111
|
options.set_option(:private_key, private_key_pem)
|
|
109
112
|
formatter.display_status("Please Log in as user #{user.red} at: #{url.red}")
|
|
@@ -31,39 +31,42 @@ module Aspera
|
|
|
31
31
|
'x-deprecation' # [String] Deprecation message for doc
|
|
32
32
|
].freeze
|
|
33
33
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
private_constant :PROPERTY_KEYS, :CLI_AGENT
|
|
34
|
+
private_constant :PROPERTY_KEYS
|
|
37
35
|
|
|
38
36
|
class << self
|
|
39
|
-
#
|
|
37
|
+
# Called by provider of definition before constructor of this class so that schema has all mandatory fields
|
|
38
|
+
def read_schema(folder, name, ascp: false)
|
|
39
|
+
schema = YAML.load_file(File.join(folder, "#{name}.schema.yaml"))
|
|
40
|
+
validate_schema(schema, ascp: ascp)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# @param agent [Symbol] Transfer agent name
|
|
44
|
+
# @param properties [Hash] Transfer spec parameter information
|
|
45
|
+
# @return [Boolean] `true` if given agent supports that field
|
|
40
46
|
def supported_by_agent(agent, properties)
|
|
41
|
-
!properties.key?('x-agents') || properties['x-agents'].include?(agent)
|
|
47
|
+
!properties.key?('x-agents') || properties['x-agents'].include?(agent.to_s)
|
|
42
48
|
end
|
|
43
49
|
|
|
50
|
+
private
|
|
51
|
+
|
|
44
52
|
# Fill default values for some fields in the schema
|
|
45
53
|
# @param schema [Hash] The JSON schema
|
|
54
|
+
# @param ascp [Bool] `true` if ascp
|
|
46
55
|
def validate_schema(schema, ascp: false)
|
|
56
|
+
direct_props = %w[x-cli-option x-cli-envvar x-cli-special].freeze
|
|
47
57
|
schema['properties'].each do |name, info|
|
|
48
58
|
Aspera.assert_type(info, Hash){"#{info.class} for #{name}"}
|
|
49
59
|
unsupported_keys = info.keys - PROPERTY_KEYS
|
|
50
60
|
Aspera.assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
|
|
51
|
-
# By default : string, unless it's without arg (switch)
|
|
52
|
-
# info['type'] ||= info['x-cli-switch'] ? 'boolean' : 'string'
|
|
53
61
|
Aspera.assert(info.key?('type') || info.key?('enum')){"Missing type for #{name} in #{schema['description']}"}
|
|
54
|
-
Aspera.assert(info['type'].eql?('boolean')){"switch must be bool: #{name}"} if info['x-cli-switch']
|
|
55
|
-
# Add default cli option name if not present, and if supported in "direct".
|
|
62
|
+
Aspera.assert(info['type'].eql?('boolean')){"switch must be bool: #{name}"} if info['x-cli-switch'] && !info['x-cli-special']
|
|
56
63
|
info['x-cli-option'] = "--#{name.to_s.tr('_', '-')}" if info['x-cli-option'].eql?(true) || (info['x-cli-switch'].eql?(true) && !info.key?('x-cli-option'))
|
|
57
|
-
Aspera.assert(
|
|
58
|
-
# info['x-cli-option'] = "--#{name.to_s.tr('_', '-')}" if ascp && !info.key?('x-cli-option') && !info['x-cli-envvar'] && (info.key?('x-cli-switch') || supported_by_agent(CLI_AGENT, info))
|
|
64
|
+
Aspera.assert(direct_props.any?{ |i| info.key?(i)}, type: :warn){name} if ascp && supported_by_agent(:direct, info)
|
|
59
65
|
info.freeze
|
|
60
66
|
validate_schema(info, ascp: ascp) if info['type'].eql?('object') && info['properties']
|
|
67
|
+
validate_schema(info['items'], ascp: ascp) if info['type'].eql?('array') && info['items'] && info['items']['properties']
|
|
61
68
|
end
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
# Called by provider of definition before constructor of this class so that schema has all mandatory fields
|
|
65
|
-
def read_schema(source_path, name)
|
|
66
|
-
YAML.load_file(File.join(File.dirname(source_path), "#{name}.schema.yaml"))
|
|
69
|
+
schema
|
|
67
70
|
end
|
|
68
71
|
end
|
|
69
72
|
|
|
@@ -173,7 +176,7 @@ module Aspera
|
|
|
173
176
|
parameter_value = converted_value
|
|
174
177
|
end
|
|
175
178
|
|
|
176
|
-
return unless self.class.supported_by_agent(
|
|
179
|
+
return unless self.class.supported_by_agent(:direct, properties)
|
|
177
180
|
|
|
178
181
|
if read
|
|
179
182
|
# just get value (deferred)
|
data/lib/aspera/coverage.rb
CHANGED
|
@@ -8,7 +8,7 @@ if ENV.key?('ENABLE_COVERAGE')
|
|
|
8
8
|
development_root = File.dirname(File.realpath(__FILE__), 3)
|
|
9
9
|
SimpleCov.root(development_root)
|
|
10
10
|
SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
|
|
11
|
-
# keep cache data for 1 day (must be longer
|
|
11
|
+
# keep cache data for 1 day (must be longer than time to run the whole test suite)
|
|
12
12
|
SimpleCov.merge_timeout(86400)
|
|
13
13
|
SimpleCov.command_name(SecureRandom.uuid)
|
|
14
14
|
SimpleCov.at_exit do
|
data/lib/aspera/environment.rb
CHANGED
|
@@ -5,7 +5,9 @@ require 'aspera/log'
|
|
|
5
5
|
require 'aspera/assert'
|
|
6
6
|
require 'rbconfig'
|
|
7
7
|
require 'singleton'
|
|
8
|
+
require 'open3'
|
|
8
9
|
require 'English'
|
|
10
|
+
require 'shellwords'
|
|
9
11
|
|
|
10
12
|
# cspell:words MEBI mswin bccwin
|
|
11
13
|
|
|
@@ -56,9 +58,9 @@ module Aspera
|
|
|
56
58
|
end
|
|
57
59
|
|
|
58
60
|
# Generate log line for external program with arguments
|
|
59
|
-
# @param
|
|
60
|
-
# @param
|
|
61
|
-
# @param
|
|
61
|
+
# @param exec [String] Path to executable
|
|
62
|
+
# @param args [Array, nil] Arguments
|
|
63
|
+
# @param env [Hash, nil] Environment variables
|
|
62
64
|
# @return [String] log line with environment, program and arguments
|
|
63
65
|
def log_spawn(exec:, args: nil, env: nil)
|
|
64
66
|
[
|
|
@@ -71,65 +73,70 @@ module Aspera
|
|
|
71
73
|
|
|
72
74
|
# Start process in background
|
|
73
75
|
# caller can call Process.wait on returned value
|
|
74
|
-
# @param exec [String]
|
|
75
|
-
# @param args [Array, nil]
|
|
76
|
-
# @param env [Hash, nil]
|
|
77
|
-
# @param
|
|
76
|
+
# @param exec [String] Path to executable
|
|
77
|
+
# @param args [Array, nil] Arguments for executable
|
|
78
|
+
# @param env [Hash, nil] Environment variables
|
|
79
|
+
# @param kwargs [Hash] Options for `Process.spawn`
|
|
78
80
|
# @return [String] PID of process
|
|
79
81
|
# @raise [Exception] if problem
|
|
80
|
-
def secure_spawn(exec:, args: nil, env: nil, **
|
|
82
|
+
def secure_spawn(exec:, args: nil, env: nil, **kwargs)
|
|
81
83
|
Aspera.assert_type(exec, String)
|
|
82
84
|
Aspera.assert_type(args, Array, NilClass)
|
|
83
85
|
Aspera.assert_type(env, Hash, NilClass)
|
|
84
|
-
Aspera.assert_type(options, Hash, NilClass)
|
|
85
86
|
Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
|
|
86
|
-
# start ascp in separate process
|
|
87
87
|
spawn_args = []
|
|
88
88
|
spawn_args.push(env) unless env.nil?
|
|
89
89
|
spawn_args.push([exec, exec])
|
|
90
90
|
spawn_args.concat(args) unless args.nil?
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
Log.
|
|
95
|
-
return
|
|
91
|
+
kwargs[:close_others] = true unless kwargs.key?(:close_others)
|
|
92
|
+
# Start separate process in background
|
|
93
|
+
pid = Process.spawn(*spawn_args, **kwargs)
|
|
94
|
+
Log.dump(:pid, pid)
|
|
95
|
+
return pid
|
|
96
96
|
end
|
|
97
97
|
|
|
98
|
-
#
|
|
99
|
-
#
|
|
100
|
-
# @param exec
|
|
101
|
-
# @param args
|
|
102
|
-
# @
|
|
103
|
-
|
|
98
|
+
# Start process (not in shell) and wait for completion.
|
|
99
|
+
# By default, sets `exception: true` in `kwargs`
|
|
100
|
+
# @param exec [String] Path to executable
|
|
101
|
+
# @param args [Array, nil] Arguments
|
|
102
|
+
# @param env [Hash, nil] Environment variables
|
|
103
|
+
# @param kwargs [Hash] Arguments for `Kernel.system`
|
|
104
|
+
# @return `nil`
|
|
105
|
+
# @raise [RuntimeError] if problem
|
|
106
|
+
def secure_execute(exec:, args: nil, env: nil, **kwargs)
|
|
104
107
|
Aspera.assert_type(exec, String)
|
|
105
108
|
Aspera.assert_type(args, Array, NilClass)
|
|
106
109
|
Aspera.assert_type(env, Hash, NilClass)
|
|
107
110
|
Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
|
|
108
|
-
|
|
111
|
+
Log.dump(:kwargs, kwargs, level: :trace1)
|
|
109
112
|
spawn_args = []
|
|
110
113
|
spawn_args.push(env) unless env.nil?
|
|
111
|
-
#
|
|
114
|
+
# Ensure no shell expansion
|
|
112
115
|
spawn_args.push([exec, exec])
|
|
113
116
|
spawn_args.concat(args) unless args.nil?
|
|
114
|
-
|
|
115
|
-
kwargs.
|
|
117
|
+
# By default: exception on error
|
|
118
|
+
kwargs[:exception] = true unless kwargs.key?(:exception)
|
|
119
|
+
# Start in separate process
|
|
116
120
|
Kernel.system(*spawn_args, **kwargs)
|
|
117
121
|
nil
|
|
118
122
|
end
|
|
119
123
|
|
|
120
124
|
# Execute process and capture stdout
|
|
121
|
-
# @param exec
|
|
122
|
-
# @param args
|
|
123
|
-
# @param
|
|
125
|
+
# @param exec [String] path to executable
|
|
126
|
+
# @param args [Array] arguments to executable
|
|
127
|
+
# @param kwargs [Hash] options to Open3.capture3
|
|
124
128
|
# @return stdout of executable or raise exception
|
|
125
|
-
def secure_capture(exec:, args: [], exception: true, **
|
|
129
|
+
def secure_capture(exec:, args: [], env: nil, exception: true, **kwargs)
|
|
126
130
|
Aspera.assert_type(exec, String)
|
|
127
131
|
Aspera.assert_type(args, Array)
|
|
128
|
-
|
|
129
|
-
Log.
|
|
130
|
-
Log.dump(:
|
|
131
|
-
|
|
132
|
-
|
|
132
|
+
Log.log.debug{log_spawn(exec: exec, args: args, env: env)}
|
|
133
|
+
Log.dump(:kwargs, kwargs, level: :trace2)
|
|
134
|
+
# Log.dump(:ENV, ENV.to_h, level: :trace1)
|
|
135
|
+
capture_args = []
|
|
136
|
+
capture_args.push(env) unless env.nil?
|
|
137
|
+
capture_args.push(exec)
|
|
138
|
+
capture_args.concat(args)
|
|
139
|
+
stdout, stderr, status = Open3.capture3(*capture_args, **kwargs)
|
|
133
140
|
Log.log.debug{"status=#{status}, stderr=#{stderr}"}
|
|
134
141
|
Log.log.trace1{"stdout=#{stdout}"}
|
|
135
142
|
raise "process failed: #{status.exitstatus} (#{stderr})" if !status.success? && exception
|
data/lib/aspera/faspex_gw.rb
CHANGED
|
@@ -55,7 +55,7 @@ module Aspera
|
|
|
55
55
|
content_type: Rest::MIME_JSON,
|
|
56
56
|
body: {paths: [{'destination'=>'/'}]},
|
|
57
57
|
headers: {'Accept' => Rest::MIME_JSON}
|
|
58
|
-
)
|
|
58
|
+
)
|
|
59
59
|
transfer_spec.delete('authentication')
|
|
60
60
|
# but we place it in a Faspex package creation response
|
|
61
61
|
return {
|
|
@@ -12,8 +12,7 @@ module Aspera
|
|
|
12
12
|
# @param folder [String] folder to store the vault (if needed)
|
|
13
13
|
# @param password [String] password to open the vault
|
|
14
14
|
def create(info, name, folder, password)
|
|
15
|
-
Aspera.
|
|
16
|
-
Aspera.assert(info.values.all?(String)){'vault info shall have only string values'}
|
|
15
|
+
Aspera.assert_hash_all(info, Symbol, String){'vault info shall have only string values'}
|
|
17
16
|
info = info.symbolize_keys
|
|
18
17
|
vault_type = info.delete(:type)
|
|
19
18
|
Aspera.assert_values(vault_type, LIST.map(&:to_s)){'vault.type'}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aspera
|
|
4
|
+
# Formatting for Markdown
|
|
5
|
+
class Markdown
|
|
6
|
+
# Matches: **bold**, `code`, or an HTML entity (&, ©, 💩)
|
|
7
|
+
FORMATS = /(?:\*\*(?<bold>[^*]+?)\*\*)|(?:`(?<code>[^`]+)`)|&(?<entity>(?:[A-Za-z][A-Za-z0-9]{1,31}|#\d{1,7}|#x[0-9A-Fa-f]{1,6}));/m
|
|
8
|
+
HTML_BREAK = '<br/>'
|
|
9
|
+
|
|
10
|
+
class << self
|
|
11
|
+
# Generate markdown from the provided 2D table
|
|
12
|
+
def table(table)
|
|
13
|
+
# get max width of each columns
|
|
14
|
+
col_widths = table.transpose.map do |col|
|
|
15
|
+
[col.flat_map{ |c| c.to_s.delete('`').split(HTML_BREAK).map(&:size)}.max, 80].min
|
|
16
|
+
end
|
|
17
|
+
headings = table.shift
|
|
18
|
+
table.unshift(col_widths.map{ |col_width| '-' * col_width})
|
|
19
|
+
table.unshift(headings)
|
|
20
|
+
lines = table.map{ |line| "| #{line.map{ |i| i.to_s.gsub('\\', '\\\\').gsub('|', '\|')}.join(' | ')} |\n"}
|
|
21
|
+
lines[1] = lines[1].tr(' ', '-')
|
|
22
|
+
return lines.join.chomp
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Generate markdown list from the provided list
|
|
26
|
+
def list(items)
|
|
27
|
+
items.map{ |i| "- #{i}"}.join("\n")
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
data/lib/aspera/nagios.rb
CHANGED
|
@@ -21,7 +21,7 @@ module Aspera
|
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
class << self
|
|
24
|
-
#
|
|
24
|
+
# Process results of a analysis and display status and exit with code
|
|
25
25
|
def process(data)
|
|
26
26
|
Aspera.assert_type(data, Array)
|
|
27
27
|
Aspera.assert(!data.empty?){'data is empty'}
|
|
@@ -54,7 +54,7 @@ module Aspera
|
|
|
54
54
|
@data = []
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
-
#
|
|
57
|
+
# Compare remote time with local time
|
|
58
58
|
def check_time_offset(remote_date, component)
|
|
59
59
|
# check date if specified : 2015-10-13T07:32:01Z
|
|
60
60
|
remote_time = Time.parse(remote_date)
|
|
@@ -76,10 +76,11 @@ module Aspera
|
|
|
76
76
|
# TODO: check on database if latest version
|
|
77
77
|
end
|
|
78
78
|
|
|
79
|
-
#
|
|
80
|
-
|
|
79
|
+
# Readable status list
|
|
80
|
+
# @return [Array] of Hash
|
|
81
|
+
def status_list
|
|
81
82
|
Aspera.assert(!@data.empty?){'missing result'}
|
|
82
|
-
|
|
83
|
+
@data.map{ |i| {'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}
|
|
83
84
|
end
|
|
84
85
|
end
|
|
85
86
|
end
|