aspera-cli 4.19.0 → 4.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +20 -0
- data/CONTRIBUTING.md +16 -4
- data/README.md +344 -164
- data/bin/asession +26 -19
- data/examples/build_exec +65 -76
- data/examples/build_exec_rubyc +40 -0
- data/examples/get_proto_file.rb +7 -0
- data/lib/aspera/agent/alpha.rb +8 -8
- data/lib/aspera/agent/base.rb +2 -18
- data/lib/aspera/agent/connect.rb +14 -13
- data/lib/aspera/agent/direct.rb +23 -24
- data/lib/aspera/agent/httpgw.rb +2 -3
- data/lib/aspera/agent/node.rb +10 -10
- data/lib/aspera/agent/trsdk.rb +17 -20
- data/lib/aspera/api/alee.rb +15 -0
- data/lib/aspera/api/aoc.rb +126 -97
- data/lib/aspera/api/ats.rb +1 -1
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/httpgw.rb +15 -10
- data/lib/aspera/api/node.rb +33 -12
- data/lib/aspera/ascmd.rb +56 -48
- data/lib/aspera/ascp/installation.rb +99 -42
- data/lib/aspera/ascp/management.rb +3 -2
- data/lib/aspera/ascp/products.rb +12 -0
- data/lib/aspera/assert.rb +10 -5
- data/lib/aspera/cli/formatter.rb +27 -17
- data/lib/aspera/cli/hints.rb +2 -1
- data/lib/aspera/cli/info.rb +12 -10
- data/lib/aspera/cli/main.rb +16 -13
- data/lib/aspera/cli/manager.rb +5 -0
- data/lib/aspera/cli/plugin.rb +15 -29
- data/lib/aspera/cli/plugins/alee.rb +3 -3
- data/lib/aspera/cli/plugins/aoc.rb +222 -194
- data/lib/aspera/cli/plugins/ats.rb +16 -14
- data/lib/aspera/cli/plugins/config.rb +53 -45
- data/lib/aspera/cli/plugins/console.rb +3 -3
- data/lib/aspera/cli/plugins/faspex.rb +11 -21
- data/lib/aspera/cli/plugins/faspex5.rb +44 -42
- data/lib/aspera/cli/plugins/faspio.rb +2 -2
- data/lib/aspera/cli/plugins/httpgw.rb +1 -1
- data/lib/aspera/cli/plugins/node.rb +153 -95
- data/lib/aspera/cli/plugins/orchestrator.rb +14 -13
- data/lib/aspera/cli/plugins/preview.rb +8 -9
- data/lib/aspera/cli/plugins/server.rb +5 -9
- data/lib/aspera/cli/plugins/shares.rb +2 -2
- data/lib/aspera/cli/sync_actions.rb +2 -2
- data/lib/aspera/cli/transfer_agent.rb +12 -14
- data/lib/aspera/cli/transfer_progress.rb +35 -17
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +3 -4
- data/lib/aspera/coverage.rb +13 -1
- data/lib/aspera/environment.rb +34 -18
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/macos_security.rb +7 -12
- data/lib/aspera/log.rb +3 -4
- data/lib/aspera/oauth/base.rb +39 -45
- data/lib/aspera/oauth/factory.rb +11 -4
- data/lib/aspera/oauth/generic.rb +4 -8
- data/lib/aspera/oauth/jwt.rb +3 -3
- data/lib/aspera/oauth/url_json.rb +1 -2
- data/lib/aspera/oauth/web.rb +5 -2
- data/lib/aspera/persistency_action_once.rb +16 -8
- data/lib/aspera/preview/utils.rb +5 -16
- data/lib/aspera/rest.rb +100 -76
- data/lib/aspera/transfer/faux_file.rb +4 -4
- data/lib/aspera/transfer/parameters.rb +14 -16
- data/lib/aspera/transfer/spec.rb +12 -12
- data/lib/aspera/transfer/sync.rb +1 -5
- data/lib/aspera/transfer/uri.rb +1 -1
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/aspera/web_auth.rb +166 -17
- data/lib/aspera/web_server_simple.rb +4 -3
- data/lib/transfer_pb.rb +84 -0
- data/lib/transfer_services_pb.rb +82 -0
- data.tar.gz.sig +0 -0
- metadata +24 -5
- metadata.gz.sig +0 -0
@@ -22,7 +22,7 @@ module Aspera
|
|
22
22
|
next unless base_url.match?('https?://')
|
23
23
|
api = Rest.new(base_url: base_url)
|
24
24
|
test_endpoint = 'api/remote_node_ping'
|
25
|
-
result = api.
|
25
|
+
result = api.call(operation: 'GET', subpath: test_endpoint, headers: {'Accept' => 'application/json'}, query: {format: :json})
|
26
26
|
next unless result[:data]['remote_orchestrator_info']
|
27
27
|
url = result[:http].uri.to_s
|
28
28
|
return {
|
@@ -69,7 +69,7 @@ module Aspera
|
|
69
69
|
# @param format [String] the format to request, 'json', 'xml', nil
|
70
70
|
# @param args [Hash] the arguments to pass
|
71
71
|
# @param xml_arrays [Boolean] if true, force arrays in xml parsing
|
72
|
-
def call_ao(endpoint, prefix: 'api', id: nil, ret_style: nil, format: 'json', args: nil, xml_arrays: true)
|
72
|
+
def call_ao(endpoint, prefix: 'api', id: nil, ret_style: nil, format: 'json', args: nil, xml_arrays: true, http: false)
|
73
73
|
# calls are GET
|
74
74
|
call_args = {operation: 'GET', subpath: endpoint}
|
75
75
|
# specify prefix if necessary
|
@@ -91,9 +91,10 @@ module Aspera
|
|
91
91
|
end
|
92
92
|
end
|
93
93
|
result = @api_orch.call(**call_args)
|
94
|
+
return result[:http] if http
|
94
95
|
result[:data] = XmlSimple.xml_in(result[:http].body, {'ForceArray' => xml_arrays}) if format.eql?('xml')
|
95
96
|
Log.log.debug{Log.dump(:data, result[:data])}
|
96
|
-
return result
|
97
|
+
return result[:data]
|
97
98
|
end
|
98
99
|
|
99
100
|
def execute_action
|
@@ -126,7 +127,7 @@ module Aspera
|
|
126
127
|
when :health
|
127
128
|
nagios = Nagios.new
|
128
129
|
begin
|
129
|
-
info = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
|
130
|
+
info = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
|
130
131
|
nagios.add_ok('api', 'accessible')
|
131
132
|
nagios.check_product_version('api', 'orchestrator', info['orchestrator-version'])
|
132
133
|
rescue StandardError => e
|
@@ -134,15 +135,15 @@ module Aspera
|
|
134
135
|
end
|
135
136
|
return nagios.result
|
136
137
|
when :info
|
137
|
-
result = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
|
138
|
+
result = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
|
138
139
|
return {type: :single_object, data: result}
|
139
140
|
when :processes
|
140
141
|
# TODO: Bug ? API has only XML format
|
141
|
-
result = call_ao('processes_status', format: 'xml')
|
142
|
+
result = call_ao('processes_status', format: 'xml')
|
142
143
|
return {type: :object_list, data: result['process']}
|
143
144
|
when :plugins
|
144
145
|
# TODO: Bug ? only json format on url
|
145
|
-
result = call_ao('plugin_version')
|
146
|
+
result = call_ao('plugin_version')
|
146
147
|
return {type: :object_list, data: result['Plugin']}
|
147
148
|
when :workflow
|
148
149
|
command = options.get_next_command(%i[list status inputs details start export])
|
@@ -152,23 +153,23 @@ module Aspera
|
|
152
153
|
case command
|
153
154
|
when :status
|
154
155
|
wf_id = nil if wf_id.eql?(SpecialValues::ALL)
|
155
|
-
result = call_ao('workflows_status', id: wf_id)
|
156
|
+
result = call_ao('workflows_status', id: wf_id)
|
156
157
|
return {type: :object_list, data: result['workflows']['workflow']}
|
157
158
|
when :list
|
158
|
-
result = call_ao('workflows_list', id: 0)
|
159
|
+
result = call_ao('workflows_list', id: 0)
|
159
160
|
return {
|
160
161
|
type: :object_list,
|
161
162
|
data: result['workflows']['workflow'],
|
162
163
|
fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification]
|
163
164
|
}
|
164
165
|
when :details
|
165
|
-
result = call_ao('workflow_details', id: wf_id)
|
166
|
+
result = call_ao('workflow_details', id: wf_id)
|
166
167
|
return {type: :object_list, data: result['workflows']['workflow']['statuses']}
|
167
168
|
when :inputs
|
168
|
-
result = call_ao('workflow_inputs_spec', id: wf_id)
|
169
|
+
result = call_ao('workflow_inputs_spec', id: wf_id)
|
169
170
|
return {type: :single_object, data: result['workflow_inputs_spec']}
|
170
171
|
when :export
|
171
|
-
result = call_ao('export_workflow', id: wf_id, format: nil)
|
172
|
+
result = call_ao('export_workflow', id: wf_id, format: nil, http: true)
|
172
173
|
return {type: :text, data: result.body}
|
173
174
|
when :start
|
174
175
|
result = {
|
@@ -196,7 +197,7 @@ module Aspera
|
|
196
197
|
if call_params['synchronous']
|
197
198
|
result[:type] = :text
|
198
199
|
end
|
199
|
-
result[:data] = call_ao('initiate', id: wf_id, args: call_params)
|
200
|
+
result[:data] = call_ao('initiate', id: wf_id, args: call_params)
|
200
201
|
return result
|
201
202
|
end
|
202
203
|
else Aspera.error_unexpected_value(command)
|
@@ -133,7 +133,6 @@ module Aspera
|
|
133
133
|
subpath: "files/#{file_id}/files",
|
134
134
|
headers: headers,
|
135
135
|
query: request_args)[:data]
|
136
|
-
# return @api_node.read("files/#{file_id}/files",request_args)[:data]
|
137
136
|
end
|
138
137
|
|
139
138
|
# old version based on folders
|
@@ -146,7 +145,7 @@ module Aspera
|
|
146
145
|
# optionally add iteration token from persistency
|
147
146
|
events_filter['iteration_token'] = iteration_persistency.data.first unless iteration_persistency.nil?
|
148
147
|
begin
|
149
|
-
events = @api_node.read('events', events_filter)
|
148
|
+
events = @api_node.read('events', events_filter)
|
150
149
|
rescue RestCallError => e
|
151
150
|
if e.message.include?('Invalid iteration_token')
|
152
151
|
Log.log.warn{"Retrying without iteration token: #{e}"}
|
@@ -164,7 +163,7 @@ module Aspera
|
|
164
163
|
folder_id = event.dig('data', 'tags', Transfer::Spec::TAG_RESERVED, 'node', 'file_id')
|
165
164
|
folder_id ||= event.dig('data', 'file_id')
|
166
165
|
if !folder_id.nil?
|
167
|
-
folder_entry = @api_node.read("files/#{folder_id}")
|
166
|
+
folder_entry = @api_node.read("files/#{folder_id}") rescue nil
|
168
167
|
scan_folder_files(folder_entry) unless folder_entry.nil?
|
169
168
|
end
|
170
169
|
end
|
@@ -188,12 +187,12 @@ module Aspera
|
|
188
187
|
}
|
189
188
|
# optionally add iteration token from persistency
|
190
189
|
events_filter['iteration_token'] = iteration_persistency.data.first unless iteration_persistency.nil?
|
191
|
-
events = @api_node.read('events', events_filter)
|
190
|
+
events = @api_node.read('events', events_filter)
|
192
191
|
return if events.empty?
|
193
192
|
events.each do |event|
|
194
193
|
# process only files
|
195
194
|
if event.dig('data', 'type').eql?('file')
|
196
|
-
file_entry = @api_node.read("files/#{event['data']['id']}")
|
195
|
+
file_entry = @api_node.read("files/#{event['data']['id']}") rescue nil
|
197
196
|
if !file_entry.nil? &&
|
198
197
|
@option_skip_folders.none?{|d|file_entry['path'].start_with?(d)}
|
199
198
|
file_entry['parent_file_id'] = event['data']['parent_file_id']
|
@@ -248,7 +247,7 @@ module Aspera
|
|
248
247
|
# original_mtime=DateTime.parse(entry['modified_time'])
|
249
248
|
# out: where previews are generated
|
250
249
|
local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
|
251
|
-
file_info = @api_node.read("files/#{entry['id']}")
|
250
|
+
file_info = @api_node.read("files/#{entry['id']}")
|
252
251
|
# TODO: this does not work because previews is hidden in api (gen4)
|
253
252
|
# this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
|
254
253
|
# TODO: use gen3 api to list files and get date
|
@@ -410,10 +409,10 @@ module Aspera
|
|
410
409
|
@api_node = Api::Node.new(**basic_auth_params)
|
411
410
|
@transfer_server_address = URI.parse(@api_node.base_url).host
|
412
411
|
# get current access key
|
413
|
-
@access_key_self = @api_node.read('access_keys/self')
|
412
|
+
@access_key_self = @api_node.read('access_keys/self')
|
414
413
|
# TODO: check events is activated here:
|
415
414
|
# note that docroot is good to look at as well
|
416
|
-
node_info = @api_node.read('info')
|
415
|
+
node_info = @api_node.read('info')
|
417
416
|
Log.log.debug{"root: #{node_info['docroot']}"}
|
418
417
|
@access_remote = @option_file_access.eql?(:remote)
|
419
418
|
Log.log.debug{"remote: #{@access_remote}"}
|
@@ -463,7 +462,7 @@ module Aspera
|
|
463
462
|
'type' => 'folder',
|
464
463
|
'path' => '/' }
|
465
464
|
else
|
466
|
-
@api_node.read("files/#{scan_id}")
|
465
|
+
@api_node.read("files/#{scan_id}")
|
467
466
|
end
|
468
467
|
@filter_block = Api::Node.file_matcher_from_argument(options)
|
469
468
|
scan_folder_files(folder_info, scan_path)
|
@@ -9,6 +9,7 @@ require 'aspera/ssh'
|
|
9
9
|
require 'aspera/nagios'
|
10
10
|
require 'aspera/log'
|
11
11
|
require 'aspera/assert'
|
12
|
+
require 'aspera/environment'
|
12
13
|
require 'tempfile'
|
13
14
|
require 'open3'
|
14
15
|
|
@@ -32,14 +33,8 @@ module Aspera
|
|
32
33
|
private_constant :SSH_SCHEME, :URI_SCHEMES, :ASCMD_ALIASES, :TRANSFER_COMMANDS
|
33
34
|
|
34
35
|
class LocalExecutor
|
35
|
-
def execute(
|
36
|
-
|
37
|
-
cmd = cmd.map{|v|%Q("#{v}")}.join(' ') if cmd.is_a?(Array)
|
38
|
-
Log.log.debug{"Executing: #{cmd} with '#{line}'"}
|
39
|
-
stdout_str, stderr_str, status = Open3.capture3(cmd, stdin_data: line, binmode: true)
|
40
|
-
Log.log.debug{"exec status: #{status} -> #{stderr_str}"}
|
41
|
-
raise "command #{cmd} failed with code #{status.exitstatus} #{stderr_str}" unless status.success?
|
42
|
-
return stdout_str
|
36
|
+
def execute(ascmd_path, line)
|
37
|
+
return Environment.secure_capture(exec: ascmd_path, stdin_data: line, binmode: true)
|
43
38
|
end
|
44
39
|
end
|
45
40
|
|
@@ -157,7 +152,8 @@ module Aspera
|
|
157
152
|
Log.log.debug{"SSH keys=#{ssh_key_list}"}
|
158
153
|
if !ssh_key_list.empty?
|
159
154
|
@ssh_opts[:keys] = ssh_key_list
|
160
|
-
|
155
|
+
# PEM as per RFC 7468
|
156
|
+
server_transfer_spec['ssh_private_key'] = File.read(ssh_key_list.first).strip
|
161
157
|
Log.log.warn{'Using only first SSH key for transfers'} unless ssh_key_list.length.eql?(1)
|
162
158
|
cred_set = true
|
163
159
|
end
|
@@ -144,11 +144,11 @@ module Aspera
|
|
144
144
|
entity_parameters.each_key do |p|
|
145
145
|
raise "unsupported field: #{p}, use: #{SAML_IMPORT_ALLOWED.join(',')}" unless SAML_IMPORT_ALLOWED.include?(p)
|
146
146
|
end
|
147
|
-
api_shares_admin.create("#{entities_path}/import", entity_parameters)
|
147
|
+
api_shares_admin.create("#{entities_path}/import", entity_parameters)
|
148
148
|
end
|
149
149
|
when :add # ldap
|
150
150
|
return do_bulk_operation(command: entity_verb, descr: "#{entity_type} name", values: String) do |entity_name|
|
151
|
-
api_shares_admin.create(entities_path, {entity_type=>entity_name})
|
151
|
+
api_shares_admin.create(entities_path, {entity_type=>entity_name})
|
152
152
|
end
|
153
153
|
when :users # group
|
154
154
|
return entity_action(api_shares_admin, "#{entities_path}/#{instance_identifier}/#{entities_prefix}users")
|
@@ -76,9 +76,9 @@ module Aspera
|
|
76
76
|
end
|
77
77
|
if !session_info.key?('name')
|
78
78
|
# if no name is specified, generate one from simple arguments
|
79
|
-
session_info['name'] = SYNC_SIMPLE_ARGS.
|
79
|
+
session_info['name'] = SYNC_SIMPLE_ARGS.filter_map do |arg_name|
|
80
80
|
arguments[arg_name]&.gsub(/[^a-zA-Z0-9]/, '')
|
81
|
-
end.
|
81
|
+
end.reject(&:empty?).join('_')
|
82
82
|
end
|
83
83
|
end
|
84
84
|
end
|
@@ -22,7 +22,7 @@ module Aspera
|
|
22
22
|
To: <<%=to%>>
|
23
23
|
Subject: <%=subject%>
|
24
24
|
|
25
|
-
Transfer is: <%=
|
25
|
+
Transfer is: <%=status%>
|
26
26
|
|
27
27
|
<%=ts.to_yaml%>
|
28
28
|
END_OF_TEMPLATE
|
@@ -65,6 +65,16 @@ module Aspera
|
|
65
65
|
@opt_mgr.declare(:transfer, 'Type of transfer agent', values: TRANSFER_AGENTS, default: :direct)
|
66
66
|
@opt_mgr.declare(:transfer_info, 'Parameters for transfer agent', types: Hash, handler: {o: self, m: :transfer_info})
|
67
67
|
@opt_mgr.parse_options!
|
68
|
+
@notification_cb = nil
|
69
|
+
if !@opt_mgr.get_option(:notify_to).nil?
|
70
|
+
@notification_cb = ->(transfer_spec, global_status) do
|
71
|
+
@config.send_email_template(email_template_default: DEFAULT_TRANSFER_NOTIFY_TEMPLATE, values: {
|
72
|
+
subject: "#{Info::CMD_NAME} transfer: #{global_status}",
|
73
|
+
status: global_status,
|
74
|
+
ts: transfer_spec
|
75
|
+
})
|
76
|
+
end
|
77
|
+
end
|
68
78
|
end
|
69
79
|
|
70
80
|
def option_transfer_spec; @transfer_spec_command_line; end
|
@@ -251,22 +261,10 @@ module Aspera
|
|
251
261
|
agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
|
252
262
|
# list of: :success or "error message string"
|
253
263
|
result = agent_instance.wait_for_completion
|
254
|
-
|
264
|
+
@notification_cb&.call(transfer_spec, self.class.session_status(result))
|
255
265
|
return result
|
256
266
|
end
|
257
267
|
|
258
|
-
def send_email_transfer_notification(transfer_spec, statuses)
|
259
|
-
return if @opt_mgr.get_option(:notify_to).nil?
|
260
|
-
global_status = self.class.session_status(statuses)
|
261
|
-
email_vars = {
|
262
|
-
global_transfer_status: global_status,
|
263
|
-
subject: "#{PROGRAM_NAME} transfer: #{global_status}",
|
264
|
-
body: "Transfer is: #{global_status}",
|
265
|
-
ts: transfer_spec
|
266
|
-
}
|
267
|
-
@config.send_email_template(email_template_default: DEFAULT_TRANSFER_NOTIFY_TEMPLATE, values: email_vars)
|
268
|
-
end
|
269
|
-
|
270
268
|
# shut down if agent requires it
|
271
269
|
def shutdown
|
272
270
|
@agent.shutdown if @agent.respond_to?(:shutdown)
|
@@ -6,27 +6,28 @@ require 'ruby-progressbar'
|
|
6
6
|
|
7
7
|
module Aspera
|
8
8
|
module Cli
|
9
|
-
#
|
9
|
+
# Progress bar for transfers.
|
10
|
+
# Supports multi-session.
|
10
11
|
class TransferProgress
|
11
12
|
def initialize
|
12
13
|
reset
|
13
14
|
end
|
14
15
|
|
16
|
+
# Reset progress bar, to re-use it.
|
15
17
|
def reset
|
16
18
|
@progress_bar = nil
|
17
19
|
# key is session id
|
18
20
|
@sessions = {}
|
19
21
|
@completed = false
|
20
|
-
@title =
|
22
|
+
@title = nil
|
21
23
|
end
|
22
24
|
|
23
|
-
|
24
|
-
|
25
|
-
end
|
26
|
-
|
27
|
-
def event(session_id:,
|
25
|
+
# Called by user of progress bar with a status on a transfer session
|
26
|
+
# @param session_id the unique identifier of a transfer session
|
27
|
+
# @param type one of: pre_start, session_start, session_size, transfer, end
|
28
|
+
# @param info optional specific additional info for the given event type
|
29
|
+
def event(type, session_id:, info: nil)
|
28
30
|
Log.log.trace1{"progress: #{type} #{session_id} #{info}"}
|
29
|
-
Aspera.assert(!session_id.nil? || type.eql?(:pre_start)){'session_id is nil'}
|
30
31
|
return if @completed
|
31
32
|
if @progress_bar.nil?
|
32
33
|
@progress_bar = ProgressBar.create(
|
@@ -35,38 +36,55 @@ module Aspera
|
|
35
36
|
title: '',
|
36
37
|
total: nil)
|
37
38
|
end
|
38
|
-
|
39
|
+
progress_provided = false
|
39
40
|
case type
|
40
41
|
when :pre_start
|
42
|
+
# give opportunity to show progress of initialization with multiple status
|
43
|
+
Aspera.assert(session_id.nil?)
|
44
|
+
Aspera.assert_type(info, String)
|
45
|
+
# initialization of progress bar
|
41
46
|
@title = info
|
42
47
|
when :session_start
|
48
|
+
Aspera.assert_type(session_id, String)
|
49
|
+
Aspera.assert(info.nil?)
|
43
50
|
raise "Session #{session_id} already started" if @sessions[session_id]
|
44
51
|
@sessions[session_id] = {
|
45
52
|
job_size: 0, # total size of transfer (pre-calc)
|
46
53
|
current: 0
|
47
54
|
}
|
48
|
-
|
55
|
+
# remove last pre-start message if any
|
56
|
+
@title = nil
|
49
57
|
when :session_size
|
58
|
+
Aspera.assert_type(session_id, String)
|
59
|
+
Aspera.assert(!info.nil?)
|
50
60
|
@sessions[session_id][:job_size] = info.to_i
|
51
61
|
current_total = total(:job_size)
|
52
62
|
@progress_bar.total = current_total unless current_total.eql?(@progress_bar.total) || current_total < @progress_bar.progress
|
53
63
|
when :transfer
|
54
|
-
|
55
|
-
|
64
|
+
Aspera.assert_type(session_id, String)
|
65
|
+
if !@progress_bar.total.nil? && !info.nil?
|
66
|
+
progress_provided = true
|
56
67
|
@sessions[session_id][:current] = info.to_i
|
57
68
|
current_total = total(:current)
|
58
69
|
@progress_bar.progress = current_total unless @progress_bar.progress.eql?(current_total)
|
59
70
|
end
|
60
71
|
when :end
|
61
|
-
|
72
|
+
Aspera.assert(session_id, String)
|
73
|
+
Aspera.assert(info.nil?)
|
74
|
+
@title = nil
|
62
75
|
@completed = true
|
63
76
|
@progress_bar.finish
|
64
|
-
else
|
65
|
-
raise "Unknown event type #{type}"
|
77
|
+
else Aspera.error_unexpected_value(type){'event type'}
|
66
78
|
end
|
67
|
-
new_title = @sessions.length < 2 ? @title : "[#{@sessions.length}] #{@title}"
|
79
|
+
new_title = @sessions.length < 2 ? @title.to_s : "[#{@sessions.length}] #{@title}"
|
68
80
|
@progress_bar.title = new_title unless @progress_bar.title.eql?(new_title)
|
69
|
-
@progress_bar.increment if
|
81
|
+
@progress_bar.increment if !progress_provided && !@completed
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
def total(key)
|
87
|
+
@sessions.values.inject(0){|m, s|m + s[key]}
|
70
88
|
end
|
71
89
|
end
|
72
90
|
end
|
data/lib/aspera/cli/version.rb
CHANGED
@@ -14,7 +14,7 @@ module Aspera
|
|
14
14
|
OPTIONS_KEYS = %i[desc accepted_types default enum agents required cli ts deprecation].freeze
|
15
15
|
CLI_KEYS = %i[type switch convert variable].freeze
|
16
16
|
|
17
|
-
private_constant :CLI_OPTION_TYPE_SWITCH, :OPTIONS_KEYS, :CLI_KEYS
|
17
|
+
private_constant :CLI_OPTION_TYPE_SWITCH, :CLI_OPTION_TYPES, :OPTIONS_KEYS, :CLI_KEYS
|
18
18
|
|
19
19
|
class << self
|
20
20
|
# transform yes/no to true/false
|
@@ -22,8 +22,8 @@ module Aspera
|
|
22
22
|
case value
|
23
23
|
when 'yes' then return true
|
24
24
|
when 'no' then return false
|
25
|
+
else Aspera.error_unexpected_value(value){'only: yes or no: '}
|
25
26
|
end
|
26
|
-
raise "unsupported value: #{value}"
|
27
27
|
end
|
28
28
|
|
29
29
|
# Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
|
@@ -177,8 +177,7 @@ module Aspera
|
|
177
177
|
parameter_value = [parameter_value] unless parameter_value.is_a?(Array)
|
178
178
|
# if transfer_spec value is an array, applies option many times
|
179
179
|
parameter_value.each{|v|add_command_line_options([options[:cli][:switch], v])}
|
180
|
-
else
|
181
|
-
raise "ERROR: unknown option processing type: #{processing_type}/#{processing_type.class}"
|
180
|
+
else Aspera.error_unexpected_value(processing_type){processing_type.class.name}
|
182
181
|
end
|
183
182
|
end
|
184
183
|
end
|
data/lib/aspera/coverage.rb
CHANGED
@@ -17,5 +17,17 @@ if ENV.key?('ENABLE_COVERAGE')
|
|
17
17
|
SimpleCov.result.format!
|
18
18
|
$stdout.reopen(original_file_descriptor)
|
19
19
|
end
|
20
|
-
|
20
|
+
# lines with those words are ignored from coverage
|
21
|
+
no_cov_functions = %w[error_unreachable_line error_unexpected_value Log.log.trace].freeze
|
22
|
+
SimpleCov.start do
|
23
|
+
add_filter 'lib/aspera/cli/plugins/faspex.rb'
|
24
|
+
add_filter 'lib/aspera/node_simulator.rb'
|
25
|
+
add_filter 'lib/aspera/keychain/macos_security.rb'
|
26
|
+
add_filter do |source_file|
|
27
|
+
source_file.lines.each do |line|
|
28
|
+
line.skipped! if no_cov_functions.any?{|i|line.src.include?(i)}
|
29
|
+
end
|
30
|
+
false
|
31
|
+
end
|
32
|
+
end
|
21
33
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -96,24 +96,41 @@ module Aspera
|
|
96
96
|
Kernel.send('lave'.reverse, code, empty_binding, file, line)
|
97
97
|
end
|
98
98
|
|
99
|
+
def log_spawn(env:, exec:, args:)
|
100
|
+
[
|
101
|
+
'execute:'.red,
|
102
|
+
env.map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
|
103
|
+
Shellwords.shellescape(exec),
|
104
|
+
args.map{|a|Shellwords.shellescape(a)}
|
105
|
+
].flatten.join(' ')
|
106
|
+
end
|
107
|
+
|
99
108
|
# start process in background, or raise exception
|
100
109
|
# caller can call Process.wait on returned value
|
101
|
-
def secure_spawn(
|
102
|
-
Log.log.debug
|
103
|
-
[
|
104
|
-
'execute:'.red,
|
105
|
-
env.map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
|
106
|
-
Shellwords.shellescape(exec),
|
107
|
-
args.map{|a|Shellwords.shellescape(a)}
|
108
|
-
].flatten.join(' ')
|
109
|
-
end
|
110
|
-
return if log_only
|
110
|
+
def secure_spawn(exec:, args: [], env: [])
|
111
|
+
Log.log.debug {log_spawn(env: env, exec: exec, args: args)}
|
111
112
|
# start ascp in separate process
|
112
113
|
ascp_pid = Process.spawn(env, [exec, exec], *args, close_others: true)
|
113
114
|
Log.log.debug{"pid: #{ascp_pid}"}
|
114
115
|
return ascp_pid
|
115
116
|
end
|
116
117
|
|
118
|
+
# @param exec [String] path to executable
|
119
|
+
# @param args [Array] arguments to executable
|
120
|
+
# @param opts [Hash] options to capture3
|
121
|
+
# @return stdout of executable or raise expcetion
|
122
|
+
def secure_capture(exec:, args: [], **opts)
|
123
|
+
Aspera.assert_type(exec, String)
|
124
|
+
Aspera.assert_type(args, Array)
|
125
|
+
Aspera.assert_type(opts, Hash)
|
126
|
+
Log.log.debug {log_spawn(env: {}, exec: exec, args: args)}
|
127
|
+
stdout, stderr, status = Open3.capture3(exec, *args, **opts)
|
128
|
+
Log.log.debug{"status=#{status}, stderr=#{stderr}"}
|
129
|
+
Log.log.trace1{"stdout=#{stdout}"}
|
130
|
+
raise "process failed: #{status.exitstatus} : #{stderr}" unless status.success?
|
131
|
+
return stdout
|
132
|
+
end
|
133
|
+
|
117
134
|
# Write content to a file, with restricted access
|
118
135
|
# @param path [String] the file path
|
119
136
|
# @param force [Boolean] if true, overwrite the file
|
@@ -192,14 +209,13 @@ module Aspera
|
|
192
209
|
@terminal_supports_unicode = nil
|
193
210
|
end
|
194
211
|
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
212
|
+
# @return true if we can display Unicode characters
|
213
|
+
# https://www.gnu.org/software/libc/manual/html_node/Locale-Categories.html
|
214
|
+
# https://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html
|
215
|
+
def terminal_supports_unicode?
|
216
|
+
@terminal_supports_unicode = self.class.terminal? && %w(LC_ALL LC_CTYPE LANG).any?{|var|ENV[var]&.include?('UTF-8')} if @terminal_supports_unicode.nil?
|
217
|
+
return @terminal_supports_unicode
|
218
|
+
end
|
203
219
|
|
204
220
|
# Allows a user to open a Url
|
205
221
|
# if method is "text", then URL is displayed on terminal
|
data/lib/aspera/faspex_gw.rb
CHANGED
@@ -44,7 +44,7 @@ module Aspera
|
|
44
44
|
'note' => faspex_pkg_delivery['note'],
|
45
45
|
'recipients' => faspex_pkg_delivery['recipients'].map{|name|{'name'=>name}}
|
46
46
|
}
|
47
|
-
package = @app_api.create('packages', package_data)
|
47
|
+
package = @app_api.create('packages', package_data)
|
48
48
|
# TODO: option to send from remote source or httpgw
|
49
49
|
transfer_spec = @app_api.call(
|
50
50
|
operation: 'POST',
|
@@ -85,7 +85,7 @@ module Aspera
|
|
85
85
|
rescue => e
|
86
86
|
response.status = 500
|
87
87
|
response['Content-Type'] = 'application/json'
|
88
|
-
response.body = {error: e.message}.to_json
|
88
|
+
response.body = {error: e.message, stacktrace: e.backtrace}.to_json
|
89
89
|
Log.log.error(e.message)
|
90
90
|
Log.log.debug{e.backtrace.join("\n")}
|
91
91
|
end
|
data/lib/aspera/json_rpc.rb
CHANGED
@@ -34,7 +34,7 @@ module Aspera
|
|
34
34
|
method: "#{@namespace}#{method}",
|
35
35
|
params: args,
|
36
36
|
id: @request_id += 1
|
37
|
-
})
|
37
|
+
})
|
38
38
|
Aspera.assert_type(data, Hash){'response'}
|
39
39
|
Aspera.assert(data['jsonrpc'] == JSON_RPC_VERSION){'bad version in response'}
|
40
40
|
Aspera.assert(data.key?('id')){'missing id in response'}
|
@@ -4,7 +4,7 @@
|
|
4
4
|
require 'aspera/cli/info'
|
5
5
|
require 'aspera/log'
|
6
6
|
require 'aspera/assert'
|
7
|
-
require '
|
7
|
+
require 'aspera/environment'
|
8
8
|
|
9
9
|
# enhance the gem to support other key chains
|
10
10
|
module Aspera
|
@@ -48,20 +48,15 @@ module Aspera
|
|
48
48
|
options[:path] = uri.path unless ['', '/'].include?(uri.path)
|
49
49
|
options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
|
50
50
|
end
|
51
|
-
|
51
|
+
command_args = [command]
|
52
52
|
options&.each do |k, v|
|
53
53
|
Aspera.assert(supported.key?(k)){"unknown option: #{k}"}
|
54
54
|
next if v.nil?
|
55
|
-
|
56
|
-
|
55
|
+
command_args.push("-#{supported[k]}")
|
56
|
+
command_args.push(v.shellescape) unless v.empty?
|
57
57
|
end
|
58
|
-
|
59
|
-
|
60
|
-
stdout, stderr, status = Open3.capture3(*command_line)
|
61
|
-
Log.log.debug{"status=#{status}, stderr=#{stderr}"}
|
62
|
-
Log.log.trace1{"stdout=#{stdout}"}
|
63
|
-
raise "#{SECURITY_UTILITY} failed: #{status.exitstatus} : #{stderr}" unless status.success?
|
64
|
-
return stdout
|
58
|
+
command_args.push(last_opt) unless last_opt.nil?
|
59
|
+
return Environment.secure_capture(exec: SECURITY_UTILITY, args: command_args)
|
65
60
|
end
|
66
61
|
|
67
62
|
def key_chains(output)
|
@@ -78,7 +73,7 @@ module Aspera
|
|
78
73
|
|
79
74
|
def list(options={})
|
80
75
|
Aspera.assert_values(options[:domain], DOMAINS, exception_class: ArgumentError){'domain'} unless options[:domain].nil?
|
81
|
-
key_chains(execute('list-
|
76
|
+
key_chains(execute('list-keychains', options, LIST_OPTIONS))
|
82
77
|
end
|
83
78
|
|
84
79
|
def by_name(name)
|
data/lib/aspera/log.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/assert'
|
3
4
|
require 'aspera/colors'
|
4
5
|
require 'aspera/secret_hider'
|
5
6
|
require 'logger'
|
@@ -73,8 +74,7 @@ module Aspera
|
|
73
74
|
JSON.pretty_generate(object) rescue PP.pp(object, +'')
|
74
75
|
when :ruby
|
75
76
|
PP.pp(object, +'')
|
76
|
-
else
|
77
|
-
raise 'wrong parameter, expect ruby or json'
|
77
|
+
else error_unexpected_value(@@format){'dump format'}
|
78
78
|
end
|
79
79
|
"#{name.to_s.green} (#{@@format})=\n#{result}"
|
80
80
|
end
|
@@ -127,8 +127,7 @@ module Aspera
|
|
127
127
|
Syslog::Logger.make_methods(severity.downcase)
|
128
128
|
end
|
129
129
|
@logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
|
130
|
-
else
|
131
|
-
raise "unknown log type: #{new_log_type}, use one of: #{LOG_TYPES.join(', ')}"
|
130
|
+
else error_unexpected_value(new_log_type){"log type (#{LOG_TYPES.join(', ')})"}
|
132
131
|
end
|
133
132
|
@logger.level = current_severity_integer
|
134
133
|
@logger_type = new_log_type
|