aspera-cli 4.19.0 → 4.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +20 -0
  4. data/CONTRIBUTING.md +16 -4
  5. data/README.md +344 -164
  6. data/bin/asession +26 -19
  7. data/examples/build_exec +65 -76
  8. data/examples/build_exec_rubyc +40 -0
  9. data/examples/get_proto_file.rb +7 -0
  10. data/lib/aspera/agent/alpha.rb +8 -8
  11. data/lib/aspera/agent/base.rb +2 -18
  12. data/lib/aspera/agent/connect.rb +14 -13
  13. data/lib/aspera/agent/direct.rb +23 -24
  14. data/lib/aspera/agent/httpgw.rb +2 -3
  15. data/lib/aspera/agent/node.rb +10 -10
  16. data/lib/aspera/agent/trsdk.rb +17 -20
  17. data/lib/aspera/api/alee.rb +15 -0
  18. data/lib/aspera/api/aoc.rb +126 -97
  19. data/lib/aspera/api/ats.rb +1 -1
  20. data/lib/aspera/api/cos_node.rb +1 -1
  21. data/lib/aspera/api/httpgw.rb +15 -10
  22. data/lib/aspera/api/node.rb +33 -12
  23. data/lib/aspera/ascmd.rb +56 -48
  24. data/lib/aspera/ascp/installation.rb +99 -42
  25. data/lib/aspera/ascp/management.rb +3 -2
  26. data/lib/aspera/ascp/products.rb +12 -0
  27. data/lib/aspera/assert.rb +10 -5
  28. data/lib/aspera/cli/formatter.rb +27 -17
  29. data/lib/aspera/cli/hints.rb +2 -1
  30. data/lib/aspera/cli/info.rb +12 -10
  31. data/lib/aspera/cli/main.rb +16 -13
  32. data/lib/aspera/cli/manager.rb +5 -0
  33. data/lib/aspera/cli/plugin.rb +15 -29
  34. data/lib/aspera/cli/plugins/alee.rb +3 -3
  35. data/lib/aspera/cli/plugins/aoc.rb +222 -194
  36. data/lib/aspera/cli/plugins/ats.rb +16 -14
  37. data/lib/aspera/cli/plugins/config.rb +53 -45
  38. data/lib/aspera/cli/plugins/console.rb +3 -3
  39. data/lib/aspera/cli/plugins/faspex.rb +11 -21
  40. data/lib/aspera/cli/plugins/faspex5.rb +44 -42
  41. data/lib/aspera/cli/plugins/faspio.rb +2 -2
  42. data/lib/aspera/cli/plugins/httpgw.rb +1 -1
  43. data/lib/aspera/cli/plugins/node.rb +153 -95
  44. data/lib/aspera/cli/plugins/orchestrator.rb +14 -13
  45. data/lib/aspera/cli/plugins/preview.rb +8 -9
  46. data/lib/aspera/cli/plugins/server.rb +5 -9
  47. data/lib/aspera/cli/plugins/shares.rb +2 -2
  48. data/lib/aspera/cli/sync_actions.rb +2 -2
  49. data/lib/aspera/cli/transfer_agent.rb +12 -14
  50. data/lib/aspera/cli/transfer_progress.rb +35 -17
  51. data/lib/aspera/cli/version.rb +1 -1
  52. data/lib/aspera/command_line_builder.rb +3 -4
  53. data/lib/aspera/coverage.rb +13 -1
  54. data/lib/aspera/environment.rb +34 -18
  55. data/lib/aspera/faspex_gw.rb +2 -2
  56. data/lib/aspera/json_rpc.rb +1 -1
  57. data/lib/aspera/keychain/macos_security.rb +7 -12
  58. data/lib/aspera/log.rb +3 -4
  59. data/lib/aspera/oauth/base.rb +39 -45
  60. data/lib/aspera/oauth/factory.rb +11 -4
  61. data/lib/aspera/oauth/generic.rb +4 -8
  62. data/lib/aspera/oauth/jwt.rb +3 -3
  63. data/lib/aspera/oauth/url_json.rb +1 -2
  64. data/lib/aspera/oauth/web.rb +5 -2
  65. data/lib/aspera/persistency_action_once.rb +16 -8
  66. data/lib/aspera/preview/utils.rb +5 -16
  67. data/lib/aspera/rest.rb +100 -76
  68. data/lib/aspera/transfer/faux_file.rb +4 -4
  69. data/lib/aspera/transfer/parameters.rb +14 -16
  70. data/lib/aspera/transfer/spec.rb +12 -12
  71. data/lib/aspera/transfer/sync.rb +1 -5
  72. data/lib/aspera/transfer/uri.rb +1 -1
  73. data/lib/aspera/uri_reader.rb +1 -1
  74. data/lib/aspera/web_auth.rb +166 -17
  75. data/lib/aspera/web_server_simple.rb +4 -3
  76. data/lib/transfer_pb.rb +84 -0
  77. data/lib/transfer_services_pb.rb +82 -0
  78. data.tar.gz.sig +0 -0
  79. metadata +24 -5
  80. metadata.gz.sig +0 -0
@@ -22,7 +22,7 @@ module Aspera
22
22
  next unless base_url.match?('https?://')
23
23
  api = Rest.new(base_url: base_url)
24
24
  test_endpoint = 'api/remote_node_ping'
25
- result = api.read(test_endpoint, {format: :json})
25
+ result = api.call(operation: 'GET', subpath: test_endpoint, headers: {'Accept' => 'application/json'}, query: {format: :json})
26
26
  next unless result[:data]['remote_orchestrator_info']
27
27
  url = result[:http].uri.to_s
28
28
  return {
@@ -69,7 +69,7 @@ module Aspera
69
69
  # @param format [String] the format to request, 'json', 'xml', nil
70
70
  # @param args [Hash] the arguments to pass
71
71
  # @param xml_arrays [Boolean] if true, force arrays in xml parsing
72
- def call_ao(endpoint, prefix: 'api', id: nil, ret_style: nil, format: 'json', args: nil, xml_arrays: true)
72
+ def call_ao(endpoint, prefix: 'api', id: nil, ret_style: nil, format: 'json', args: nil, xml_arrays: true, http: false)
73
73
  # calls are GET
74
74
  call_args = {operation: 'GET', subpath: endpoint}
75
75
  # specify prefix if necessary
@@ -91,9 +91,10 @@ module Aspera
91
91
  end
92
92
  end
93
93
  result = @api_orch.call(**call_args)
94
+ return result[:http] if http
94
95
  result[:data] = XmlSimple.xml_in(result[:http].body, {'ForceArray' => xml_arrays}) if format.eql?('xml')
95
96
  Log.log.debug{Log.dump(:data, result[:data])}
96
- return result
97
+ return result[:data]
97
98
  end
98
99
 
99
100
  def execute_action
@@ -126,7 +127,7 @@ module Aspera
126
127
  when :health
127
128
  nagios = Nagios.new
128
129
  begin
129
- info = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)[:data]
130
+ info = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
130
131
  nagios.add_ok('api', 'accessible')
131
132
  nagios.check_product_version('api', 'orchestrator', info['orchestrator-version'])
132
133
  rescue StandardError => e
@@ -134,15 +135,15 @@ module Aspera
134
135
  end
135
136
  return nagios.result
136
137
  when :info
137
- result = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)[:data]
138
+ result = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
138
139
  return {type: :single_object, data: result}
139
140
  when :processes
140
141
  # TODO: Bug ? API has only XML format
141
- result = call_ao('processes_status', format: 'xml')[:data]
142
+ result = call_ao('processes_status', format: 'xml')
142
143
  return {type: :object_list, data: result['process']}
143
144
  when :plugins
144
145
  # TODO: Bug ? only json format on url
145
- result = call_ao('plugin_version')[:data]
146
+ result = call_ao('plugin_version')
146
147
  return {type: :object_list, data: result['Plugin']}
147
148
  when :workflow
148
149
  command = options.get_next_command(%i[list status inputs details start export])
@@ -152,23 +153,23 @@ module Aspera
152
153
  case command
153
154
  when :status
154
155
  wf_id = nil if wf_id.eql?(SpecialValues::ALL)
155
- result = call_ao('workflows_status', id: wf_id)[:data]
156
+ result = call_ao('workflows_status', id: wf_id)
156
157
  return {type: :object_list, data: result['workflows']['workflow']}
157
158
  when :list
158
- result = call_ao('workflows_list', id: 0)[:data]
159
+ result = call_ao('workflows_list', id: 0)
159
160
  return {
160
161
  type: :object_list,
161
162
  data: result['workflows']['workflow'],
162
163
  fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification]
163
164
  }
164
165
  when :details
165
- result = call_ao('workflow_details', id: wf_id)[:data]
166
+ result = call_ao('workflow_details', id: wf_id)
166
167
  return {type: :object_list, data: result['workflows']['workflow']['statuses']}
167
168
  when :inputs
168
- result = call_ao('workflow_inputs_spec', id: wf_id)[:data]
169
+ result = call_ao('workflow_inputs_spec', id: wf_id)
169
170
  return {type: :single_object, data: result['workflow_inputs_spec']}
170
171
  when :export
171
- result = call_ao('export_workflow', id: wf_id, format: nil)[:http]
172
+ result = call_ao('export_workflow', id: wf_id, format: nil, http: true)
172
173
  return {type: :text, data: result.body}
173
174
  when :start
174
175
  result = {
@@ -196,7 +197,7 @@ module Aspera
196
197
  if call_params['synchronous']
197
198
  result[:type] = :text
198
199
  end
199
- result[:data] = call_ao('initiate', id: wf_id, args: call_params)[:data]
200
+ result[:data] = call_ao('initiate', id: wf_id, args: call_params)
200
201
  return result
201
202
  end
202
203
  else Aspera.error_unexpected_value(command)
@@ -133,7 +133,6 @@ module Aspera
133
133
  subpath: "files/#{file_id}/files",
134
134
  headers: headers,
135
135
  query: request_args)[:data]
136
- # return @api_node.read("files/#{file_id}/files",request_args)[:data]
137
136
  end
138
137
 
139
138
  # old version based on folders
@@ -146,7 +145,7 @@ module Aspera
146
145
  # optionally add iteration token from persistency
147
146
  events_filter['iteration_token'] = iteration_persistency.data.first unless iteration_persistency.nil?
148
147
  begin
149
- events = @api_node.read('events', events_filter)[:data]
148
+ events = @api_node.read('events', events_filter)
150
149
  rescue RestCallError => e
151
150
  if e.message.include?('Invalid iteration_token')
152
151
  Log.log.warn{"Retrying without iteration token: #{e}"}
@@ -164,7 +163,7 @@ module Aspera
164
163
  folder_id = event.dig('data', 'tags', Transfer::Spec::TAG_RESERVED, 'node', 'file_id')
165
164
  folder_id ||= event.dig('data', 'file_id')
166
165
  if !folder_id.nil?
167
- folder_entry = @api_node.read("files/#{folder_id}")[:data] rescue nil
166
+ folder_entry = @api_node.read("files/#{folder_id}") rescue nil
168
167
  scan_folder_files(folder_entry) unless folder_entry.nil?
169
168
  end
170
169
  end
@@ -188,12 +187,12 @@ module Aspera
188
187
  }
189
188
  # optionally add iteration token from persistency
190
189
  events_filter['iteration_token'] = iteration_persistency.data.first unless iteration_persistency.nil?
191
- events = @api_node.read('events', events_filter)[:data]
190
+ events = @api_node.read('events', events_filter)
192
191
  return if events.empty?
193
192
  events.each do |event|
194
193
  # process only files
195
194
  if event.dig('data', 'type').eql?('file')
196
- file_entry = @api_node.read("files/#{event['data']['id']}")[:data] rescue nil
195
+ file_entry = @api_node.read("files/#{event['data']['id']}") rescue nil
197
196
  if !file_entry.nil? &&
198
197
  @option_skip_folders.none?{|d|file_entry['path'].start_with?(d)}
199
198
  file_entry['parent_file_id'] = event['data']['parent_file_id']
@@ -248,7 +247,7 @@ module Aspera
248
247
  # original_mtime=DateTime.parse(entry['modified_time'])
249
248
  # out: where previews are generated
250
249
  local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
251
- file_info = @api_node.read("files/#{entry['id']}")[:data]
250
+ file_info = @api_node.read("files/#{entry['id']}")
252
251
  # TODO: this does not work because previews is hidden in api (gen4)
253
252
  # this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{name: @entry_preview_folder_name})
254
253
  # TODO: use gen3 api to list files and get date
@@ -410,10 +409,10 @@ module Aspera
410
409
  @api_node = Api::Node.new(**basic_auth_params)
411
410
  @transfer_server_address = URI.parse(@api_node.base_url).host
412
411
  # get current access key
413
- @access_key_self = @api_node.read('access_keys/self')[:data]
412
+ @access_key_self = @api_node.read('access_keys/self')
414
413
  # TODO: check events is activated here:
415
414
  # note that docroot is good to look at as well
416
- node_info = @api_node.read('info')[:data]
415
+ node_info = @api_node.read('info')
417
416
  Log.log.debug{"root: #{node_info['docroot']}"}
418
417
  @access_remote = @option_file_access.eql?(:remote)
419
418
  Log.log.debug{"remote: #{@access_remote}"}
@@ -463,7 +462,7 @@ module Aspera
463
462
  'type' => 'folder',
464
463
  'path' => '/' }
465
464
  else
466
- @api_node.read("files/#{scan_id}")[:data]
465
+ @api_node.read("files/#{scan_id}")
467
466
  end
468
467
  @filter_block = Api::Node.file_matcher_from_argument(options)
469
468
  scan_folder_files(folder_info, scan_path)
@@ -9,6 +9,7 @@ require 'aspera/ssh'
9
9
  require 'aspera/nagios'
10
10
  require 'aspera/log'
11
11
  require 'aspera/assert'
12
+ require 'aspera/environment'
12
13
  require 'tempfile'
13
14
  require 'open3'
14
15
 
@@ -32,14 +33,8 @@ module Aspera
32
33
  private_constant :SSH_SCHEME, :URI_SCHEMES, :ASCMD_ALIASES, :TRANSFER_COMMANDS
33
34
 
34
35
  class LocalExecutor
35
- def execute(cmd, line)
36
- # concatenate arguments, enclose in double quotes
37
- cmd = cmd.map{|v|%Q("#{v}")}.join(' ') if cmd.is_a?(Array)
38
- Log.log.debug{"Executing: #{cmd} with '#{line}'"}
39
- stdout_str, stderr_str, status = Open3.capture3(cmd, stdin_data: line, binmode: true)
40
- Log.log.debug{"exec status: #{status} -> #{stderr_str}"}
41
- raise "command #{cmd} failed with code #{status.exitstatus} #{stderr_str}" unless status.success?
42
- return stdout_str
36
+ def execute(ascmd_path, line)
37
+ return Environment.secure_capture(exec: ascmd_path, stdin_data: line, binmode: true)
43
38
  end
44
39
  end
45
40
 
@@ -157,7 +152,8 @@ module Aspera
157
152
  Log.log.debug{"SSH keys=#{ssh_key_list}"}
158
153
  if !ssh_key_list.empty?
159
154
  @ssh_opts[:keys] = ssh_key_list
160
- server_transfer_spec['ssh_private_key'] = File.read(ssh_key_list.first)
155
+ # PEM as per RFC 7468
156
+ server_transfer_spec['ssh_private_key'] = File.read(ssh_key_list.first).strip
161
157
  Log.log.warn{'Using only first SSH key for transfers'} unless ssh_key_list.length.eql?(1)
162
158
  cred_set = true
163
159
  end
@@ -144,11 +144,11 @@ module Aspera
144
144
  entity_parameters.each_key do |p|
145
145
  raise "unsupported field: #{p}, use: #{SAML_IMPORT_ALLOWED.join(',')}" unless SAML_IMPORT_ALLOWED.include?(p)
146
146
  end
147
- api_shares_admin.create("#{entities_path}/import", entity_parameters)[:data]
147
+ api_shares_admin.create("#{entities_path}/import", entity_parameters)
148
148
  end
149
149
  when :add # ldap
150
150
  return do_bulk_operation(command: entity_verb, descr: "#{entity_type} name", values: String) do |entity_name|
151
- api_shares_admin.create(entities_path, {entity_type=>entity_name})[:data]
151
+ api_shares_admin.create(entities_path, {entity_type=>entity_name})
152
152
  end
153
153
  when :users # group
154
154
  return entity_action(api_shares_admin, "#{entities_path}/#{instance_identifier}/#{entities_prefix}users")
@@ -76,9 +76,9 @@ module Aspera
76
76
  end
77
77
  if !session_info.key?('name')
78
78
  # if no name is specified, generate one from simple arguments
79
- session_info['name'] = SYNC_SIMPLE_ARGS.map do |arg_name|
79
+ session_info['name'] = SYNC_SIMPLE_ARGS.filter_map do |arg_name|
80
80
  arguments[arg_name]&.gsub(/[^a-zA-Z0-9]/, '')
81
- end.compact.reject(&:empty?).join('_')
81
+ end.reject(&:empty?).join('_')
82
82
  end
83
83
  end
84
84
  end
@@ -22,7 +22,7 @@ module Aspera
22
22
  To: <<%=to%>>
23
23
  Subject: <%=subject%>
24
24
 
25
- Transfer is: <%=global_transfer_status%>
25
+ Transfer is: <%=status%>
26
26
 
27
27
  <%=ts.to_yaml%>
28
28
  END_OF_TEMPLATE
@@ -65,6 +65,16 @@ module Aspera
65
65
  @opt_mgr.declare(:transfer, 'Type of transfer agent', values: TRANSFER_AGENTS, default: :direct)
66
66
  @opt_mgr.declare(:transfer_info, 'Parameters for transfer agent', types: Hash, handler: {o: self, m: :transfer_info})
67
67
  @opt_mgr.parse_options!
68
+ @notification_cb = nil
69
+ if !@opt_mgr.get_option(:notify_to).nil?
70
+ @notification_cb = ->(transfer_spec, global_status) do
71
+ @config.send_email_template(email_template_default: DEFAULT_TRANSFER_NOTIFY_TEMPLATE, values: {
72
+ subject: "#{Info::CMD_NAME} transfer: #{global_status}",
73
+ status: global_status,
74
+ ts: transfer_spec
75
+ })
76
+ end
77
+ end
68
78
  end
69
79
 
70
80
  def option_transfer_spec; @transfer_spec_command_line; end
@@ -251,22 +261,10 @@ module Aspera
251
261
  agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
252
262
  # list of: :success or "error message string"
253
263
  result = agent_instance.wait_for_completion
254
- send_email_transfer_notification(transfer_spec, result)
264
+ @notification_cb&.call(transfer_spec, self.class.session_status(result))
255
265
  return result
256
266
  end
257
267
 
258
- def send_email_transfer_notification(transfer_spec, statuses)
259
- return if @opt_mgr.get_option(:notify_to).nil?
260
- global_status = self.class.session_status(statuses)
261
- email_vars = {
262
- global_transfer_status: global_status,
263
- subject: "#{PROGRAM_NAME} transfer: #{global_status}",
264
- body: "Transfer is: #{global_status}",
265
- ts: transfer_spec
266
- }
267
- @config.send_email_template(email_template_default: DEFAULT_TRANSFER_NOTIFY_TEMPLATE, values: email_vars)
268
- end
269
-
270
268
  # shut down if agent requires it
271
269
  def shutdown
272
270
  @agent.shutdown if @agent.respond_to?(:shutdown)
@@ -6,27 +6,28 @@ require 'ruby-progressbar'
6
6
 
7
7
  module Aspera
8
8
  module Cli
9
- # progress bar for transfers, supports multi-session
9
+ # Progress bar for transfers.
10
+ # Supports multi-session.
10
11
  class TransferProgress
11
12
  def initialize
12
13
  reset
13
14
  end
14
15
 
16
+ # Reset progress bar, to re-use it.
15
17
  def reset
16
18
  @progress_bar = nil
17
19
  # key is session id
18
20
  @sessions = {}
19
21
  @completed = false
20
- @title = ''
22
+ @title = nil
21
23
  end
22
24
 
23
- def total(key)
24
- @sessions.values.inject(0){|m, s|m + s[key]}
25
- end
26
-
27
- def event(session_id:, type:, info: nil)
25
+ # Called by user of progress bar with a status on a transfer session
26
+ # @param session_id the unique identifier of a transfer session
27
+ # @param type one of: pre_start, session_start, session_size, transfer, end
28
+ # @param info optional specific additional info for the given event type
29
+ def event(type, session_id:, info: nil)
28
30
  Log.log.trace1{"progress: #{type} #{session_id} #{info}"}
29
- Aspera.assert(!session_id.nil? || type.eql?(:pre_start)){'session_id is nil'}
30
31
  return if @completed
31
32
  if @progress_bar.nil?
32
33
  @progress_bar = ProgressBar.create(
@@ -35,38 +36,55 @@ module Aspera
35
36
  title: '',
36
37
  total: nil)
37
38
  end
38
- need_increment = true
39
+ progress_provided = false
39
40
  case type
40
41
  when :pre_start
42
+ # give opportunity to show progress of initialization with multiple status
43
+ Aspera.assert(session_id.nil?)
44
+ Aspera.assert_type(info, String)
45
+ # initialization of progress bar
41
46
  @title = info
42
47
  when :session_start
48
+ Aspera.assert_type(session_id, String)
49
+ Aspera.assert(info.nil?)
43
50
  raise "Session #{session_id} already started" if @sessions[session_id]
44
51
  @sessions[session_id] = {
45
52
  job_size: 0, # total size of transfer (pre-calc)
46
53
  current: 0
47
54
  }
48
- @title = ''
55
+ # remove last pre-start message if any
56
+ @title = nil
49
57
  when :session_size
58
+ Aspera.assert_type(session_id, String)
59
+ Aspera.assert(!info.nil?)
50
60
  @sessions[session_id][:job_size] = info.to_i
51
61
  current_total = total(:job_size)
52
62
  @progress_bar.total = current_total unless current_total.eql?(@progress_bar.total) || current_total < @progress_bar.progress
53
63
  when :transfer
54
- if !@progress_bar.total.nil?
55
- need_increment = false
64
+ Aspera.assert_type(session_id, String)
65
+ if !@progress_bar.total.nil? && !info.nil?
66
+ progress_provided = true
56
67
  @sessions[session_id][:current] = info.to_i
57
68
  current_total = total(:current)
58
69
  @progress_bar.progress = current_total unless @progress_bar.progress.eql?(current_total)
59
70
  end
60
71
  when :end
61
- @title = ''
72
+ Aspera.assert(session_id, String)
73
+ Aspera.assert(info.nil?)
74
+ @title = nil
62
75
  @completed = true
63
76
  @progress_bar.finish
64
- else
65
- raise "Unknown event type #{type}"
77
+ else Aspera.error_unexpected_value(type){'event type'}
66
78
  end
67
- new_title = @sessions.length < 2 ? @title : "[#{@sessions.length}] #{@title}"
79
+ new_title = @sessions.length < 2 ? @title.to_s : "[#{@sessions.length}] #{@title}"
68
80
  @progress_bar.title = new_title unless @progress_bar.title.eql?(new_title)
69
- @progress_bar.increment if need_increment && !@completed
81
+ @progress_bar.increment if !progress_provided && !@completed
82
+ end
83
+
84
+ private
85
+
86
+ def total(key)
87
+ @sessions.values.inject(0){|m, s|m + s[key]}
70
88
  end
71
89
  end
72
90
  end
@@ -4,6 +4,6 @@ module Aspera
4
4
  module Cli
5
5
  # for beta add extension : .beta1
6
6
  # for dev version add extension : .pre
7
- VERSION = '4.19.0'
7
+ VERSION = '4.20.0'
8
8
  end
9
9
  end
@@ -14,7 +14,7 @@ module Aspera
14
14
  OPTIONS_KEYS = %i[desc accepted_types default enum agents required cli ts deprecation].freeze
15
15
  CLI_KEYS = %i[type switch convert variable].freeze
16
16
 
17
- private_constant :CLI_OPTION_TYPE_SWITCH, :OPTIONS_KEYS, :CLI_KEYS
17
+ private_constant :CLI_OPTION_TYPE_SWITCH, :CLI_OPTION_TYPES, :OPTIONS_KEYS, :CLI_KEYS
18
18
 
19
19
  class << self
20
20
  # transform yes/no to true/false
@@ -22,8 +22,8 @@ module Aspera
22
22
  case value
23
23
  when 'yes' then return true
24
24
  when 'no' then return false
25
+ else Aspera.error_unexpected_value(value){'only: yes or no: '}
25
26
  end
26
- raise "unsupported value: #{value}"
27
27
  end
28
28
 
29
29
  # Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
@@ -177,8 +177,7 @@ module Aspera
177
177
  parameter_value = [parameter_value] unless parameter_value.is_a?(Array)
178
178
  # if transfer_spec value is an array, applies option many times
179
179
  parameter_value.each{|v|add_command_line_options([options[:cli][:switch], v])}
180
- else
181
- raise "ERROR: unknown option processing type: #{processing_type}/#{processing_type.class}"
180
+ else Aspera.error_unexpected_value(processing_type){processing_type.class.name}
182
181
  end
183
182
  end
184
183
  end
@@ -17,5 +17,17 @@ if ENV.key?('ENABLE_COVERAGE')
17
17
  SimpleCov.result.format!
18
18
  $stdout.reopen(original_file_descriptor)
19
19
  end
20
- SimpleCov.start
20
+ # lines with those words are ignored from coverage
21
+ no_cov_functions = %w[error_unreachable_line error_unexpected_value Log.log.trace].freeze
22
+ SimpleCov.start do
23
+ add_filter 'lib/aspera/cli/plugins/faspex.rb'
24
+ add_filter 'lib/aspera/node_simulator.rb'
25
+ add_filter 'lib/aspera/keychain/macos_security.rb'
26
+ add_filter do |source_file|
27
+ source_file.lines.each do |line|
28
+ line.skipped! if no_cov_functions.any?{|i|line.src.include?(i)}
29
+ end
30
+ false
31
+ end
32
+ end
21
33
  end
@@ -96,24 +96,41 @@ module Aspera
96
96
  Kernel.send('lave'.reverse, code, empty_binding, file, line)
97
97
  end
98
98
 
99
+ def log_spawn(env:, exec:, args:)
100
+ [
101
+ 'execute:'.red,
102
+ env.map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
103
+ Shellwords.shellescape(exec),
104
+ args.map{|a|Shellwords.shellescape(a)}
105
+ ].flatten.join(' ')
106
+ end
107
+
99
108
  # start process in background, or raise exception
100
109
  # caller can call Process.wait on returned value
101
- def secure_spawn(env:, exec:, args:, log_only: false)
102
- Log.log.debug do
103
- [
104
- 'execute:'.red,
105
- env.map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
106
- Shellwords.shellescape(exec),
107
- args.map{|a|Shellwords.shellescape(a)}
108
- ].flatten.join(' ')
109
- end
110
- return if log_only
110
+ def secure_spawn(exec:, args: [], env: [])
111
+ Log.log.debug {log_spawn(env: env, exec: exec, args: args)}
111
112
  # start ascp in separate process
112
113
  ascp_pid = Process.spawn(env, [exec, exec], *args, close_others: true)
113
114
  Log.log.debug{"pid: #{ascp_pid}"}
114
115
  return ascp_pid
115
116
  end
116
117
 
118
+ # @param exec [String] path to executable
119
+ # @param args [Array] arguments to executable
120
+ # @param opts [Hash] options to capture3
121
+ # @return stdout of executable or raise expcetion
122
+ def secure_capture(exec:, args: [], **opts)
123
+ Aspera.assert_type(exec, String)
124
+ Aspera.assert_type(args, Array)
125
+ Aspera.assert_type(opts, Hash)
126
+ Log.log.debug {log_spawn(env: {}, exec: exec, args: args)}
127
+ stdout, stderr, status = Open3.capture3(exec, *args, **opts)
128
+ Log.log.debug{"status=#{status}, stderr=#{stderr}"}
129
+ Log.log.trace1{"stdout=#{stdout}"}
130
+ raise "process failed: #{status.exitstatus} : #{stderr}" unless status.success?
131
+ return stdout
132
+ end
133
+
117
134
  # Write content to a file, with restricted access
118
135
  # @param path [String] the file path
119
136
  # @param force [Boolean] if true, overwrite the file
@@ -192,14 +209,13 @@ module Aspera
192
209
  @terminal_supports_unicode = nil
193
210
  end
194
211
 
195
- # @return true if we can display Unicode characters
196
- # https://www.gnu.org/software/libc/manual/html_node/Locale-Categories.html
197
- # https://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html
198
- def terminal_supports_unicode?
199
- @terminal_supports_unicode = self.class.terminal? && %w(LC_ALL LC_CTYPE LANG).any?{|var|ENV[var]&.include?('UTF-8')} if @terminal_supports_unicode.nil?
200
- return @terminal_supports_unicode
201
- end
202
-
212
+ # @return true if we can display Unicode characters
213
+ # https://www.gnu.org/software/libc/manual/html_node/Locale-Categories.html
214
+ # https://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html
215
+ def terminal_supports_unicode?
216
+ @terminal_supports_unicode = self.class.terminal? && %w(LC_ALL LC_CTYPE LANG).any?{|var|ENV[var]&.include?('UTF-8')} if @terminal_supports_unicode.nil?
217
+ return @terminal_supports_unicode
218
+ end
203
219
 
204
220
  # Allows a user to open a Url
205
221
  # if method is "text", then URL is displayed on terminal
@@ -44,7 +44,7 @@ module Aspera
44
44
  'note' => faspex_pkg_delivery['note'],
45
45
  'recipients' => faspex_pkg_delivery['recipients'].map{|name|{'name'=>name}}
46
46
  }
47
- package = @app_api.create('packages', package_data)[:data]
47
+ package = @app_api.create('packages', package_data)
48
48
  # TODO: option to send from remote source or httpgw
49
49
  transfer_spec = @app_api.call(
50
50
  operation: 'POST',
@@ -85,7 +85,7 @@ module Aspera
85
85
  rescue => e
86
86
  response.status = 500
87
87
  response['Content-Type'] = 'application/json'
88
- response.body = {error: e.message}.to_json
88
+ response.body = {error: e.message, stacktrace: e.backtrace}.to_json
89
89
  Log.log.error(e.message)
90
90
  Log.log.debug{e.backtrace.join("\n")}
91
91
  end
@@ -34,7 +34,7 @@ module Aspera
34
34
  method: "#{@namespace}#{method}",
35
35
  params: args,
36
36
  id: @request_id += 1
37
- })[:data]
37
+ })
38
38
  Aspera.assert_type(data, Hash){'response'}
39
39
  Aspera.assert(data['jsonrpc'] == JSON_RPC_VERSION){'bad version in response'}
40
40
  Aspera.assert(data.key?('id')){'missing id in response'}
@@ -4,7 +4,7 @@
4
4
  require 'aspera/cli/info'
5
5
  require 'aspera/log'
6
6
  require 'aspera/assert'
7
- require 'open3'
7
+ require 'aspera/environment'
8
8
 
9
9
  # enhance the gem to support other key chains
10
10
  module Aspera
@@ -48,20 +48,15 @@ module Aspera
48
48
  options[:path] = uri.path unless ['', '/'].include?(uri.path)
49
49
  options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
50
50
  end
51
- command_line = [SECURITY_UTILITY, command]
51
+ command_args = [command]
52
52
  options&.each do |k, v|
53
53
  Aspera.assert(supported.key?(k)){"unknown option: #{k}"}
54
54
  next if v.nil?
55
- command_line.push("-#{supported[k]}")
56
- command_line.push(v.shellescape) unless v.empty?
55
+ command_args.push("-#{supported[k]}")
56
+ command_args.push(v.shellescape) unless v.empty?
57
57
  end
58
- command_line.push(last_opt) unless last_opt.nil?
59
- Log.log.debug{"executing>>#{command_line.join(' ')}"}
60
- stdout, stderr, status = Open3.capture3(*command_line)
61
- Log.log.debug{"status=#{status}, stderr=#{stderr}"}
62
- Log.log.trace1{"stdout=#{stdout}"}
63
- raise "#{SECURITY_UTILITY} failed: #{status.exitstatus} : #{stderr}" unless status.success?
64
- return stdout
58
+ command_args.push(last_opt) unless last_opt.nil?
59
+ return Environment.secure_capture(exec: SECURITY_UTILITY, args: command_args)
65
60
  end
66
61
 
67
62
  def key_chains(output)
@@ -78,7 +73,7 @@ module Aspera
78
73
 
79
74
  def list(options={})
80
75
  Aspera.assert_values(options[:domain], DOMAINS, exception_class: ArgumentError){'domain'} unless options[:domain].nil?
81
- key_chains(execute('list-key_chains', options, LIST_OPTIONS))
76
+ key_chains(execute('list-keychains', options, LIST_OPTIONS))
82
77
  end
83
78
 
84
79
  def by_name(name)
data/lib/aspera/log.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/assert'
3
4
  require 'aspera/colors'
4
5
  require 'aspera/secret_hider'
5
6
  require 'logger'
@@ -73,8 +74,7 @@ module Aspera
73
74
  JSON.pretty_generate(object) rescue PP.pp(object, +'')
74
75
  when :ruby
75
76
  PP.pp(object, +'')
76
- else
77
- raise 'wrong parameter, expect ruby or json'
77
+ else error_unexpected_value(@@format){'dump format'}
78
78
  end
79
79
  "#{name.to_s.green} (#{@@format})=\n#{result}"
80
80
  end
@@ -127,8 +127,7 @@ module Aspera
127
127
  Syslog::Logger.make_methods(severity.downcase)
128
128
  end
129
129
  @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
130
- else
131
- raise "unknown log type: #{new_log_type}, use one of: #{LOG_TYPES.join(', ')}"
130
+ else error_unexpected_value(new_log_type){"log type (#{LOG_TYPES.join(', ')})"}
132
131
  end
133
132
  @logger.level = current_severity_integer
134
133
  @logger_type = new_log_type