aspera-cli 4.24.2 → 4.25.0.pre2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +1067 -758
  4. data/CONTRIBUTING.md +93 -120
  5. data/README.md +817 -510
  6. data/lib/aspera/agent/direct.rb +14 -12
  7. data/lib/aspera/agent/transferd.rb +4 -4
  8. data/lib/aspera/api/aoc.rb +71 -43
  9. data/lib/aspera/api/cos_node.rb +3 -2
  10. data/lib/aspera/api/faspex.rb +6 -5
  11. data/lib/aspera/api/node.rb +10 -12
  12. data/lib/aspera/ascmd.rb +1 -2
  13. data/lib/aspera/ascp/installation.rb +55 -41
  14. data/lib/aspera/ascp/management.rb +9 -5
  15. data/lib/aspera/assert.rb +28 -6
  16. data/lib/aspera/cli/error.rb +4 -2
  17. data/lib/aspera/cli/extended_value.rb +94 -62
  18. data/lib/aspera/cli/formatter.rb +55 -22
  19. data/lib/aspera/cli/main.rb +21 -14
  20. data/lib/aspera/cli/manager.rb +349 -248
  21. data/lib/aspera/cli/plugins/alee.rb +3 -3
  22. data/lib/aspera/cli/plugins/aoc.rb +94 -51
  23. data/lib/aspera/cli/plugins/base.rb +62 -49
  24. data/lib/aspera/cli/plugins/config.rb +85 -96
  25. data/lib/aspera/cli/plugins/console.rb +15 -9
  26. data/lib/aspera/cli/plugins/cos.rb +1 -1
  27. data/lib/aspera/cli/plugins/faspex.rb +34 -27
  28. data/lib/aspera/cli/plugins/faspex5.rb +47 -44
  29. data/lib/aspera/cli/plugins/faspio.rb +7 -6
  30. data/lib/aspera/cli/plugins/httpgw.rb +3 -2
  31. data/lib/aspera/cli/plugins/node.rb +132 -120
  32. data/lib/aspera/cli/plugins/oauth.rb +1 -1
  33. data/lib/aspera/cli/plugins/orchestrator.rb +116 -33
  34. data/lib/aspera/cli/plugins/preview.rb +26 -46
  35. data/lib/aspera/cli/plugins/server.rb +9 -10
  36. data/lib/aspera/cli/plugins/shares.rb +77 -43
  37. data/lib/aspera/cli/sync_actions.rb +49 -38
  38. data/lib/aspera/cli/transfer_agent.rb +16 -34
  39. data/lib/aspera/cli/version.rb +1 -1
  40. data/lib/aspera/cli/wizard.rb +8 -5
  41. data/lib/aspera/command_line_builder.rb +20 -17
  42. data/lib/aspera/coverage.rb +6 -2
  43. data/lib/aspera/environment.rb +71 -84
  44. data/lib/aspera/faspex_gw.rb +1 -1
  45. data/lib/aspera/faspex_postproc.rb +1 -1
  46. data/lib/aspera/keychain/factory.rb +1 -2
  47. data/lib/aspera/keychain/macos_security.rb +2 -2
  48. data/lib/aspera/log.rb +2 -1
  49. data/lib/aspera/markdown.rb +31 -0
  50. data/lib/aspera/nagios.rb +6 -5
  51. data/lib/aspera/oauth/base.rb +17 -27
  52. data/lib/aspera/oauth/factory.rb +1 -1
  53. data/lib/aspera/oauth/url_json.rb +2 -1
  54. data/lib/aspera/preview/file_types.rb +23 -37
  55. data/lib/aspera/preview/terminal.rb +95 -29
  56. data/lib/aspera/preview/utils.rb +6 -5
  57. data/lib/aspera/products/connect.rb +3 -3
  58. data/lib/aspera/rest.rb +51 -39
  59. data/lib/aspera/rest_error_analyzer.rb +4 -4
  60. data/lib/aspera/ssh.rb +5 -2
  61. data/lib/aspera/ssl.rb +41 -0
  62. data/lib/aspera/sync/conf.schema.yaml +182 -34
  63. data/lib/aspera/sync/database.rb +2 -1
  64. data/lib/aspera/sync/operations.rb +128 -72
  65. data/lib/aspera/transfer/parameters.rb +3 -4
  66. data/lib/aspera/transfer/spec.rb +2 -3
  67. data/lib/aspera/transfer/spec.schema.yaml +49 -19
  68. data/lib/aspera/transfer/spec_doc.rb +14 -14
  69. data/lib/aspera/uri_reader.rb +1 -1
  70. data/lib/transferd_pb.rb +2 -2
  71. data.tar.gz.sig +0 -0
  72. metadata +33 -6
  73. metadata.gz.sig +0 -0
@@ -10,11 +10,13 @@ require 'xmlsimple'
10
10
  module Aspera
11
11
  module Cli
12
12
  module Plugins
13
+ # Aspera Orchestrator
13
14
  class Orchestrator < BasicAuth
14
15
  STANDARD_PATH = '/aspera/orchestrator'
15
16
  TEST_ENDPOINT = 'api/remote_node_ping'
16
17
  private_constant :STANDARD_PATH, :TEST_ENDPOINT
17
18
  class << self
19
+ # @return [Hash,NilClass]
18
20
  def detect(address_or_url)
19
21
  address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
20
22
  urls = [address_or_url]
@@ -23,11 +25,11 @@ module Aspera
23
25
  urls.each do |base_url|
24
26
  next unless base_url.match?('https?://')
25
27
  api = Rest.new(base_url: base_url)
26
- result = api.call(operation: 'GET', subpath: TEST_ENDPOINT, headers: {'Accept' => Rest::MIME_JSON}, query: {format: :json})
27
- next unless result[:data]['remote_orchestrator_info']
28
- url = result[:http].uri.to_s
28
+ data, http = api.read(TEST_ENDPOINT, query: {format: :json}, ret: :both)
29
+ next unless data['remote_orchestrator_info']
30
+ url = http.uri.to_s
29
31
  return {
30
- version: result[:data]['remote_orchestrator_info']['orchestrator-version'],
32
+ version: data['remote_orchestrator_info']['orchestrator-version'],
31
33
  url: url[0..url.index(TEST_ENDPOINT) - 2]
32
34
  }
33
35
  rescue StandardError => e
@@ -40,7 +42,7 @@ module Aspera
40
42
  end
41
43
 
42
44
  # @param wizard [Wizard] The wizard object
43
- # @param app_url [Wizard] The wizard object
45
+ # @param app_url [String] Tested URL
44
46
  # @return [Hash] :preset_value, :test_args
45
47
  def wizard(wizard, app_url)
46
48
  return {
@@ -57,14 +59,12 @@ module Aspera
57
59
  super
58
60
  @api_orch = nil
59
61
  options.declare(:result, "Specify result value as: 'work_step:parameter'")
60
- options.declare(:synchronous, 'Wait for completion', values: :bool, default: :no)
61
- options.declare(:ret_style, 'How return type is requested in api', values: %i[header arg ext], default: :arg)
62
- options.declare(:auth_style, 'Authentication type', values: %i[arg_pass head_basic apikey], default: :head_basic)
62
+ options.declare(:synchronous, 'Wait for completion', allowed: Allowed::TYPES_BOOLEAN, default: false)
63
+ options.declare(:ret_style, 'How return type is requested in api', allowed: %i[header arg ext], default: :arg)
64
+ options.declare(:auth_style, 'Authentication type', allowed: %i[arg_pass head_basic apikey], default: :head_basic)
63
65
  options.parse_options!
64
66
  end
65
67
 
66
- ACTIONS = %i[health info workflow plugins processes].freeze
67
-
68
68
  # Call orchestrator API, it's a bit special
69
69
  # @param endpoint [String] the endpoint to call
70
70
  # @param ret_style [Symbol] the return style, :header, :arg, :ext(extension)
@@ -74,28 +74,31 @@ module Aspera
74
74
  # @param http [Boolean] if true, returns the HttpResponse, else
75
75
  def call_ao(endpoint, ret_style: nil, format: 'json', args: nil, xml_arrays: true, http: false)
76
76
  # calls are all GET
77
- call_args = {operation: 'GET', subpath: "api/#{endpoint}"}
77
+ call_args = {operation: 'GET', subpath: "api/#{endpoint}", ret: :both, query: {}}
78
78
  ret_style = options.get_option(:ret_style, mandatory: true) if ret_style.nil?
79
- call_args[:query] = args unless args.nil?
79
+ call_args[:query].merge!(args) unless args.nil?
80
80
  unless format.nil?
81
81
  case ret_style
82
82
  when :header
83
83
  call_args[:headers] = {'Accept' => "application/#{format}"}
84
84
  when :arg
85
- call_args[:query] ||= {}
86
85
  call_args[:query][:format] = format
87
86
  when :ext
88
87
  call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
89
88
  else Aspera.error_unexpected_value(ret_style)
90
89
  end
91
90
  end
92
- result = @api_orch.call(**call_args)
93
- return result[:http] if http
94
- result = format.eql?('xml') ? XmlSimple.xml_in(result[:http].body, {'ForceArray' => xml_arrays}) : result[:data]
91
+ add_query = options.get_option(:query)
92
+ call_args[:query].merge!(add_query.symbolize_keys) unless add_query.nil?
93
+ data, resp = @api_orch.call(**call_args)
94
+ return resp if http
95
+ result = format.eql?('xml') ? XmlSimple.xml_in(resp.body, {'ForceArray' => xml_arrays}) : data
95
96
  Log.dump(:data, result)
96
97
  return result
97
98
  end
98
99
 
100
+ ACTIONS = %i[health info workflows workorders workstep plugins processes monitors].freeze
101
+
99
102
  def execute_action
100
103
  auth_params =
101
104
  case options.get_option(:auth_style, mandatory: true)
@@ -133,37 +136,33 @@ module Aspera
133
136
  rescue StandardError => e
134
137
  nagios.add_critical('node api', e.to_s)
135
138
  end
136
- return nagios.result
139
+ Main.result_object_list(nagios.status_list)
140
+ # 14. Ping the remote Instance
137
141
  when :info
138
142
  result = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
139
143
  return Main.result_single_object(result)
144
+ # 12. Orchestrator Background Process status
140
145
  when :processes
141
146
  # TODO: Bug ? API has only XML format
142
147
  result = call_ao('processes_status', format: 'xml')
143
148
  return Main.result_object_list(result['process'])
149
+ # 13. Orchestrator Monitor
150
+ when :monitors
151
+ result = call_ao('monitor_snapshot')
152
+ return Main.result_single_object(result['monitor'])
144
153
  when :plugins
145
154
  # TODO: Bug ? only json format on url
146
155
  result = call_ao('plugin_version')
147
156
  return Main.result_object_list(result['Plugin'])
148
- when :workflow
149
- command = options.get_next_command(%i[list status inputs details start export])
157
+ when :workflows
158
+ command = options.get_next_command(%i[list status inputs details start export workorders outputs])
150
159
  case command
151
- when :status
152
- wf_id = instance_identifier
153
- result = call_ao(wf_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{wf_id}")
154
- return Main.result_object_list(result['workflows']['workflow'])
160
+ # 1. List all available workflows on the system
155
161
  when :list
156
- result = call_ao('workflows_list/0')
162
+ result = call_ao('workflows_list')
157
163
  return Main.result_object_list(result['workflows']['workflow'], fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification])
158
- when :details
159
- result = call_ao("workflow_details/#{instance_identifier}")
160
- return Main.result_object_list(result['workflows']['workflow']['statuses'])
161
- when :inputs
162
- result = call_ao("workflow_inputs_spec/#{instance_identifier}")
163
- return Main.result_single_object(result['workflow_inputs_spec'])
164
- when :export
165
- result = call_ao("export_workflow/#{instance_identifier}", format: nil, http: true)
166
- return Main.result_text(result.body)
164
+ # 2.1 Initiate a workorder - Asynchronous
165
+ # 2.2 Initiate a workorder - Synchronous
167
166
  when :start
168
167
  result = {
169
168
  type: :single_object,
@@ -191,6 +190,69 @@ module Aspera
191
190
  result[:type] = :text if call_params['synchronous']
192
191
  result[:data] = call_ao("initiate/#{wf_id}", args: call_params)
193
192
  return result
193
+ # 3. Fetch input specification for a workflow
194
+ when :inputs
195
+ result = call_ao("workflow_inputs_spec/#{instance_identifier}")
196
+ return Main.result_single_object(result['workflow_inputs_spec'])
197
+ # 4. Check the running status for all workflows
198
+ # 5. Check the running status for a particular workflow
199
+ when :status
200
+ wf_id = instance_identifier
201
+ result = call_ao(wf_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{wf_id}")
202
+ return Main.result_object_list(result['workflows']['workflow'])
203
+ # 6. Check the detailed running status for a particular workflow
204
+ when :details
205
+ result = call_ao("workflow_details/#{instance_identifier}")
206
+ return Main.result_object_list(result['workflows']['workflow']['statuses'])
207
+ # 15. Fetch output specification for a particular work flow
208
+ when :outputs
209
+ result = call_ao("workflow_outputs_spec/#{instance_identifier}")
210
+ return Main.result_object_list(result['workflow_outputs_spec']['output'])
211
+ # 19.Fetch all workorders from a workflow
212
+ when :workorders
213
+ result = call_ao("work_orders_list/#{instance_identifier}")
214
+ return Main.result_object_list(result['work_orders'])
215
+ when :export
216
+ result = call_ao("export_workflow/#{instance_identifier}", format: nil, http: true)
217
+ return Main.result_text(result.body)
218
+ end
219
+ when :workorders
220
+ command = options.get_next_command(%i[status cancel reset output])
221
+ case command
222
+ # 7. Check the status for a particular work order
223
+ when :status
224
+ wo_id = instance_identifier
225
+ result = call_ao("work_order_status/#{wo_id}")
226
+ return Main.result_single_object(result['work_order'])
227
+ # 9. Cancel a Work Order
228
+ when :cancel
229
+ wo_id = instance_identifier
230
+ result = call_ao("work_order_cancel/#{wo_id}")
231
+ return Main.result_single_object(result['work_order'])
232
+ # 11. Reset a Work order
233
+ when :reset
234
+ wo_id = instance_identifier
235
+ result = call_ao("work_order_reset/#{wo_id}")
236
+ return Main.result_single_object(result['work_order'])
237
+ # 16. Fetch output of a work order
238
+ when :output
239
+ wo_id = instance_identifier
240
+ result = call_ao("work_order_output/#{wo_id}", format: 'xml')
241
+ return Main.result_object_list(result['variable'])
242
+ end
243
+ when :workstep
244
+ command = options.get_next_command(%i[status cancel])
245
+ case command
246
+ # 8. Check the status of a Step
247
+ when :status
248
+ ws_id = instance_identifier
249
+ result = call_ao("work_step_status/#{ws_id}")
250
+ return Main.result_single_object(result)
251
+ # 10. Cancel a Work Step
252
+ when :cancel
253
+ ws_id = instance_identifier
254
+ result = call_ao("work_step_cancel/#{ws_id}")
255
+ return Main.result_single_object(result)
194
256
  end
195
257
  else Aspera.error_unexpected_value(command)
196
258
  end
@@ -200,3 +262,24 @@ module Aspera
200
262
  end
201
263
  end
202
264
  end
265
+
266
+ # 17.Persist custom data
267
+ # 18.Fetch queued items from queue
268
+ # 20.List Task for a User
269
+ # 21. Fetch Task details
270
+ # 22. Submit Task
271
+ # 23. Control Process
272
+ # engine monitor worker
273
+ # 24. Lookup Queued Item
274
+ # 25. Reorder Queued Items
275
+ # 26. Bulk Reorder Queued Items
276
+ # 27. Queue Item (Add an item to a Queue)
277
+ #
278
+ # Required Input:
279
+ # Optional Input:
280
+ # 28.List all queues
281
+ # 29. Portlet Version
282
+ # 30. Plugin Version
283
+ # 31. Restart Work Order from a Step
284
+ # 32. Delete element from a Managed Queue
285
+ #
@@ -46,16 +46,15 @@ module Aspera
46
46
  :AK_MARKER_FILE,
47
47
  :LOG_LIMITER_SEC
48
48
 
49
- # option_skip_format has special accessors
50
- attr_accessor :option_previews_folder
51
- attr_accessor :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
49
+ attr_accessor :option_skip_types, :option_previews_folder, :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
52
50
 
53
51
  def initialize(**_)
54
52
  super
55
- @skip_types = []
56
- @default_transfer_spec = nil
57
- # by default generate all supported formats (clone, as altered by options)
58
- @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
53
+ @option_skip_types = []
54
+ @option_skip_folders = []
55
+ @option_previews_folder = nil
56
+ @option_overwrite = nil
57
+ @option_folder_reset_cache = nil
59
58
  # options for generation
60
59
  @gen_options = Aspera::Preview::Options.new
61
60
  # used to trigger periodic processing
@@ -64,68 +63,49 @@ module Aspera
64
63
  @filter_block = nil
65
64
  # link CLI options to gen_info attributes
66
65
  options.declare(
67
- :skip_format, 'Skip this preview format (multiple possible)', values: Aspera::Preview::Generator::PREVIEW_FORMATS,
68
- handler: {o: self, m: :option_skip_format}, default: []
66
+ :skip_format, 'Skip this preview format',
67
+ allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
69
68
  )
70
69
  options.declare(
71
70
  :folder_reset_cache, 'Force detection of generated preview by refresh cache',
72
- values: %i[no header read],
71
+ allowed: %i[no header read],
73
72
  handler: {o: self, m: :option_folder_reset_cache},
74
73
  default: :no
75
74
  )
76
- options.declare(:skip_types, 'Skip types in comma separated list', handler: {o: self, m: :option_skip_types})
75
+ options.declare(:skip_types, 'Skip generation for those types of files', handler: {o: self, m: :option_skip_types}, allowed: Allowed::TYPES_SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES)
77
76
  options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
78
- options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, default: [])
77
+ options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, allowed: Allowed::TYPES_STRING_ARRAY)
79
78
  options.declare(:base, 'Basename of output for for test')
80
79
  options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
81
80
  options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
82
- options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', values: :bool, default: false)
83
- options.declare(:overwrite, 'When to overwrite result file', values: %i[always never mtime], handler: {o: self, m: :option_overwrite}, default: :mtime)
81
+ options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', allowed: Allowed::TYPES_BOOLEAN, default: false)
82
+ options.declare(:overwrite, 'When to overwrite result file', handler: {o: self, m: :option_overwrite}, allowed: %i[always never mtime], default: :mtime)
84
83
  options.declare(
85
84
  :file_access, 'How to read and write files in repository',
86
- values: %i[local remote],
85
+ allowed: %i[local remote],
87
86
  handler: {o: self, m: :option_file_access},
88
87
  default: :local
89
88
  )
90
-
91
89
  # add other options for generator (and set default values)
92
90
  Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
93
91
  values = if opt.key?(:values)
94
92
  opt[:values]
95
93
  elsif Cli::Manager::BOOLEAN_SIMPLE.include?(opt[:default])
96
- :bool
94
+ Allowed::TYPES_BOOLEAN
97
95
  end
98
- options.declare(opt[:name], opt[:description].capitalize, values: values, handler: {o: @gen_options, m: opt[:name]}, default: opt[:default])
96
+ options.declare(opt[:name], opt[:description].capitalize, allowed: values, handler: {o: @gen_options, m: opt[:name]}, default: opt[:default])
99
97
  end
100
98
 
101
99
  options.parse_options!
102
- Aspera.assert_type(@option_skip_folders, Array){'skip_folder'}
100
+ # by default generate all supported formats (clone, as altered by options)
101
+ @preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
102
+ skip = options.get_option(:skip_format)
103
+ @preview_formats_to_generate.delete(skip) if skip
103
104
  @tmp_folder = File.join(TempFileManager.instance.global_temp, "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
104
105
  FileUtils.mkdir_p(@tmp_folder)
105
106
  Log.log.debug{"tmpdir: #{@tmp_folder}"}
106
107
  end
107
108
 
108
- def option_skip_types=(value)
109
- @skip_types = []
110
- value.split(',').each do |v|
111
- s = v.to_sym
112
- Aspera.assert_values(s, Aspera::Preview::FileTypes::CONVERSION_TYPES){'skip_types'}
113
- @skip_types.push(s)
114
- end
115
- end
116
-
117
- def option_skip_types
118
- return @skip_types.map(&:to_s).join(',')
119
- end
120
-
121
- def option_skip_format=(value)
122
- @preview_formats_to_generate.delete(value)
123
- end
124
-
125
- def option_skip_format
126
- return @preview_formats_to_generate.map(&:to_s).join(',')
127
- end
128
-
129
109
  # /files/id/files is normally cached in redis, but we can discard the cache
130
110
  # but /files/id is not cached
131
111
  def get_folder_entries(file_id, request_args = nil)
@@ -136,7 +116,7 @@ module Aspera
136
116
  subpath: "files/#{file_id}/files",
137
117
  headers: headers,
138
118
  query: request_args
139
- )[:data]
119
+ )
140
120
  end
141
121
 
142
122
  # old version based on folders
@@ -221,8 +201,8 @@ module Aspera
221
201
  'paths' => [{'source' => source_filename}],
222
202
  'tags' => {Transfer::Spec::TAG_RESERVED => {PREV_GEN_TAG => true}}
223
203
  })
224
- # force destination, need to set this in transfer agent else it gets overwritten, not do: t_spec['destination_root']=destination
225
- transfer.option_transfer_spec_deep_merge({'destination_root' => destination})
204
+ # force destination, need to set this in transfer agent else it gets overwritten, do not do: t_spec['destination_root']=destination
205
+ transfer.user_transfer_spec['destination_root'] = destination
226
206
  Main.result_transfer(transfer.start(t_spec))
227
207
  end
228
208
 
@@ -311,7 +291,7 @@ module Aspera
311
291
  next false
312
292
  end
313
293
  # shall we skip it ?
314
- next false if @skip_types.include?(gen_info[:generator].conversion_type)
294
+ next false if @option_skip_types.include?(gen_info[:generator].conversion_type)
315
295
  # ok we need to generate
316
296
  true
317
297
  end
@@ -444,7 +424,7 @@ module Aspera
444
424
  end
445
425
  Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, mandatory: true)
446
426
  # check tools that are anyway required for all cases
447
- Aspera::Preview::Utils.check_tools(@skip_types)
427
+ Aspera::Preview::Utils.check_tools(@option_skip_types)
448
428
  case command
449
429
  when :scan
450
430
  scan_path = options.get_option(:scan_path)
@@ -491,7 +471,7 @@ module Aspera
491
471
  g = Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder, nil)
492
472
  g.generate
493
473
  if command.eql?(:show)
494
- terminal_options = options.get_option(:query, default: {}).symbolize_keys
474
+ terminal_options = (options.get_option(:query) || {}).symbolize_keys
495
475
  Log.log.debug{"preview: #{generated_file_path}"}
496
476
  formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
497
477
  end
@@ -32,7 +32,7 @@ module Aspera
32
32
 
33
33
  class LocalExecutor
34
34
  def execute(ascmd_path, input:)
35
- return Environment.secure_capture(exec: ascmd_path, stdin_data: input, binmode: true, exception: false)
35
+ return Environment.secure_execute(ascmd_path, mode: :capture, stdin_data: input, binmode: true, exception: false)
36
36
  end
37
37
  end
38
38
 
@@ -41,6 +41,7 @@ module Aspera
41
41
  'HSTS Fasp/SSH'
42
42
  end
43
43
 
44
+ # @return [Hash,NilClass]
44
45
  def detect(address_or_url)
45
46
  urls = if address_or_url.match?(%r{^[a-z]{1,6}://})
46
47
  [address_or_url]
@@ -70,7 +71,7 @@ module Aspera
70
71
  end
71
72
 
72
73
  # @param wizard [Wizard] The wizard object
73
- # @param app_url [Wizard] The wizard object
74
+ # @param app_url [String] Tested URL
74
75
  # @return [Hash] :preset_value, :test_args
75
76
  def wizard(wizard, app_url)
76
77
  return {
@@ -87,9 +88,9 @@ module Aspera
87
88
  super
88
89
  @ssh_opts = {}
89
90
  @connection_type = :ssh
90
- options.declare(:ssh_keys, 'SSH key path list (Array or single)')
91
+ options.declare(:ssh_keys, 'SSH key path list', allowed: Allowed::TYPES_STRING_ARRAY)
91
92
  options.declare(:passphrase, 'SSH private key passphrase')
92
- options.declare(:ssh_options, 'SSH options', types: Hash, handler: {o: self, m: :option_ssh_opts})
93
+ options.declare(:ssh_options, 'SSH options', allowed: Hash, handler: {o: self, m: :option_ssh_opts})
93
94
  SyncActions.declare_options(options)
94
95
  options.parse_options!
95
96
  end
@@ -121,7 +122,7 @@ module Aspera
121
122
  ENV['SSH_CLIENT'] = 'local 0 0'
122
123
  @connection_type = :local
123
124
  return server_transfer_spec
124
- elsif transfer.option_transfer_spec['token'].is_a?(String) && server_uri.scheme.eql?(HTTPS_SCHEME)
125
+ elsif transfer.user_transfer_spec['token'].is_a?(String) && server_uri.scheme.eql?(HTTPS_SCHEME)
125
126
  server_transfer_spec['wss_enabled'] = true
126
127
  server_transfer_spec['wss_port'] = server_uri.port
127
128
  @connection_type = :wss
@@ -154,9 +155,7 @@ module Aspera
154
155
  end
155
156
  ssh_key_list = options.get_option(:ssh_keys)
156
157
  if !ssh_key_list.nil?
157
- ssh_key_list = [ssh_key_list] if ssh_key_list.is_a?(String)
158
- Aspera.assert_type(ssh_key_list, Array){'ssh_keys'}
159
- Aspera.assert(ssh_key_list.all?(String))
158
+ Aspera.assert_array_all(ssh_key_list, String){'ssh_keys'}
160
159
  ssh_key_list.map!{ |p| File.expand_path(p)}
161
160
  Log.log.debug{"SSH keys=#{ssh_key_list}"}
162
161
  if !ssh_key_list.empty?
@@ -173,7 +172,7 @@ module Aspera
173
172
  server_transfer_spec['ssh_private_key_passphrase'] = ssh_passphrase
174
173
  end
175
174
  # if user provided transfer spec has a token, we will use bypass keys
176
- cred_set = true if transfer.option_transfer_spec['token'].is_a?(String)
175
+ cred_set = true if transfer.user_transfer_spec['token'].is_a?(String)
177
176
  Aspera.assert(cred_set, type: BadArgument){'Either password, key , or transfer spec token must be provided'}
178
177
  return server_transfer_spec
179
178
  end
@@ -227,7 +226,7 @@ module Aspera
227
226
  end
228
227
  else Aspera.error_unexpected_value(command_nagios)
229
228
  end
230
- return nagios.result
229
+ Main.result_object_list(nagios.status_list)
231
230
  when *TRANSFER_COMMANDS
232
231
  return execute_transfer(command, server_transfer_spec)
233
232
  when *AsCmd::OPERATIONS
@@ -13,32 +13,62 @@ module Aspera
13
13
  # path for node admin after base url
14
14
  ADMIN_API_PATH = 'api/v1'
15
15
  class << self
16
+ # Check various endpoints on Shares
17
+ # @return [Hash] with version, ping, api
18
+ def health_check(url)
19
+ result = {}
20
+ result[:version] =
21
+ begin
22
+ version = nil
23
+ login_page = Rest
24
+ .new(base_url: url, redirect_max: 2)
25
+ .read('', headers: {'Accept'=>'text/html'})
26
+ if (m = login_page.match(/\(v([0-9a-f\.]+)\)/))
27
+ version = m[1]
28
+ if (m = login_page.match(/Patch level ([0-9]+)/))
29
+ version = "#{result[:version]} #{m[0]}"
30
+ end
31
+ end
32
+ raise 'no version' if version.nil?
33
+ version
34
+ rescue => e
35
+ e
36
+ end
37
+ result[:ping] =
38
+ begin
39
+ Rest
40
+ .new(base_url: "#{url}/#{NODE_API_PATH}")
41
+ .read('ping', headers: {'Content-Type'=>'application/json'})
42
+ 'ping ok'
43
+ rescue => e
44
+ e
45
+ end
46
+ result[:api] =
47
+ begin
48
+ resp = Rest.new(base_url: url, redirect_max: 1).read("#{NODE_API_PATH}/app", exception: false, ret: :resp)
49
+ # shall fail: shares requires auth, but we check error message
50
+ raise 'not found' unless resp.code.to_s.eql?('401') && resp.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
51
+ 'available'
52
+ rescue => e
53
+ e
54
+ end
55
+ result
56
+ end
57
+
58
+ # @return [Hash,NilClass]
16
59
  def detect(address_or_url)
17
60
  address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
18
- api = Rest.new(base_url: address_or_url, redirect_max: 1)
19
- found = false
20
- begin
21
- # shall fail: shares requires auth, but we check error message
22
- # TODO: use ping instead ?
23
- api.read("#{NODE_API_PATH}/app")
24
- rescue RestCallError => e
25
- found = true if e.response.code.to_s.eql?('401') && e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
26
- end
27
- return unless found
28
- version = 'unknown'
29
- test_page = api.call(operation: 'GET', subpath: 'login')
30
- if (m = test_page[:http].body.match(/\(v(1\..*)\)/))
31
- version = m[1]
32
- end
61
+ health = health_check(address_or_url)
62
+ return unless health[:api].is_a?(String)
33
63
  return {
34
- version: version,
64
+ version: health[:version].is_a?(String) ? health[:version] : 'unknown',
35
65
  url: address_or_url
36
66
  }
37
67
  end
38
68
  end
39
69
 
40
70
  # @param wizard [Wizard] The wizard object
41
- # @param app_url [Wizard] The wizard object
71
+ # @param app_url [String] Tested URL
42
72
  # @return [Hash] :preset_value, :test_args
43
73
  def wizard(wizard, app_url)
44
74
  return {
@@ -67,20 +97,20 @@ module Aspera
67
97
  case command
68
98
  when :health
69
99
  nagios = Nagios.new
70
- begin
71
- res = Rest
72
- .new(base_url: "#{options.get_option(:url, mandatory: true)}/#{NODE_API_PATH}")
73
- .call(
74
- operation: 'GET',
75
- subpath: 'ping',
76
- headers: {'content-type': Rest::MIME_JSON}
77
- )
78
- raise Error, 'Shares not detected' unless res[:http].body.eql?(' ')
79
- nagios.add_ok('shares api', 'accessible')
80
- rescue StandardError => e
81
- nagios.add_critical('API', e.to_s)
100
+ shares_url = options.get_option(:url, mandatory: true)
101
+ health = self.class.health_check(shares_url)
102
+ nagios.add_ok('version', health[:version]) if health[:version].is_a?(String)
103
+ if health[:ping].is_a?(String)
104
+ nagios.add_ok('ping', health[:ping])
105
+ else
106
+ nagios.add_critical('ping', health[:ping].to_s)
107
+ end
108
+ if health[:api].is_a?(String)
109
+ nagios.add_ok('API', health[:api])
110
+ else
111
+ nagios.add_critical('API', health[:api].to_s)
82
112
  end
83
- return nagios.result
113
+ Main.result_object_list(nagios.status_list)
84
114
  when :files
85
115
  api_shares_node = basic_auth_api(NODE_API_PATH)
86
116
  repo_command = options.get_next_command(Node::COMMANDS_SHARES)
@@ -90,6 +120,7 @@ module Aspera
90
120
  when :admin
91
121
  api_shares_admin = basic_auth_api(ADMIN_API_PATH)
92
122
  admin_command = options.get_next_command(%i[node share transfer_settings user group].freeze)
123
+ lookup_share = ->(field, value){lookup_entity_generic(entity: 'share', field: field, value: value){api_shares_admin.read('data/shares')}['id']}
93
124
  case admin_command
94
125
  when :node
95
126
  return entity_execute(api: api_shares_admin, entity: 'data/nodes')
@@ -98,13 +129,14 @@ module Aspera
98
129
  case share_command
99
130
  when *ALL_OPS
100
131
  return entity_execute(
101
- api: api_shares_admin,
102
- entity: 'data/shares',
103
- command: share_command,
104
- display_fields: %w[id name node_id directory percent_free]
132
+ api: api_shares_admin,
133
+ entity: 'data/shares',
134
+ command: share_command,
135
+ display_fields: %w[id name node_id directory percent_free],
136
+ &lookup_share
105
137
  )
106
138
  when :user_permissions, :group_permissions
107
- share_id = instance_identifier
139
+ share_id = instance_identifier(&lookup_share)
108
140
  return entity_execute(api: api_shares_admin, entity: "data/shares/#{share_id}/#{share_command}")
109
141
  end
110
142
  when :transfer_settings
@@ -137,20 +169,22 @@ module Aspera
137
169
  entity_commands = %i[import].freeze
138
170
  end
139
171
  entity_verb = options.get_next_command(entity_commands)
172
+ lookup_block = ->(field, value){lookup_entity_generic(entity: entity_type, field: field, value: value){api_shares_admin.read(entities_path)}['id']}
140
173
  case entity_verb
141
174
  when *ALL_OPS # list, show, delete, create, modify
142
- display_fields = entity_type.eql?(:user) ? %w[id username first_name last_name email] : nil
175
+ display_fields = entity_type.eql?(:user) ? %w[id user_id username first_name last_name email] : nil
143
176
  display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:all)
144
177
  return entity_execute(
145
- api: api_shares_admin,
146
- entity: entities_path,
147
- command: entity_verb,
148
- display_fields: display_fields
178
+ api: api_shares_admin,
179
+ entity: entities_path,
180
+ command: entity_verb,
181
+ display_fields: display_fields,
182
+ &lookup_block
149
183
  )
150
184
  when *USR_GRP_SETTINGS # transfer_settings, app_authorizations, share_permissions
151
- group_id = instance_identifier
185
+ group_id = instance_identifier(&lookup_block)
152
186
  entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
153
- return entity_execute(api: api_shares_admin, entity: entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
187
+ return entity_execute(api: api_shares_admin, entity: entities_path, is_singleton: !entity_verb.eql?(:share_permissions), &lookup_share)
154
188
  when :import # saml
155
189
  return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
156
190
  entity_parameters = entity_parameters.transform_keys{ |k| k.gsub(/\s+/, '_').downcase}
@@ -166,7 +200,7 @@ module Aspera
166
200
  api_shares_admin.create(entities_path, {entity_type=>entity_name})
167
201
  end
168
202
  when :users # group
169
- return entity_execute(api: api_shares_admin, entity: "#{entities_path}/#{instance_identifier}/#{entities_prefix}users")
203
+ return entity_execute(api: api_shares_admin, entity: "#{entities_path}/#{instance_identifier(&lookup_block)}/#{entities_prefix}users")
170
204
  else Aspera.error_unexpected_value(entity_verb)
171
205
  end
172
206
  end