aspera-cli 4.24.2 → 4.25.0.pre2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1067 -758
- data/CONTRIBUTING.md +93 -120
- data/README.md +817 -510
- data/lib/aspera/agent/direct.rb +14 -12
- data/lib/aspera/agent/transferd.rb +4 -4
- data/lib/aspera/api/aoc.rb +71 -43
- data/lib/aspera/api/cos_node.rb +3 -2
- data/lib/aspera/api/faspex.rb +6 -5
- data/lib/aspera/api/node.rb +10 -12
- data/lib/aspera/ascmd.rb +1 -2
- data/lib/aspera/ascp/installation.rb +55 -41
- data/lib/aspera/ascp/management.rb +9 -5
- data/lib/aspera/assert.rb +28 -6
- data/lib/aspera/cli/error.rb +4 -2
- data/lib/aspera/cli/extended_value.rb +94 -62
- data/lib/aspera/cli/formatter.rb +55 -22
- data/lib/aspera/cli/main.rb +21 -14
- data/lib/aspera/cli/manager.rb +349 -248
- data/lib/aspera/cli/plugins/alee.rb +3 -3
- data/lib/aspera/cli/plugins/aoc.rb +94 -51
- data/lib/aspera/cli/plugins/base.rb +62 -49
- data/lib/aspera/cli/plugins/config.rb +85 -96
- data/lib/aspera/cli/plugins/console.rb +15 -9
- data/lib/aspera/cli/plugins/cos.rb +1 -1
- data/lib/aspera/cli/plugins/faspex.rb +34 -27
- data/lib/aspera/cli/plugins/faspex5.rb +47 -44
- data/lib/aspera/cli/plugins/faspio.rb +7 -6
- data/lib/aspera/cli/plugins/httpgw.rb +3 -2
- data/lib/aspera/cli/plugins/node.rb +132 -120
- data/lib/aspera/cli/plugins/oauth.rb +1 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +116 -33
- data/lib/aspera/cli/plugins/preview.rb +26 -46
- data/lib/aspera/cli/plugins/server.rb +9 -10
- data/lib/aspera/cli/plugins/shares.rb +77 -43
- data/lib/aspera/cli/sync_actions.rb +49 -38
- data/lib/aspera/cli/transfer_agent.rb +16 -34
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +8 -5
- data/lib/aspera/command_line_builder.rb +20 -17
- data/lib/aspera/coverage.rb +6 -2
- data/lib/aspera/environment.rb +71 -84
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/keychain/factory.rb +1 -2
- data/lib/aspera/keychain/macos_security.rb +2 -2
- data/lib/aspera/log.rb +2 -1
- data/lib/aspera/markdown.rb +31 -0
- data/lib/aspera/nagios.rb +6 -5
- data/lib/aspera/oauth/base.rb +17 -27
- data/lib/aspera/oauth/factory.rb +1 -1
- data/lib/aspera/oauth/url_json.rb +2 -1
- data/lib/aspera/preview/file_types.rb +23 -37
- data/lib/aspera/preview/terminal.rb +95 -29
- data/lib/aspera/preview/utils.rb +6 -5
- data/lib/aspera/products/connect.rb +3 -3
- data/lib/aspera/rest.rb +51 -39
- data/lib/aspera/rest_error_analyzer.rb +4 -4
- data/lib/aspera/ssh.rb +5 -2
- data/lib/aspera/ssl.rb +41 -0
- data/lib/aspera/sync/conf.schema.yaml +182 -34
- data/lib/aspera/sync/database.rb +2 -1
- data/lib/aspera/sync/operations.rb +128 -72
- data/lib/aspera/transfer/parameters.rb +3 -4
- data/lib/aspera/transfer/spec.rb +2 -3
- data/lib/aspera/transfer/spec.schema.yaml +49 -19
- data/lib/aspera/transfer/spec_doc.rb +14 -14
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/transferd_pb.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +33 -6
- metadata.gz.sig +0 -0
|
@@ -10,11 +10,13 @@ require 'xmlsimple'
|
|
|
10
10
|
module Aspera
|
|
11
11
|
module Cli
|
|
12
12
|
module Plugins
|
|
13
|
+
# Aspera Orchestrator
|
|
13
14
|
class Orchestrator < BasicAuth
|
|
14
15
|
STANDARD_PATH = '/aspera/orchestrator'
|
|
15
16
|
TEST_ENDPOINT = 'api/remote_node_ping'
|
|
16
17
|
private_constant :STANDARD_PATH, :TEST_ENDPOINT
|
|
17
18
|
class << self
|
|
19
|
+
# @return [Hash,NilClass]
|
|
18
20
|
def detect(address_or_url)
|
|
19
21
|
address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
|
|
20
22
|
urls = [address_or_url]
|
|
@@ -23,11 +25,11 @@ module Aspera
|
|
|
23
25
|
urls.each do |base_url|
|
|
24
26
|
next unless base_url.match?('https?://')
|
|
25
27
|
api = Rest.new(base_url: base_url)
|
|
26
|
-
|
|
27
|
-
next unless
|
|
28
|
-
url =
|
|
28
|
+
data, http = api.read(TEST_ENDPOINT, query: {format: :json}, ret: :both)
|
|
29
|
+
next unless data['remote_orchestrator_info']
|
|
30
|
+
url = http.uri.to_s
|
|
29
31
|
return {
|
|
30
|
-
version:
|
|
32
|
+
version: data['remote_orchestrator_info']['orchestrator-version'],
|
|
31
33
|
url: url[0..url.index(TEST_ENDPOINT) - 2]
|
|
32
34
|
}
|
|
33
35
|
rescue StandardError => e
|
|
@@ -40,7 +42,7 @@ module Aspera
|
|
|
40
42
|
end
|
|
41
43
|
|
|
42
44
|
# @param wizard [Wizard] The wizard object
|
|
43
|
-
# @param app_url [
|
|
45
|
+
# @param app_url [String] Tested URL
|
|
44
46
|
# @return [Hash] :preset_value, :test_args
|
|
45
47
|
def wizard(wizard, app_url)
|
|
46
48
|
return {
|
|
@@ -57,14 +59,12 @@ module Aspera
|
|
|
57
59
|
super
|
|
58
60
|
@api_orch = nil
|
|
59
61
|
options.declare(:result, "Specify result value as: 'work_step:parameter'")
|
|
60
|
-
options.declare(:synchronous, 'Wait for completion',
|
|
61
|
-
options.declare(:ret_style, 'How return type is requested in api',
|
|
62
|
-
options.declare(:auth_style, 'Authentication type',
|
|
62
|
+
options.declare(:synchronous, 'Wait for completion', allowed: Allowed::TYPES_BOOLEAN, default: false)
|
|
63
|
+
options.declare(:ret_style, 'How return type is requested in api', allowed: %i[header arg ext], default: :arg)
|
|
64
|
+
options.declare(:auth_style, 'Authentication type', allowed: %i[arg_pass head_basic apikey], default: :head_basic)
|
|
63
65
|
options.parse_options!
|
|
64
66
|
end
|
|
65
67
|
|
|
66
|
-
ACTIONS = %i[health info workflow plugins processes].freeze
|
|
67
|
-
|
|
68
68
|
# Call orchestrator API, it's a bit special
|
|
69
69
|
# @param endpoint [String] the endpoint to call
|
|
70
70
|
# @param ret_style [Symbol] the return style, :header, :arg, :ext(extension)
|
|
@@ -74,28 +74,31 @@ module Aspera
|
|
|
74
74
|
# @param http [Boolean] if true, returns the HttpResponse, else
|
|
75
75
|
def call_ao(endpoint, ret_style: nil, format: 'json', args: nil, xml_arrays: true, http: false)
|
|
76
76
|
# calls are all GET
|
|
77
|
-
call_args = {operation: 'GET', subpath: "api/#{endpoint}"}
|
|
77
|
+
call_args = {operation: 'GET', subpath: "api/#{endpoint}", ret: :both, query: {}}
|
|
78
78
|
ret_style = options.get_option(:ret_style, mandatory: true) if ret_style.nil?
|
|
79
|
-
call_args[:query]
|
|
79
|
+
call_args[:query].merge!(args) unless args.nil?
|
|
80
80
|
unless format.nil?
|
|
81
81
|
case ret_style
|
|
82
82
|
when :header
|
|
83
83
|
call_args[:headers] = {'Accept' => "application/#{format}"}
|
|
84
84
|
when :arg
|
|
85
|
-
call_args[:query] ||= {}
|
|
86
85
|
call_args[:query][:format] = format
|
|
87
86
|
when :ext
|
|
88
87
|
call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
|
|
89
88
|
else Aspera.error_unexpected_value(ret_style)
|
|
90
89
|
end
|
|
91
90
|
end
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
91
|
+
add_query = options.get_option(:query)
|
|
92
|
+
call_args[:query].merge!(add_query.symbolize_keys) unless add_query.nil?
|
|
93
|
+
data, resp = @api_orch.call(**call_args)
|
|
94
|
+
return resp if http
|
|
95
|
+
result = format.eql?('xml') ? XmlSimple.xml_in(resp.body, {'ForceArray' => xml_arrays}) : data
|
|
95
96
|
Log.dump(:data, result)
|
|
96
97
|
return result
|
|
97
98
|
end
|
|
98
99
|
|
|
100
|
+
ACTIONS = %i[health info workflows workorders workstep plugins processes monitors].freeze
|
|
101
|
+
|
|
99
102
|
def execute_action
|
|
100
103
|
auth_params =
|
|
101
104
|
case options.get_option(:auth_style, mandatory: true)
|
|
@@ -133,37 +136,33 @@ module Aspera
|
|
|
133
136
|
rescue StandardError => e
|
|
134
137
|
nagios.add_critical('node api', e.to_s)
|
|
135
138
|
end
|
|
136
|
-
|
|
139
|
+
Main.result_object_list(nagios.status_list)
|
|
140
|
+
# 14. Ping the remote Instance
|
|
137
141
|
when :info
|
|
138
142
|
result = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
|
|
139
143
|
return Main.result_single_object(result)
|
|
144
|
+
# 12. Orchestrator Background Process status
|
|
140
145
|
when :processes
|
|
141
146
|
# TODO: Bug ? API has only XML format
|
|
142
147
|
result = call_ao('processes_status', format: 'xml')
|
|
143
148
|
return Main.result_object_list(result['process'])
|
|
149
|
+
# 13. Orchestrator Monitor
|
|
150
|
+
when :monitors
|
|
151
|
+
result = call_ao('monitor_snapshot')
|
|
152
|
+
return Main.result_single_object(result['monitor'])
|
|
144
153
|
when :plugins
|
|
145
154
|
# TODO: Bug ? only json format on url
|
|
146
155
|
result = call_ao('plugin_version')
|
|
147
156
|
return Main.result_object_list(result['Plugin'])
|
|
148
|
-
when :
|
|
149
|
-
command = options.get_next_command(%i[list status inputs details start export])
|
|
157
|
+
when :workflows
|
|
158
|
+
command = options.get_next_command(%i[list status inputs details start export workorders outputs])
|
|
150
159
|
case command
|
|
151
|
-
|
|
152
|
-
wf_id = instance_identifier
|
|
153
|
-
result = call_ao(wf_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{wf_id}")
|
|
154
|
-
return Main.result_object_list(result['workflows']['workflow'])
|
|
160
|
+
# 1. List all available workflows on the system
|
|
155
161
|
when :list
|
|
156
|
-
result = call_ao('workflows_list
|
|
162
|
+
result = call_ao('workflows_list')
|
|
157
163
|
return Main.result_object_list(result['workflows']['workflow'], fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification])
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
return Main.result_object_list(result['workflows']['workflow']['statuses'])
|
|
161
|
-
when :inputs
|
|
162
|
-
result = call_ao("workflow_inputs_spec/#{instance_identifier}")
|
|
163
|
-
return Main.result_single_object(result['workflow_inputs_spec'])
|
|
164
|
-
when :export
|
|
165
|
-
result = call_ao("export_workflow/#{instance_identifier}", format: nil, http: true)
|
|
166
|
-
return Main.result_text(result.body)
|
|
164
|
+
# 2.1 Initiate a workorder - Asynchronous
|
|
165
|
+
# 2.2 Initiate a workorder - Synchronous
|
|
167
166
|
when :start
|
|
168
167
|
result = {
|
|
169
168
|
type: :single_object,
|
|
@@ -191,6 +190,69 @@ module Aspera
|
|
|
191
190
|
result[:type] = :text if call_params['synchronous']
|
|
192
191
|
result[:data] = call_ao("initiate/#{wf_id}", args: call_params)
|
|
193
192
|
return result
|
|
193
|
+
# 3. Fetch input specification for a workflow
|
|
194
|
+
when :inputs
|
|
195
|
+
result = call_ao("workflow_inputs_spec/#{instance_identifier}")
|
|
196
|
+
return Main.result_single_object(result['workflow_inputs_spec'])
|
|
197
|
+
# 4. Check the running status for all workflows
|
|
198
|
+
# 5. Check the running status for a particular workflow
|
|
199
|
+
when :status
|
|
200
|
+
wf_id = instance_identifier
|
|
201
|
+
result = call_ao(wf_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{wf_id}")
|
|
202
|
+
return Main.result_object_list(result['workflows']['workflow'])
|
|
203
|
+
# 6. Check the detailed running status for a particular workflow
|
|
204
|
+
when :details
|
|
205
|
+
result = call_ao("workflow_details/#{instance_identifier}")
|
|
206
|
+
return Main.result_object_list(result['workflows']['workflow']['statuses'])
|
|
207
|
+
# 15. Fetch output specification for a particular work flow
|
|
208
|
+
when :outputs
|
|
209
|
+
result = call_ao("workflow_outputs_spec/#{instance_identifier}")
|
|
210
|
+
return Main.result_object_list(result['workflow_outputs_spec']['output'])
|
|
211
|
+
# 19.Fetch all workorders from a workflow
|
|
212
|
+
when :workorders
|
|
213
|
+
result = call_ao("work_orders_list/#{instance_identifier}")
|
|
214
|
+
return Main.result_object_list(result['work_orders'])
|
|
215
|
+
when :export
|
|
216
|
+
result = call_ao("export_workflow/#{instance_identifier}", format: nil, http: true)
|
|
217
|
+
return Main.result_text(result.body)
|
|
218
|
+
end
|
|
219
|
+
when :workorders
|
|
220
|
+
command = options.get_next_command(%i[status cancel reset output])
|
|
221
|
+
case command
|
|
222
|
+
# 7. Check the status for a particular work order
|
|
223
|
+
when :status
|
|
224
|
+
wo_id = instance_identifier
|
|
225
|
+
result = call_ao("work_order_status/#{wo_id}")
|
|
226
|
+
return Main.result_single_object(result['work_order'])
|
|
227
|
+
# 9. Cancel a Work Order
|
|
228
|
+
when :cancel
|
|
229
|
+
wo_id = instance_identifier
|
|
230
|
+
result = call_ao("work_order_cancel/#{wo_id}")
|
|
231
|
+
return Main.result_single_object(result['work_order'])
|
|
232
|
+
# 11. Reset a Work order
|
|
233
|
+
when :reset
|
|
234
|
+
wo_id = instance_identifier
|
|
235
|
+
result = call_ao("work_order_reset/#{wo_id}")
|
|
236
|
+
return Main.result_single_object(result['work_order'])
|
|
237
|
+
# 16. Fetch output of a work order
|
|
238
|
+
when :output
|
|
239
|
+
wo_id = instance_identifier
|
|
240
|
+
result = call_ao("work_order_output/#{wo_id}", format: 'xml')
|
|
241
|
+
return Main.result_object_list(result['variable'])
|
|
242
|
+
end
|
|
243
|
+
when :workstep
|
|
244
|
+
command = options.get_next_command(%i[status cancel])
|
|
245
|
+
case command
|
|
246
|
+
# 8. Check the status of a Step
|
|
247
|
+
when :status
|
|
248
|
+
ws_id = instance_identifier
|
|
249
|
+
result = call_ao("work_step_status/#{ws_id}")
|
|
250
|
+
return Main.result_single_object(result)
|
|
251
|
+
# 10. Cancel a Work Step
|
|
252
|
+
when :cancel
|
|
253
|
+
ws_id = instance_identifier
|
|
254
|
+
result = call_ao("work_step_cancel/#{ws_id}")
|
|
255
|
+
return Main.result_single_object(result)
|
|
194
256
|
end
|
|
195
257
|
else Aspera.error_unexpected_value(command)
|
|
196
258
|
end
|
|
@@ -200,3 +262,24 @@ module Aspera
|
|
|
200
262
|
end
|
|
201
263
|
end
|
|
202
264
|
end
|
|
265
|
+
|
|
266
|
+
# 17.Persist custom data
|
|
267
|
+
# 18.Fetch queued items from queue
|
|
268
|
+
# 20.List Task for a User
|
|
269
|
+
# 21. Fetch Task details
|
|
270
|
+
# 22. Submit Task
|
|
271
|
+
# 23. Control Process
|
|
272
|
+
# engine monitor worker
|
|
273
|
+
# 24. Lookup Queued Item
|
|
274
|
+
# 25. Reorder Queued Items
|
|
275
|
+
# 26. Bulk Reorder Queued Items
|
|
276
|
+
# 27. Queue Item (Add an item to a Queue)
|
|
277
|
+
#
|
|
278
|
+
# Required Input:
|
|
279
|
+
# Optional Input:
|
|
280
|
+
# 28.List all queues
|
|
281
|
+
# 29. Portlet Version
|
|
282
|
+
# 30. Plugin Version
|
|
283
|
+
# 31. Restart Work Order from a Step
|
|
284
|
+
# 32. Delete element from a Managed Queue
|
|
285
|
+
#
|
|
@@ -46,16 +46,15 @@ module Aspera
|
|
|
46
46
|
:AK_MARKER_FILE,
|
|
47
47
|
:LOG_LIMITER_SEC
|
|
48
48
|
|
|
49
|
-
|
|
50
|
-
attr_accessor :option_previews_folder
|
|
51
|
-
attr_accessor :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
|
|
49
|
+
attr_accessor :option_skip_types, :option_previews_folder, :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
|
|
52
50
|
|
|
53
51
|
def initialize(**_)
|
|
54
52
|
super
|
|
55
|
-
@
|
|
56
|
-
@
|
|
57
|
-
|
|
58
|
-
@
|
|
53
|
+
@option_skip_types = []
|
|
54
|
+
@option_skip_folders = []
|
|
55
|
+
@option_previews_folder = nil
|
|
56
|
+
@option_overwrite = nil
|
|
57
|
+
@option_folder_reset_cache = nil
|
|
59
58
|
# options for generation
|
|
60
59
|
@gen_options = Aspera::Preview::Options.new
|
|
61
60
|
# used to trigger periodic processing
|
|
@@ -64,68 +63,49 @@ module Aspera
|
|
|
64
63
|
@filter_block = nil
|
|
65
64
|
# link CLI options to gen_info attributes
|
|
66
65
|
options.declare(
|
|
67
|
-
:skip_format, 'Skip this preview format
|
|
68
|
-
|
|
66
|
+
:skip_format, 'Skip this preview format',
|
|
67
|
+
allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
|
|
69
68
|
)
|
|
70
69
|
options.declare(
|
|
71
70
|
:folder_reset_cache, 'Force detection of generated preview by refresh cache',
|
|
72
|
-
|
|
71
|
+
allowed: %i[no header read],
|
|
73
72
|
handler: {o: self, m: :option_folder_reset_cache},
|
|
74
73
|
default: :no
|
|
75
74
|
)
|
|
76
|
-
options.declare(:skip_types, 'Skip
|
|
75
|
+
options.declare(:skip_types, 'Skip generation for those types of files', handler: {o: self, m: :option_skip_types}, allowed: Allowed::TYPES_SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES)
|
|
77
76
|
options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
|
|
78
|
-
options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders},
|
|
77
|
+
options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, allowed: Allowed::TYPES_STRING_ARRAY)
|
|
79
78
|
options.declare(:base, 'Basename of output for for test')
|
|
80
79
|
options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
|
|
81
80
|
options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
|
|
82
|
-
options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic',
|
|
83
|
-
options.declare(:overwrite, 'When to overwrite result file',
|
|
81
|
+
options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', allowed: Allowed::TYPES_BOOLEAN, default: false)
|
|
82
|
+
options.declare(:overwrite, 'When to overwrite result file', handler: {o: self, m: :option_overwrite}, allowed: %i[always never mtime], default: :mtime)
|
|
84
83
|
options.declare(
|
|
85
84
|
:file_access, 'How to read and write files in repository',
|
|
86
|
-
|
|
85
|
+
allowed: %i[local remote],
|
|
87
86
|
handler: {o: self, m: :option_file_access},
|
|
88
87
|
default: :local
|
|
89
88
|
)
|
|
90
|
-
|
|
91
89
|
# add other options for generator (and set default values)
|
|
92
90
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
|
93
91
|
values = if opt.key?(:values)
|
|
94
92
|
opt[:values]
|
|
95
93
|
elsif Cli::Manager::BOOLEAN_SIMPLE.include?(opt[:default])
|
|
96
|
-
|
|
94
|
+
Allowed::TYPES_BOOLEAN
|
|
97
95
|
end
|
|
98
|
-
options.declare(opt[:name], opt[:description].capitalize,
|
|
96
|
+
options.declare(opt[:name], opt[:description].capitalize, allowed: values, handler: {o: @gen_options, m: opt[:name]}, default: opt[:default])
|
|
99
97
|
end
|
|
100
98
|
|
|
101
99
|
options.parse_options!
|
|
102
|
-
|
|
100
|
+
# by default generate all supported formats (clone, as altered by options)
|
|
101
|
+
@preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
|
|
102
|
+
skip = options.get_option(:skip_format)
|
|
103
|
+
@preview_formats_to_generate.delete(skip) if skip
|
|
103
104
|
@tmp_folder = File.join(TempFileManager.instance.global_temp, "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
|
|
104
105
|
FileUtils.mkdir_p(@tmp_folder)
|
|
105
106
|
Log.log.debug{"tmpdir: #{@tmp_folder}"}
|
|
106
107
|
end
|
|
107
108
|
|
|
108
|
-
def option_skip_types=(value)
|
|
109
|
-
@skip_types = []
|
|
110
|
-
value.split(',').each do |v|
|
|
111
|
-
s = v.to_sym
|
|
112
|
-
Aspera.assert_values(s, Aspera::Preview::FileTypes::CONVERSION_TYPES){'skip_types'}
|
|
113
|
-
@skip_types.push(s)
|
|
114
|
-
end
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
def option_skip_types
|
|
118
|
-
return @skip_types.map(&:to_s).join(',')
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
def option_skip_format=(value)
|
|
122
|
-
@preview_formats_to_generate.delete(value)
|
|
123
|
-
end
|
|
124
|
-
|
|
125
|
-
def option_skip_format
|
|
126
|
-
return @preview_formats_to_generate.map(&:to_s).join(',')
|
|
127
|
-
end
|
|
128
|
-
|
|
129
109
|
# /files/id/files is normally cached in redis, but we can discard the cache
|
|
130
110
|
# but /files/id is not cached
|
|
131
111
|
def get_folder_entries(file_id, request_args = nil)
|
|
@@ -136,7 +116,7 @@ module Aspera
|
|
|
136
116
|
subpath: "files/#{file_id}/files",
|
|
137
117
|
headers: headers,
|
|
138
118
|
query: request_args
|
|
139
|
-
)
|
|
119
|
+
)
|
|
140
120
|
end
|
|
141
121
|
|
|
142
122
|
# old version based on folders
|
|
@@ -221,8 +201,8 @@ module Aspera
|
|
|
221
201
|
'paths' => [{'source' => source_filename}],
|
|
222
202
|
'tags' => {Transfer::Spec::TAG_RESERVED => {PREV_GEN_TAG => true}}
|
|
223
203
|
})
|
|
224
|
-
# force destination, need to set this in transfer agent else it gets overwritten, not do: t_spec['destination_root']=destination
|
|
225
|
-
transfer.
|
|
204
|
+
# force destination, need to set this in transfer agent else it gets overwritten, do not do: t_spec['destination_root']=destination
|
|
205
|
+
transfer.user_transfer_spec['destination_root'] = destination
|
|
226
206
|
Main.result_transfer(transfer.start(t_spec))
|
|
227
207
|
end
|
|
228
208
|
|
|
@@ -311,7 +291,7 @@ module Aspera
|
|
|
311
291
|
next false
|
|
312
292
|
end
|
|
313
293
|
# shall we skip it ?
|
|
314
|
-
next false if @
|
|
294
|
+
next false if @option_skip_types.include?(gen_info[:generator].conversion_type)
|
|
315
295
|
# ok we need to generate
|
|
316
296
|
true
|
|
317
297
|
end
|
|
@@ -444,7 +424,7 @@ module Aspera
|
|
|
444
424
|
end
|
|
445
425
|
Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, mandatory: true)
|
|
446
426
|
# check tools that are anyway required for all cases
|
|
447
|
-
Aspera::Preview::Utils.check_tools(@
|
|
427
|
+
Aspera::Preview::Utils.check_tools(@option_skip_types)
|
|
448
428
|
case command
|
|
449
429
|
when :scan
|
|
450
430
|
scan_path = options.get_option(:scan_path)
|
|
@@ -491,7 +471,7 @@ module Aspera
|
|
|
491
471
|
g = Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder, nil)
|
|
492
472
|
g.generate
|
|
493
473
|
if command.eql?(:show)
|
|
494
|
-
terminal_options = options.get_option(:query
|
|
474
|
+
terminal_options = (options.get_option(:query) || {}).symbolize_keys
|
|
495
475
|
Log.log.debug{"preview: #{generated_file_path}"}
|
|
496
476
|
formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
|
|
497
477
|
end
|
|
@@ -32,7 +32,7 @@ module Aspera
|
|
|
32
32
|
|
|
33
33
|
class LocalExecutor
|
|
34
34
|
def execute(ascmd_path, input:)
|
|
35
|
-
return Environment.
|
|
35
|
+
return Environment.secure_execute(ascmd_path, mode: :capture, stdin_data: input, binmode: true, exception: false)
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
38
|
|
|
@@ -41,6 +41,7 @@ module Aspera
|
|
|
41
41
|
'HSTS Fasp/SSH'
|
|
42
42
|
end
|
|
43
43
|
|
|
44
|
+
# @return [Hash,NilClass]
|
|
44
45
|
def detect(address_or_url)
|
|
45
46
|
urls = if address_or_url.match?(%r{^[a-z]{1,6}://})
|
|
46
47
|
[address_or_url]
|
|
@@ -70,7 +71,7 @@ module Aspera
|
|
|
70
71
|
end
|
|
71
72
|
|
|
72
73
|
# @param wizard [Wizard] The wizard object
|
|
73
|
-
# @param app_url [
|
|
74
|
+
# @param app_url [String] Tested URL
|
|
74
75
|
# @return [Hash] :preset_value, :test_args
|
|
75
76
|
def wizard(wizard, app_url)
|
|
76
77
|
return {
|
|
@@ -87,9 +88,9 @@ module Aspera
|
|
|
87
88
|
super
|
|
88
89
|
@ssh_opts = {}
|
|
89
90
|
@connection_type = :ssh
|
|
90
|
-
options.declare(:ssh_keys, 'SSH key path list
|
|
91
|
+
options.declare(:ssh_keys, 'SSH key path list', allowed: Allowed::TYPES_STRING_ARRAY)
|
|
91
92
|
options.declare(:passphrase, 'SSH private key passphrase')
|
|
92
|
-
options.declare(:ssh_options, 'SSH options',
|
|
93
|
+
options.declare(:ssh_options, 'SSH options', allowed: Hash, handler: {o: self, m: :option_ssh_opts})
|
|
93
94
|
SyncActions.declare_options(options)
|
|
94
95
|
options.parse_options!
|
|
95
96
|
end
|
|
@@ -121,7 +122,7 @@ module Aspera
|
|
|
121
122
|
ENV['SSH_CLIENT'] = 'local 0 0'
|
|
122
123
|
@connection_type = :local
|
|
123
124
|
return server_transfer_spec
|
|
124
|
-
elsif transfer.
|
|
125
|
+
elsif transfer.user_transfer_spec['token'].is_a?(String) && server_uri.scheme.eql?(HTTPS_SCHEME)
|
|
125
126
|
server_transfer_spec['wss_enabled'] = true
|
|
126
127
|
server_transfer_spec['wss_port'] = server_uri.port
|
|
127
128
|
@connection_type = :wss
|
|
@@ -154,9 +155,7 @@ module Aspera
|
|
|
154
155
|
end
|
|
155
156
|
ssh_key_list = options.get_option(:ssh_keys)
|
|
156
157
|
if !ssh_key_list.nil?
|
|
157
|
-
ssh_key_list
|
|
158
|
-
Aspera.assert_type(ssh_key_list, Array){'ssh_keys'}
|
|
159
|
-
Aspera.assert(ssh_key_list.all?(String))
|
|
158
|
+
Aspera.assert_array_all(ssh_key_list, String){'ssh_keys'}
|
|
160
159
|
ssh_key_list.map!{ |p| File.expand_path(p)}
|
|
161
160
|
Log.log.debug{"SSH keys=#{ssh_key_list}"}
|
|
162
161
|
if !ssh_key_list.empty?
|
|
@@ -173,7 +172,7 @@ module Aspera
|
|
|
173
172
|
server_transfer_spec['ssh_private_key_passphrase'] = ssh_passphrase
|
|
174
173
|
end
|
|
175
174
|
# if user provided transfer spec has a token, we will use bypass keys
|
|
176
|
-
cred_set = true if transfer.
|
|
175
|
+
cred_set = true if transfer.user_transfer_spec['token'].is_a?(String)
|
|
177
176
|
Aspera.assert(cred_set, type: BadArgument){'Either password, key , or transfer spec token must be provided'}
|
|
178
177
|
return server_transfer_spec
|
|
179
178
|
end
|
|
@@ -227,7 +226,7 @@ module Aspera
|
|
|
227
226
|
end
|
|
228
227
|
else Aspera.error_unexpected_value(command_nagios)
|
|
229
228
|
end
|
|
230
|
-
|
|
229
|
+
Main.result_object_list(nagios.status_list)
|
|
231
230
|
when *TRANSFER_COMMANDS
|
|
232
231
|
return execute_transfer(command, server_transfer_spec)
|
|
233
232
|
when *AsCmd::OPERATIONS
|
|
@@ -13,32 +13,62 @@ module Aspera
|
|
|
13
13
|
# path for node admin after base url
|
|
14
14
|
ADMIN_API_PATH = 'api/v1'
|
|
15
15
|
class << self
|
|
16
|
+
# Check various endpoints on Shares
|
|
17
|
+
# @return [Hash] with version, ping, api
|
|
18
|
+
def health_check(url)
|
|
19
|
+
result = {}
|
|
20
|
+
result[:version] =
|
|
21
|
+
begin
|
|
22
|
+
version = nil
|
|
23
|
+
login_page = Rest
|
|
24
|
+
.new(base_url: url, redirect_max: 2)
|
|
25
|
+
.read('', headers: {'Accept'=>'text/html'})
|
|
26
|
+
if (m = login_page.match(/\(v([0-9a-f\.]+)\)/))
|
|
27
|
+
version = m[1]
|
|
28
|
+
if (m = login_page.match(/Patch level ([0-9]+)/))
|
|
29
|
+
version = "#{result[:version]} #{m[0]}"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
raise 'no version' if version.nil?
|
|
33
|
+
version
|
|
34
|
+
rescue => e
|
|
35
|
+
e
|
|
36
|
+
end
|
|
37
|
+
result[:ping] =
|
|
38
|
+
begin
|
|
39
|
+
Rest
|
|
40
|
+
.new(base_url: "#{url}/#{NODE_API_PATH}")
|
|
41
|
+
.read('ping', headers: {'Content-Type'=>'application/json'})
|
|
42
|
+
'ping ok'
|
|
43
|
+
rescue => e
|
|
44
|
+
e
|
|
45
|
+
end
|
|
46
|
+
result[:api] =
|
|
47
|
+
begin
|
|
48
|
+
resp = Rest.new(base_url: url, redirect_max: 1).read("#{NODE_API_PATH}/app", exception: false, ret: :resp)
|
|
49
|
+
# shall fail: shares requires auth, but we check error message
|
|
50
|
+
raise 'not found' unless resp.code.to_s.eql?('401') && resp.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
|
51
|
+
'available'
|
|
52
|
+
rescue => e
|
|
53
|
+
e
|
|
54
|
+
end
|
|
55
|
+
result
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# @return [Hash,NilClass]
|
|
16
59
|
def detect(address_or_url)
|
|
17
60
|
address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
begin
|
|
21
|
-
# shall fail: shares requires auth, but we check error message
|
|
22
|
-
# TODO: use ping instead ?
|
|
23
|
-
api.read("#{NODE_API_PATH}/app")
|
|
24
|
-
rescue RestCallError => e
|
|
25
|
-
found = true if e.response.code.to_s.eql?('401') && e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
|
26
|
-
end
|
|
27
|
-
return unless found
|
|
28
|
-
version = 'unknown'
|
|
29
|
-
test_page = api.call(operation: 'GET', subpath: 'login')
|
|
30
|
-
if (m = test_page[:http].body.match(/\(v(1\..*)\)/))
|
|
31
|
-
version = m[1]
|
|
32
|
-
end
|
|
61
|
+
health = health_check(address_or_url)
|
|
62
|
+
return unless health[:api].is_a?(String)
|
|
33
63
|
return {
|
|
34
|
-
version: version,
|
|
64
|
+
version: health[:version].is_a?(String) ? health[:version] : 'unknown',
|
|
35
65
|
url: address_or_url
|
|
36
66
|
}
|
|
37
67
|
end
|
|
38
68
|
end
|
|
39
69
|
|
|
40
70
|
# @param wizard [Wizard] The wizard object
|
|
41
|
-
# @param app_url [
|
|
71
|
+
# @param app_url [String] Tested URL
|
|
42
72
|
# @return [Hash] :preset_value, :test_args
|
|
43
73
|
def wizard(wizard, app_url)
|
|
44
74
|
return {
|
|
@@ -67,20 +97,20 @@ module Aspera
|
|
|
67
97
|
case command
|
|
68
98
|
when :health
|
|
69
99
|
nagios = Nagios.new
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
nagios.add_ok('
|
|
80
|
-
|
|
81
|
-
nagios.add_critical('API',
|
|
100
|
+
shares_url = options.get_option(:url, mandatory: true)
|
|
101
|
+
health = self.class.health_check(shares_url)
|
|
102
|
+
nagios.add_ok('version', health[:version]) if health[:version].is_a?(String)
|
|
103
|
+
if health[:ping].is_a?(String)
|
|
104
|
+
nagios.add_ok('ping', health[:ping])
|
|
105
|
+
else
|
|
106
|
+
nagios.add_critical('ping', health[:ping].to_s)
|
|
107
|
+
end
|
|
108
|
+
if health[:api].is_a?(String)
|
|
109
|
+
nagios.add_ok('API', health[:api])
|
|
110
|
+
else
|
|
111
|
+
nagios.add_critical('API', health[:api].to_s)
|
|
82
112
|
end
|
|
83
|
-
|
|
113
|
+
Main.result_object_list(nagios.status_list)
|
|
84
114
|
when :files
|
|
85
115
|
api_shares_node = basic_auth_api(NODE_API_PATH)
|
|
86
116
|
repo_command = options.get_next_command(Node::COMMANDS_SHARES)
|
|
@@ -90,6 +120,7 @@ module Aspera
|
|
|
90
120
|
when :admin
|
|
91
121
|
api_shares_admin = basic_auth_api(ADMIN_API_PATH)
|
|
92
122
|
admin_command = options.get_next_command(%i[node share transfer_settings user group].freeze)
|
|
123
|
+
lookup_share = ->(field, value){lookup_entity_generic(entity: 'share', field: field, value: value){api_shares_admin.read('data/shares')}['id']}
|
|
93
124
|
case admin_command
|
|
94
125
|
when :node
|
|
95
126
|
return entity_execute(api: api_shares_admin, entity: 'data/nodes')
|
|
@@ -98,13 +129,14 @@ module Aspera
|
|
|
98
129
|
case share_command
|
|
99
130
|
when *ALL_OPS
|
|
100
131
|
return entity_execute(
|
|
101
|
-
api:
|
|
102
|
-
entity:
|
|
103
|
-
command:
|
|
104
|
-
display_fields: %w[id name node_id directory percent_free]
|
|
132
|
+
api: api_shares_admin,
|
|
133
|
+
entity: 'data/shares',
|
|
134
|
+
command: share_command,
|
|
135
|
+
display_fields: %w[id name node_id directory percent_free],
|
|
136
|
+
&lookup_share
|
|
105
137
|
)
|
|
106
138
|
when :user_permissions, :group_permissions
|
|
107
|
-
share_id = instance_identifier
|
|
139
|
+
share_id = instance_identifier(&lookup_share)
|
|
108
140
|
return entity_execute(api: api_shares_admin, entity: "data/shares/#{share_id}/#{share_command}")
|
|
109
141
|
end
|
|
110
142
|
when :transfer_settings
|
|
@@ -137,20 +169,22 @@ module Aspera
|
|
|
137
169
|
entity_commands = %i[import].freeze
|
|
138
170
|
end
|
|
139
171
|
entity_verb = options.get_next_command(entity_commands)
|
|
172
|
+
lookup_block = ->(field, value){lookup_entity_generic(entity: entity_type, field: field, value: value){api_shares_admin.read(entities_path)}['id']}
|
|
140
173
|
case entity_verb
|
|
141
174
|
when *ALL_OPS # list, show, delete, create, modify
|
|
142
|
-
display_fields = entity_type.eql?(:user) ? %w[id username first_name last_name email] : nil
|
|
175
|
+
display_fields = entity_type.eql?(:user) ? %w[id user_id username first_name last_name email] : nil
|
|
143
176
|
display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:all)
|
|
144
177
|
return entity_execute(
|
|
145
|
-
api:
|
|
146
|
-
entity:
|
|
147
|
-
command:
|
|
148
|
-
display_fields: display_fields
|
|
178
|
+
api: api_shares_admin,
|
|
179
|
+
entity: entities_path,
|
|
180
|
+
command: entity_verb,
|
|
181
|
+
display_fields: display_fields,
|
|
182
|
+
&lookup_block
|
|
149
183
|
)
|
|
150
184
|
when *USR_GRP_SETTINGS # transfer_settings, app_authorizations, share_permissions
|
|
151
|
-
group_id = instance_identifier
|
|
185
|
+
group_id = instance_identifier(&lookup_block)
|
|
152
186
|
entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
|
|
153
|
-
return entity_execute(api: api_shares_admin, entity: entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
|
|
187
|
+
return entity_execute(api: api_shares_admin, entity: entities_path, is_singleton: !entity_verb.eql?(:share_permissions), &lookup_share)
|
|
154
188
|
when :import # saml
|
|
155
189
|
return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
|
|
156
190
|
entity_parameters = entity_parameters.transform_keys{ |k| k.gsub(/\s+/, '_').downcase}
|
|
@@ -166,7 +200,7 @@ module Aspera
|
|
|
166
200
|
api_shares_admin.create(entities_path, {entity_type=>entity_name})
|
|
167
201
|
end
|
|
168
202
|
when :users # group
|
|
169
|
-
return entity_execute(api: api_shares_admin, entity: "#{entities_path}/#{instance_identifier}/#{entities_prefix}users")
|
|
203
|
+
return entity_execute(api: api_shares_admin, entity: "#{entities_path}/#{instance_identifier(&lookup_block)}/#{entities_prefix}users")
|
|
170
204
|
else Aspera.error_unexpected_value(entity_verb)
|
|
171
205
|
end
|
|
172
206
|
end
|