aspera-cli 4.24.2 → 4.25.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1064 -758
- data/CONTRIBUTING.md +43 -100
- data/README.md +671 -419
- data/lib/aspera/api/aoc.rb +71 -43
- data/lib/aspera/api/cos_node.rb +3 -2
- data/lib/aspera/api/faspex.rb +6 -5
- data/lib/aspera/api/node.rb +10 -12
- data/lib/aspera/ascmd.rb +1 -2
- data/lib/aspera/ascp/installation.rb +53 -39
- data/lib/aspera/assert.rb +25 -3
- data/lib/aspera/cli/error.rb +4 -2
- data/lib/aspera/cli/extended_value.rb +84 -60
- data/lib/aspera/cli/formatter.rb +55 -22
- data/lib/aspera/cli/main.rb +21 -14
- data/lib/aspera/cli/manager.rb +348 -247
- data/lib/aspera/cli/plugins/alee.rb +3 -3
- data/lib/aspera/cli/plugins/aoc.rb +70 -14
- data/lib/aspera/cli/plugins/base.rb +57 -49
- data/lib/aspera/cli/plugins/config.rb +69 -84
- data/lib/aspera/cli/plugins/console.rb +13 -8
- data/lib/aspera/cli/plugins/cos.rb +1 -1
- data/lib/aspera/cli/plugins/faspex.rb +32 -26
- data/lib/aspera/cli/plugins/faspex5.rb +45 -43
- data/lib/aspera/cli/plugins/faspio.rb +5 -5
- data/lib/aspera/cli/plugins/httpgw.rb +1 -1
- data/lib/aspera/cli/plugins/node.rb +131 -120
- data/lib/aspera/cli/plugins/oauth.rb +1 -1
- data/lib/aspera/cli/plugins/orchestrator.rb +114 -32
- data/lib/aspera/cli/plugins/preview.rb +26 -46
- data/lib/aspera/cli/plugins/server.rb +6 -8
- data/lib/aspera/cli/plugins/shares.rb +27 -32
- data/lib/aspera/cli/sync_actions.rb +49 -38
- data/lib/aspera/cli/transfer_agent.rb +16 -34
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +8 -5
- data/lib/aspera/command_line_builder.rb +20 -17
- data/lib/aspera/coverage.rb +1 -1
- data/lib/aspera/environment.rb +41 -34
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/keychain/factory.rb +1 -2
- data/lib/aspera/markdown.rb +31 -0
- data/lib/aspera/nagios.rb +6 -5
- data/lib/aspera/oauth/base.rb +17 -27
- data/lib/aspera/oauth/factory.rb +1 -1
- data/lib/aspera/oauth/url_json.rb +2 -1
- data/lib/aspera/preview/file_types.rb +23 -37
- data/lib/aspera/products/connect.rb +3 -3
- data/lib/aspera/rest.rb +51 -39
- data/lib/aspera/rest_error_analyzer.rb +4 -4
- data/lib/aspera/ssh.rb +5 -2
- data/lib/aspera/ssl.rb +41 -0
- data/lib/aspera/sync/conf.schema.yaml +182 -34
- data/lib/aspera/sync/database.rb +2 -1
- data/lib/aspera/sync/operations.rb +125 -69
- data/lib/aspera/transfer/parameters.rb +3 -4
- data/lib/aspera/transfer/spec.rb +2 -3
- data/lib/aspera/transfer/spec.schema.yaml +48 -18
- data/lib/aspera/transfer/spec_doc.rb +14 -14
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/transferd_pb.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +19 -6
- metadata.gz.sig +3 -2
|
@@ -10,6 +10,7 @@ require 'xmlsimple'
|
|
|
10
10
|
module Aspera
|
|
11
11
|
module Cli
|
|
12
12
|
module Plugins
|
|
13
|
+
# Aspera Orchestrator
|
|
13
14
|
class Orchestrator < BasicAuth
|
|
14
15
|
STANDARD_PATH = '/aspera/orchestrator'
|
|
15
16
|
TEST_ENDPOINT = 'api/remote_node_ping'
|
|
@@ -23,11 +24,11 @@ module Aspera
|
|
|
23
24
|
urls.each do |base_url|
|
|
24
25
|
next unless base_url.match?('https?://')
|
|
25
26
|
api = Rest.new(base_url: base_url)
|
|
26
|
-
|
|
27
|
-
next unless
|
|
28
|
-
url =
|
|
27
|
+
data, http = api.read(TEST_ENDPOINT, query: {format: :json}, ret: :both)
|
|
28
|
+
next unless data['remote_orchestrator_info']
|
|
29
|
+
url = http.uri.to_s
|
|
29
30
|
return {
|
|
30
|
-
version:
|
|
31
|
+
version: data['remote_orchestrator_info']['orchestrator-version'],
|
|
31
32
|
url: url[0..url.index(TEST_ENDPOINT) - 2]
|
|
32
33
|
}
|
|
33
34
|
rescue StandardError => e
|
|
@@ -57,14 +58,12 @@ module Aspera
|
|
|
57
58
|
super
|
|
58
59
|
@api_orch = nil
|
|
59
60
|
options.declare(:result, "Specify result value as: 'work_step:parameter'")
|
|
60
|
-
options.declare(:synchronous, 'Wait for completion',
|
|
61
|
-
options.declare(:ret_style, 'How return type is requested in api',
|
|
62
|
-
options.declare(:auth_style, 'Authentication type',
|
|
61
|
+
options.declare(:synchronous, 'Wait for completion', allowed: Allowed::TYPES_BOOLEAN, default: false)
|
|
62
|
+
options.declare(:ret_style, 'How return type is requested in api', allowed: %i[header arg ext], default: :arg)
|
|
63
|
+
options.declare(:auth_style, 'Authentication type', allowed: %i[arg_pass head_basic apikey], default: :head_basic)
|
|
63
64
|
options.parse_options!
|
|
64
65
|
end
|
|
65
66
|
|
|
66
|
-
ACTIONS = %i[health info workflow plugins processes].freeze
|
|
67
|
-
|
|
68
67
|
# Call orchestrator API, it's a bit special
|
|
69
68
|
# @param endpoint [String] the endpoint to call
|
|
70
69
|
# @param ret_style [Symbol] the return style, :header, :arg, :ext(extension)
|
|
@@ -74,28 +73,31 @@ module Aspera
|
|
|
74
73
|
# @param http [Boolean] if true, returns the HttpResponse, else
|
|
75
74
|
def call_ao(endpoint, ret_style: nil, format: 'json', args: nil, xml_arrays: true, http: false)
|
|
76
75
|
# calls are all GET
|
|
77
|
-
call_args = {operation: 'GET', subpath: "api/#{endpoint}"}
|
|
76
|
+
call_args = {operation: 'GET', subpath: "api/#{endpoint}", ret: :both, query: {}}
|
|
78
77
|
ret_style = options.get_option(:ret_style, mandatory: true) if ret_style.nil?
|
|
79
|
-
call_args[:query]
|
|
78
|
+
call_args[:query].merge!(args) unless args.nil?
|
|
80
79
|
unless format.nil?
|
|
81
80
|
case ret_style
|
|
82
81
|
when :header
|
|
83
82
|
call_args[:headers] = {'Accept' => "application/#{format}"}
|
|
84
83
|
when :arg
|
|
85
|
-
call_args[:query] ||= {}
|
|
86
84
|
call_args[:query][:format] = format
|
|
87
85
|
when :ext
|
|
88
86
|
call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
|
|
89
87
|
else Aspera.error_unexpected_value(ret_style)
|
|
90
88
|
end
|
|
91
89
|
end
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
90
|
+
add_query = options.get_option(:query)
|
|
91
|
+
call_args[:query].merge!(add_query.symbolize_keys) unless add_query.nil?
|
|
92
|
+
data, resp = @api_orch.call(**call_args)
|
|
93
|
+
return resp if http
|
|
94
|
+
result = format.eql?('xml') ? XmlSimple.xml_in(resp.body, {'ForceArray' => xml_arrays}) : data
|
|
95
95
|
Log.dump(:data, result)
|
|
96
96
|
return result
|
|
97
97
|
end
|
|
98
98
|
|
|
99
|
+
ACTIONS = %i[health info workflows workorders workstep plugins processes monitors].freeze
|
|
100
|
+
|
|
99
101
|
def execute_action
|
|
100
102
|
auth_params =
|
|
101
103
|
case options.get_option(:auth_style, mandatory: true)
|
|
@@ -133,37 +135,33 @@ module Aspera
|
|
|
133
135
|
rescue StandardError => e
|
|
134
136
|
nagios.add_critical('node api', e.to_s)
|
|
135
137
|
end
|
|
136
|
-
|
|
138
|
+
Main.result_object_list(nagios.status_list)
|
|
139
|
+
# 14. Ping the remote Instance
|
|
137
140
|
when :info
|
|
138
141
|
result = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
|
|
139
142
|
return Main.result_single_object(result)
|
|
143
|
+
# 12. Orchestrator Background Process status
|
|
140
144
|
when :processes
|
|
141
145
|
# TODO: Bug ? API has only XML format
|
|
142
146
|
result = call_ao('processes_status', format: 'xml')
|
|
143
147
|
return Main.result_object_list(result['process'])
|
|
148
|
+
# 13. Orchestrator Monitor
|
|
149
|
+
when :monitors
|
|
150
|
+
result = call_ao('monitor_snapshot')
|
|
151
|
+
return Main.result_single_object(result['monitor'])
|
|
144
152
|
when :plugins
|
|
145
153
|
# TODO: Bug ? only json format on url
|
|
146
154
|
result = call_ao('plugin_version')
|
|
147
155
|
return Main.result_object_list(result['Plugin'])
|
|
148
|
-
when :
|
|
149
|
-
command = options.get_next_command(%i[list status inputs details start export])
|
|
156
|
+
when :workflows
|
|
157
|
+
command = options.get_next_command(%i[list status inputs details start export workorders outputs])
|
|
150
158
|
case command
|
|
151
|
-
|
|
152
|
-
wf_id = instance_identifier
|
|
153
|
-
result = call_ao(wf_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{wf_id}")
|
|
154
|
-
return Main.result_object_list(result['workflows']['workflow'])
|
|
159
|
+
# 1. List all available workflows on the system
|
|
155
160
|
when :list
|
|
156
|
-
result = call_ao('workflows_list
|
|
161
|
+
result = call_ao('workflows_list')
|
|
157
162
|
return Main.result_object_list(result['workflows']['workflow'], fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification])
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
return Main.result_object_list(result['workflows']['workflow']['statuses'])
|
|
161
|
-
when :inputs
|
|
162
|
-
result = call_ao("workflow_inputs_spec/#{instance_identifier}")
|
|
163
|
-
return Main.result_single_object(result['workflow_inputs_spec'])
|
|
164
|
-
when :export
|
|
165
|
-
result = call_ao("export_workflow/#{instance_identifier}", format: nil, http: true)
|
|
166
|
-
return Main.result_text(result.body)
|
|
163
|
+
# 2.1 Initiate a workorder - Asynchronous
|
|
164
|
+
# 2.2 Initiate a workorder - Synchronous
|
|
167
165
|
when :start
|
|
168
166
|
result = {
|
|
169
167
|
type: :single_object,
|
|
@@ -191,6 +189,69 @@ module Aspera
|
|
|
191
189
|
result[:type] = :text if call_params['synchronous']
|
|
192
190
|
result[:data] = call_ao("initiate/#{wf_id}", args: call_params)
|
|
193
191
|
return result
|
|
192
|
+
# 3. Fetch input specification for a workflow
|
|
193
|
+
when :inputs
|
|
194
|
+
result = call_ao("workflow_inputs_spec/#{instance_identifier}")
|
|
195
|
+
return Main.result_single_object(result['workflow_inputs_spec'])
|
|
196
|
+
# 4. Check the running status for all workflows
|
|
197
|
+
# 5. Check the running status for a particular workflow
|
|
198
|
+
when :status
|
|
199
|
+
wf_id = instance_identifier
|
|
200
|
+
result = call_ao(wf_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{wf_id}")
|
|
201
|
+
return Main.result_object_list(result['workflows']['workflow'])
|
|
202
|
+
# 6. Check the detailed running status for a particular workflow
|
|
203
|
+
when :details
|
|
204
|
+
result = call_ao("workflow_details/#{instance_identifier}")
|
|
205
|
+
return Main.result_object_list(result['workflows']['workflow']['statuses'])
|
|
206
|
+
# 15. Fetch output specification for a particular work flow
|
|
207
|
+
when :outputs
|
|
208
|
+
result = call_ao("workflow_outputs_spec/#{instance_identifier}")
|
|
209
|
+
return Main.result_object_list(result['workflow_outputs_spec']['output'])
|
|
210
|
+
# 19.Fetch all workorders from a workflow
|
|
211
|
+
when :workorders
|
|
212
|
+
result = call_ao("work_orders_list/#{instance_identifier}")
|
|
213
|
+
return Main.result_object_list(result['work_orders'])
|
|
214
|
+
when :export
|
|
215
|
+
result = call_ao("export_workflow/#{instance_identifier}", format: nil, http: true)
|
|
216
|
+
return Main.result_text(result.body)
|
|
217
|
+
end
|
|
218
|
+
when :workorders
|
|
219
|
+
command = options.get_next_command(%i[status cancel reset output])
|
|
220
|
+
case command
|
|
221
|
+
# 7. Check the status for a particular work order
|
|
222
|
+
when :status
|
|
223
|
+
wo_id = instance_identifier
|
|
224
|
+
result = call_ao("work_order_status/#{wo_id}")
|
|
225
|
+
return Main.result_single_object(result['work_order'])
|
|
226
|
+
# 9. Cancel a Work Order
|
|
227
|
+
when :cancel
|
|
228
|
+
wo_id = instance_identifier
|
|
229
|
+
result = call_ao("work_order_cancel/#{wo_id}")
|
|
230
|
+
return Main.result_single_object(result['work_order'])
|
|
231
|
+
# 11. Reset a Work order
|
|
232
|
+
when :reset
|
|
233
|
+
wo_id = instance_identifier
|
|
234
|
+
result = call_ao("work_order_reset/#{wo_id}")
|
|
235
|
+
return Main.result_single_object(result['work_order'])
|
|
236
|
+
# 16. Fetch output of a work order
|
|
237
|
+
when :output
|
|
238
|
+
wo_id = instance_identifier
|
|
239
|
+
result = call_ao("work_order_output/#{wo_id}", format: 'xml')
|
|
240
|
+
return Main.result_object_list(result['variable'])
|
|
241
|
+
end
|
|
242
|
+
when :workstep
|
|
243
|
+
command = options.get_next_command(%i[status cancel])
|
|
244
|
+
case command
|
|
245
|
+
# 8. Check the status of a Step
|
|
246
|
+
when :status
|
|
247
|
+
ws_id = instance_identifier
|
|
248
|
+
result = call_ao("work_step_status/#{ws_id}")
|
|
249
|
+
return Main.result_single_object(result)
|
|
250
|
+
# 10. Cancel a Work Step
|
|
251
|
+
when :cancel
|
|
252
|
+
ws_id = instance_identifier
|
|
253
|
+
result = call_ao("work_step_cancel/#{ws_id}")
|
|
254
|
+
return Main.result_single_object(result)
|
|
194
255
|
end
|
|
195
256
|
else Aspera.error_unexpected_value(command)
|
|
196
257
|
end
|
|
@@ -200,3 +261,24 @@ module Aspera
|
|
|
200
261
|
end
|
|
201
262
|
end
|
|
202
263
|
end
|
|
264
|
+
|
|
265
|
+
# 17.Persist custom data
|
|
266
|
+
# 18.Fetch queued items from queue
|
|
267
|
+
# 20.List Task for a User
|
|
268
|
+
# 21. Fetch Task details
|
|
269
|
+
# 22. Submit Task
|
|
270
|
+
# 23. Control Process
|
|
271
|
+
# engine monitor worker
|
|
272
|
+
# 24. Lookup Queued Item
|
|
273
|
+
# 25. Reorder Queued Items
|
|
274
|
+
# 26. Bulk Reorder Queued Items
|
|
275
|
+
# 27. Queue Item (Add an item to a Queue)
|
|
276
|
+
#
|
|
277
|
+
# Required Input:
|
|
278
|
+
# Optional Input:
|
|
279
|
+
# 28.List all queues
|
|
280
|
+
# 29. Portlet Version
|
|
281
|
+
# 30. Plugin Version
|
|
282
|
+
# 31. Restart Work Order from a Step
|
|
283
|
+
# 32. Delete element from a Managed Queue
|
|
284
|
+
#
|
|
@@ -46,16 +46,15 @@ module Aspera
|
|
|
46
46
|
:AK_MARKER_FILE,
|
|
47
47
|
:LOG_LIMITER_SEC
|
|
48
48
|
|
|
49
|
-
|
|
50
|
-
attr_accessor :option_previews_folder
|
|
51
|
-
attr_accessor :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
|
|
49
|
+
attr_accessor :option_skip_types, :option_previews_folder, :option_folder_reset_cache, :option_skip_folders, :option_overwrite, :option_file_access
|
|
52
50
|
|
|
53
51
|
def initialize(**_)
|
|
54
52
|
super
|
|
55
|
-
@
|
|
56
|
-
@
|
|
57
|
-
|
|
58
|
-
@
|
|
53
|
+
@option_skip_types = []
|
|
54
|
+
@option_skip_folders = []
|
|
55
|
+
@option_previews_folder = nil
|
|
56
|
+
@option_overwrite = nil
|
|
57
|
+
@option_folder_reset_cache = nil
|
|
59
58
|
# options for generation
|
|
60
59
|
@gen_options = Aspera::Preview::Options.new
|
|
61
60
|
# used to trigger periodic processing
|
|
@@ -64,68 +63,49 @@ module Aspera
|
|
|
64
63
|
@filter_block = nil
|
|
65
64
|
# link CLI options to gen_info attributes
|
|
66
65
|
options.declare(
|
|
67
|
-
:skip_format, 'Skip this preview format
|
|
68
|
-
|
|
66
|
+
:skip_format, 'Skip this preview format',
|
|
67
|
+
allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
|
|
69
68
|
)
|
|
70
69
|
options.declare(
|
|
71
70
|
:folder_reset_cache, 'Force detection of generated preview by refresh cache',
|
|
72
|
-
|
|
71
|
+
allowed: %i[no header read],
|
|
73
72
|
handler: {o: self, m: :option_folder_reset_cache},
|
|
74
73
|
default: :no
|
|
75
74
|
)
|
|
76
|
-
options.declare(:skip_types, 'Skip
|
|
75
|
+
options.declare(:skip_types, 'Skip generation for those types of files', handler: {o: self, m: :option_skip_types}, allowed: Allowed::TYPES_SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES)
|
|
77
76
|
options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
|
|
78
|
-
options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders},
|
|
77
|
+
options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, allowed: Allowed::TYPES_STRING_ARRAY)
|
|
79
78
|
options.declare(:base, 'Basename of output for for test')
|
|
80
79
|
options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
|
|
81
80
|
options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
|
|
82
|
-
options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic',
|
|
83
|
-
options.declare(:overwrite, 'When to overwrite result file',
|
|
81
|
+
options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', allowed: Allowed::TYPES_BOOLEAN, default: false)
|
|
82
|
+
options.declare(:overwrite, 'When to overwrite result file', handler: {o: self, m: :option_overwrite}, allowed: %i[always never mtime], default: :mtime)
|
|
84
83
|
options.declare(
|
|
85
84
|
:file_access, 'How to read and write files in repository',
|
|
86
|
-
|
|
85
|
+
allowed: %i[local remote],
|
|
87
86
|
handler: {o: self, m: :option_file_access},
|
|
88
87
|
default: :local
|
|
89
88
|
)
|
|
90
|
-
|
|
91
89
|
# add other options for generator (and set default values)
|
|
92
90
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
|
93
91
|
values = if opt.key?(:values)
|
|
94
92
|
opt[:values]
|
|
95
93
|
elsif Cli::Manager::BOOLEAN_SIMPLE.include?(opt[:default])
|
|
96
|
-
|
|
94
|
+
Allowed::TYPES_BOOLEAN
|
|
97
95
|
end
|
|
98
|
-
options.declare(opt[:name], opt[:description].capitalize,
|
|
96
|
+
options.declare(opt[:name], opt[:description].capitalize, allowed: values, handler: {o: @gen_options, m: opt[:name]}, default: opt[:default])
|
|
99
97
|
end
|
|
100
98
|
|
|
101
99
|
options.parse_options!
|
|
102
|
-
|
|
100
|
+
# by default generate all supported formats (clone, as altered by options)
|
|
101
|
+
@preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
|
|
102
|
+
skip = options.get_option(:skip_format)
|
|
103
|
+
@preview_formats_to_generate.delete(skip) if skip
|
|
103
104
|
@tmp_folder = File.join(TempFileManager.instance.global_temp, "#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
|
|
104
105
|
FileUtils.mkdir_p(@tmp_folder)
|
|
105
106
|
Log.log.debug{"tmpdir: #{@tmp_folder}"}
|
|
106
107
|
end
|
|
107
108
|
|
|
108
|
-
def option_skip_types=(value)
|
|
109
|
-
@skip_types = []
|
|
110
|
-
value.split(',').each do |v|
|
|
111
|
-
s = v.to_sym
|
|
112
|
-
Aspera.assert_values(s, Aspera::Preview::FileTypes::CONVERSION_TYPES){'skip_types'}
|
|
113
|
-
@skip_types.push(s)
|
|
114
|
-
end
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
def option_skip_types
|
|
118
|
-
return @skip_types.map(&:to_s).join(',')
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
def option_skip_format=(value)
|
|
122
|
-
@preview_formats_to_generate.delete(value)
|
|
123
|
-
end
|
|
124
|
-
|
|
125
|
-
def option_skip_format
|
|
126
|
-
return @preview_formats_to_generate.map(&:to_s).join(',')
|
|
127
|
-
end
|
|
128
|
-
|
|
129
109
|
# /files/id/files is normally cached in redis, but we can discard the cache
|
|
130
110
|
# but /files/id is not cached
|
|
131
111
|
def get_folder_entries(file_id, request_args = nil)
|
|
@@ -136,7 +116,7 @@ module Aspera
|
|
|
136
116
|
subpath: "files/#{file_id}/files",
|
|
137
117
|
headers: headers,
|
|
138
118
|
query: request_args
|
|
139
|
-
)
|
|
119
|
+
)
|
|
140
120
|
end
|
|
141
121
|
|
|
142
122
|
# old version based on folders
|
|
@@ -221,8 +201,8 @@ module Aspera
|
|
|
221
201
|
'paths' => [{'source' => source_filename}],
|
|
222
202
|
'tags' => {Transfer::Spec::TAG_RESERVED => {PREV_GEN_TAG => true}}
|
|
223
203
|
})
|
|
224
|
-
# force destination, need to set this in transfer agent else it gets overwritten, not do: t_spec['destination_root']=destination
|
|
225
|
-
transfer.
|
|
204
|
+
# force destination, need to set this in transfer agent else it gets overwritten, do not do: t_spec['destination_root']=destination
|
|
205
|
+
transfer.user_transfer_spec['destination_root'] = destination
|
|
226
206
|
Main.result_transfer(transfer.start(t_spec))
|
|
227
207
|
end
|
|
228
208
|
|
|
@@ -311,7 +291,7 @@ module Aspera
|
|
|
311
291
|
next false
|
|
312
292
|
end
|
|
313
293
|
# shall we skip it ?
|
|
314
|
-
next false if @
|
|
294
|
+
next false if @option_skip_types.include?(gen_info[:generator].conversion_type)
|
|
315
295
|
# ok we need to generate
|
|
316
296
|
true
|
|
317
297
|
end
|
|
@@ -444,7 +424,7 @@ module Aspera
|
|
|
444
424
|
end
|
|
445
425
|
Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, mandatory: true)
|
|
446
426
|
# check tools that are anyway required for all cases
|
|
447
|
-
Aspera::Preview::Utils.check_tools(@
|
|
427
|
+
Aspera::Preview::Utils.check_tools(@option_skip_types)
|
|
448
428
|
case command
|
|
449
429
|
when :scan
|
|
450
430
|
scan_path = options.get_option(:scan_path)
|
|
@@ -491,7 +471,7 @@ module Aspera
|
|
|
491
471
|
g = Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder, nil)
|
|
492
472
|
g.generate
|
|
493
473
|
if command.eql?(:show)
|
|
494
|
-
terminal_options = options.get_option(:query
|
|
474
|
+
terminal_options = (options.get_option(:query) || {}).symbolize_keys
|
|
495
475
|
Log.log.debug{"preview: #{generated_file_path}"}
|
|
496
476
|
formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
|
|
497
477
|
end
|
|
@@ -87,9 +87,9 @@ module Aspera
|
|
|
87
87
|
super
|
|
88
88
|
@ssh_opts = {}
|
|
89
89
|
@connection_type = :ssh
|
|
90
|
-
options.declare(:ssh_keys, 'SSH key path list
|
|
90
|
+
options.declare(:ssh_keys, 'SSH key path list', allowed: Allowed::TYPES_STRING_ARRAY)
|
|
91
91
|
options.declare(:passphrase, 'SSH private key passphrase')
|
|
92
|
-
options.declare(:ssh_options, 'SSH options',
|
|
92
|
+
options.declare(:ssh_options, 'SSH options', allowed: Hash, handler: {o: self, m: :option_ssh_opts})
|
|
93
93
|
SyncActions.declare_options(options)
|
|
94
94
|
options.parse_options!
|
|
95
95
|
end
|
|
@@ -121,7 +121,7 @@ module Aspera
|
|
|
121
121
|
ENV['SSH_CLIENT'] = 'local 0 0'
|
|
122
122
|
@connection_type = :local
|
|
123
123
|
return server_transfer_spec
|
|
124
|
-
elsif transfer.
|
|
124
|
+
elsif transfer.user_transfer_spec['token'].is_a?(String) && server_uri.scheme.eql?(HTTPS_SCHEME)
|
|
125
125
|
server_transfer_spec['wss_enabled'] = true
|
|
126
126
|
server_transfer_spec['wss_port'] = server_uri.port
|
|
127
127
|
@connection_type = :wss
|
|
@@ -154,9 +154,7 @@ module Aspera
|
|
|
154
154
|
end
|
|
155
155
|
ssh_key_list = options.get_option(:ssh_keys)
|
|
156
156
|
if !ssh_key_list.nil?
|
|
157
|
-
ssh_key_list
|
|
158
|
-
Aspera.assert_type(ssh_key_list, Array){'ssh_keys'}
|
|
159
|
-
Aspera.assert(ssh_key_list.all?(String))
|
|
157
|
+
Aspera.assert_array_all(ssh_key_list, String){'ssh_keys'}
|
|
160
158
|
ssh_key_list.map!{ |p| File.expand_path(p)}
|
|
161
159
|
Log.log.debug{"SSH keys=#{ssh_key_list}"}
|
|
162
160
|
if !ssh_key_list.empty?
|
|
@@ -173,7 +171,7 @@ module Aspera
|
|
|
173
171
|
server_transfer_spec['ssh_private_key_passphrase'] = ssh_passphrase
|
|
174
172
|
end
|
|
175
173
|
# if user provided transfer spec has a token, we will use bypass keys
|
|
176
|
-
cred_set = true if transfer.
|
|
174
|
+
cred_set = true if transfer.user_transfer_spec['token'].is_a?(String)
|
|
177
175
|
Aspera.assert(cred_set, type: BadArgument){'Either password, key , or transfer spec token must be provided'}
|
|
178
176
|
return server_transfer_spec
|
|
179
177
|
end
|
|
@@ -227,7 +225,7 @@ module Aspera
|
|
|
227
225
|
end
|
|
228
226
|
else Aspera.error_unexpected_value(command_nagios)
|
|
229
227
|
end
|
|
230
|
-
|
|
228
|
+
Main.result_object_list(nagios.status_list)
|
|
231
229
|
when *TRANSFER_COMMANDS
|
|
232
230
|
return execute_transfer(command, server_transfer_spec)
|
|
233
231
|
when *AsCmd::OPERATIONS
|
|
@@ -16,18 +16,13 @@ module Aspera
|
|
|
16
16
|
def detect(address_or_url)
|
|
17
17
|
address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
|
|
18
18
|
api = Rest.new(base_url: address_or_url, redirect_max: 1)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
api.read("#{NODE_API_PATH}/app")
|
|
24
|
-
rescue RestCallError => e
|
|
25
|
-
found = true if e.response.code.to_s.eql?('401') && e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
|
26
|
-
end
|
|
27
|
-
return unless found
|
|
19
|
+
# TODO: use ping instead ?
|
|
20
|
+
resp = api.read("#{NODE_API_PATH}/app", exception: false, ret: :resp)
|
|
21
|
+
# shall fail: shares requires auth, but we check error message
|
|
22
|
+
return unless resp.code.to_s.eql?('401') && resp.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
|
28
23
|
version = 'unknown'
|
|
29
|
-
|
|
30
|
-
if (m =
|
|
24
|
+
http = api.read('login', headers: {'Accept'=>'*/*'}, ret: :resp)
|
|
25
|
+
if (m = http.body.match(/\(v(1\..*)\)/))
|
|
31
26
|
version = m[1]
|
|
32
27
|
end
|
|
33
28
|
return {
|
|
@@ -68,19 +63,15 @@ module Aspera
|
|
|
68
63
|
when :health
|
|
69
64
|
nagios = Nagios.new
|
|
70
65
|
begin
|
|
71
|
-
|
|
66
|
+
http = Rest
|
|
72
67
|
.new(base_url: "#{options.get_option(:url, mandatory: true)}/#{NODE_API_PATH}")
|
|
73
|
-
.
|
|
74
|
-
|
|
75
|
-
subpath: 'ping',
|
|
76
|
-
headers: {'content-type': Rest::MIME_JSON}
|
|
77
|
-
)
|
|
78
|
-
raise Error, 'Shares not detected' unless res[:http].body.eql?(' ')
|
|
68
|
+
.read('ping', ret: :resp)
|
|
69
|
+
raise Error, 'Shares not detected' unless http.body.eql?(' ')
|
|
79
70
|
nagios.add_ok('shares api', 'accessible')
|
|
80
71
|
rescue StandardError => e
|
|
81
72
|
nagios.add_critical('API', e.to_s)
|
|
82
73
|
end
|
|
83
|
-
|
|
74
|
+
Main.result_object_list(nagios.status_list)
|
|
84
75
|
when :files
|
|
85
76
|
api_shares_node = basic_auth_api(NODE_API_PATH)
|
|
86
77
|
repo_command = options.get_next_command(Node::COMMANDS_SHARES)
|
|
@@ -90,6 +81,7 @@ module Aspera
|
|
|
90
81
|
when :admin
|
|
91
82
|
api_shares_admin = basic_auth_api(ADMIN_API_PATH)
|
|
92
83
|
admin_command = options.get_next_command(%i[node share transfer_settings user group].freeze)
|
|
84
|
+
lookup_share = ->(field, value){lookup_entity_generic(entity: 'share', field: field, value: value){api_shares_admin.read('data/shares')}['id']}
|
|
93
85
|
case admin_command
|
|
94
86
|
when :node
|
|
95
87
|
return entity_execute(api: api_shares_admin, entity: 'data/nodes')
|
|
@@ -98,13 +90,14 @@ module Aspera
|
|
|
98
90
|
case share_command
|
|
99
91
|
when *ALL_OPS
|
|
100
92
|
return entity_execute(
|
|
101
|
-
api:
|
|
102
|
-
entity:
|
|
103
|
-
command:
|
|
104
|
-
display_fields: %w[id name node_id directory percent_free]
|
|
93
|
+
api: api_shares_admin,
|
|
94
|
+
entity: 'data/shares',
|
|
95
|
+
command: share_command,
|
|
96
|
+
display_fields: %w[id name node_id directory percent_free],
|
|
97
|
+
&lookup_share
|
|
105
98
|
)
|
|
106
99
|
when :user_permissions, :group_permissions
|
|
107
|
-
share_id = instance_identifier
|
|
100
|
+
share_id = instance_identifier(&lookup_share)
|
|
108
101
|
return entity_execute(api: api_shares_admin, entity: "data/shares/#{share_id}/#{share_command}")
|
|
109
102
|
end
|
|
110
103
|
when :transfer_settings
|
|
@@ -137,20 +130,22 @@ module Aspera
|
|
|
137
130
|
entity_commands = %i[import].freeze
|
|
138
131
|
end
|
|
139
132
|
entity_verb = options.get_next_command(entity_commands)
|
|
133
|
+
lookup_block = ->(field, value){lookup_entity_generic(entity: entity_type, field: field, value: value){api_shares_admin.read(entities_path)}['id']}
|
|
140
134
|
case entity_verb
|
|
141
135
|
when *ALL_OPS # list, show, delete, create, modify
|
|
142
|
-
display_fields = entity_type.eql?(:user) ? %w[id username first_name last_name email] : nil
|
|
136
|
+
display_fields = entity_type.eql?(:user) ? %w[id user_id username first_name last_name email] : nil
|
|
143
137
|
display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:all)
|
|
144
138
|
return entity_execute(
|
|
145
|
-
api:
|
|
146
|
-
entity:
|
|
147
|
-
command:
|
|
148
|
-
display_fields: display_fields
|
|
139
|
+
api: api_shares_admin,
|
|
140
|
+
entity: entities_path,
|
|
141
|
+
command: entity_verb,
|
|
142
|
+
display_fields: display_fields,
|
|
143
|
+
&lookup_block
|
|
149
144
|
)
|
|
150
145
|
when *USR_GRP_SETTINGS # transfer_settings, app_authorizations, share_permissions
|
|
151
|
-
group_id = instance_identifier
|
|
146
|
+
group_id = instance_identifier(&lookup_block)
|
|
152
147
|
entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
|
|
153
|
-
return entity_execute(api: api_shares_admin, entity: entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
|
|
148
|
+
return entity_execute(api: api_shares_admin, entity: entities_path, is_singleton: !entity_verb.eql?(:share_permissions), &lookup_share)
|
|
154
149
|
when :import # saml
|
|
155
150
|
return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
|
|
156
151
|
entity_parameters = entity_parameters.transform_keys{ |k| k.gsub(/\s+/, '_').downcase}
|
|
@@ -166,7 +161,7 @@ module Aspera
|
|
|
166
161
|
api_shares_admin.create(entities_path, {entity_type=>entity_name})
|
|
167
162
|
end
|
|
168
163
|
when :users # group
|
|
169
|
-
return entity_execute(api: api_shares_admin, entity: "#{entities_path}/#{instance_identifier}/#{entities_prefix}users")
|
|
164
|
+
return entity_execute(api: api_shares_admin, entity: "#{entities_path}/#{instance_identifier(&lookup_block)}/#{entities_prefix}users")
|
|
170
165
|
else Aspera.error_unexpected_value(entity_verb)
|
|
171
166
|
end
|
|
172
167
|
end
|