aspera-cli 4.27.1 → 4.27.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +67 -1
- data/bin/ascli +2 -1
- data/docs/README.md +805 -747
- data/lib/aspera/agent/connect.rb +6 -4
- data/lib/aspera/agent/desktop.rb +2 -2
- data/lib/aspera/agent/direct.rb +3 -1
- data/lib/aspera/agent/node.rb +3 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -12
- data/lib/aspera/api/ats.rb +1 -1
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +9 -7
- data/lib/aspera/api/httpgw.rb +37 -33
- data/lib/aspera/api/node.rb +38 -33
- data/lib/aspera/ascmd.rb +3 -1
- data/lib/aspera/ascp/installation.rb +62 -27
- data/lib/aspera/ascp/management.rb +1 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +20 -41
- data/lib/aspera/cli/async_transfer_store.rb +2 -2
- data/lib/aspera/cli/bootstrapper.rb +11 -15
- data/lib/aspera/cli/command_line.rb +252 -0
- data/lib/aspera/cli/command_registry.rb +149 -33
- data/lib/aspera/cli/command_spec.rb +103 -14
- data/lib/aspera/cli/completion/ascli.bash +12 -0
- data/lib/aspera/cli/completion/ascli.fish +16 -0
- data/lib/aspera/cli/completion/ascli.zsh +19 -0
- data/lib/aspera/cli/context.rb +3 -0
- data/lib/aspera/cli/deprecation.rb +37 -0
- data/lib/aspera/cli/extended_value.rb +2 -0
- data/lib/aspera/cli/formatter.rb +87 -75
- data/lib/aspera/cli/gem_checker.rb +1 -1
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/http.rb +21 -21
- data/lib/aspera/cli/info.rb +3 -0
- data/lib/aspera/cli/mcp_tool.rb +47 -83
- data/lib/aspera/cli/option_declarator.rb +33 -42
- data/lib/aspera/cli/option_registry.rb +69 -0
- data/lib/aspera/cli/option_types.rb +103 -0
- data/lib/aspera/cli/option_value.rb +281 -0
- data/lib/aspera/cli/options.schema.yaml +38 -5
- data/lib/aspera/cli/parser.rb +307 -848
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +435 -380
- data/lib/aspera/cli/plugins/ats.rb +58 -73
- data/lib/aspera/cli/plugins/base.rb +190 -240
- data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
- data/lib/aspera/cli/plugins/config.rb +244 -178
- data/lib/aspera/cli/plugins/console.rb +102 -38
- data/lib/aspera/cli/plugins/cos.rb +6 -23
- data/lib/aspera/cli/plugins/factory.rb +3 -0
- data/lib/aspera/cli/plugins/faspex5.rb +176 -173
- data/lib/aspera/cli/plugins/faspio.rb +5 -10
- data/lib/aspera/cli/plugins/httpgw.rb +8 -11
- data/lib/aspera/cli/plugins/mcp.rb +20 -55
- data/lib/aspera/cli/plugins/node.rb +277 -311
- data/lib/aspera/cli/plugins/orchestrator.rb +90 -77
- data/lib/aspera/cli/plugins/preview.rb +79 -90
- data/lib/aspera/cli/plugins/server.rb +76 -50
- data/lib/aspera/cli/plugins/shares.rb +68 -116
- data/lib/aspera/cli/preset_actions.rb +17 -10
- data/lib/aspera/cli/preset_manager.rb +12 -2
- data/lib/aspera/cli/prompt.rb +35 -0
- data/lib/aspera/cli/result.rb +13 -18
- data/lib/aspera/cli/runner.rb +31 -54
- data/lib/aspera/cli/special_values.rb +5 -0
- data/lib/aspera/cli/sync_actions.rb +41 -37
- data/lib/aspera/cli/terminal_formatter.rb +9 -3
- data/lib/aspera/cli/transfer_actions.rb +0 -6
- data/lib/aspera/cli/transfer_agent.rb +29 -35
- data/lib/aspera/cli/vault_manager.rb +0 -17
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +4 -2
- data/lib/aspera/command_line_builder.rb +1 -0
- data/lib/aspera/coverage.rb +1 -0
- data/lib/aspera/environment.rb +7 -1
- data/lib/aspera/faspex_gw.rb +2 -1
- data/lib/aspera/faspex_postproc.rb +1 -0
- data/lib/aspera/graphql.rb +5 -5
- data/lib/aspera/json_rpc/client.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +1 -1
- data/lib/aspera/keychain/one_password_api.rb +1 -1
- data/lib/aspera/link_header.rb +2 -2
- data/lib/aspera/log.rb +22 -25
- data/lib/aspera/markdown.rb +2 -0
- data/lib/aspera/mime.rb +25 -0
- data/lib/aspera/node_simulator.rb +1 -0
- data/lib/aspera/oauth/base.rb +35 -25
- data/lib/aspera/oauth/factory.rb +1 -0
- data/lib/aspera/oauth/generic.rb +1 -1
- data/lib/aspera/oauth/jwt.rb +1 -1
- data/lib/aspera/oauth/web.rb +9 -8
- data/lib/aspera/preview/file_types.rb +4 -4
- data/lib/aspera/preview/generator.rb +7 -0
- data/lib/aspera/preview/options.rb +4 -4
- data/lib/aspera/preview/terminal.rb +4 -3
- data/lib/aspera/preview/utils.rb +9 -6
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/rainbow.rb +7 -0
- data/lib/aspera/rest/aspera_errors.rb +60 -0
- data/lib/aspera/rest/call_error.rb +27 -0
- data/lib/aspera/rest/client.rb +514 -0
- data/lib/aspera/rest/error_analyzer.rb +113 -0
- data/lib/aspera/rest/list.rb +143 -0
- data/lib/aspera/rest/parameters.rb +55 -0
- data/lib/aspera/rest/util.rb +176 -0
- data/lib/aspera/rest.rb +7 -621
- data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +39 -0
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
- data/lib/aspera/schema/documentation.rb +13 -3
- data/lib/aspera/schema/registry.rb +18 -1
- data/lib/aspera/schema/validator.rb +92 -0
- data/lib/aspera/secret_hider.rb +36 -25
- data/lib/aspera/string_ext.rb +15 -0
- data/lib/aspera/temp_file_manager.rb +6 -5
- data/lib/aspera/transfer/parameters.rb +2 -0
- data/lib/aspera/transfer/spec.rb +1 -0
- data/lib/aspera/transfer/spec.schema.yaml +1 -0
- data/lib/aspera/uri_reader.rb +11 -11
- data/lib/aspera/web_auth/index.html +147 -0
- data/lib/aspera/web_auth/server.rb +81 -0
- data.tar.gz.sig +0 -0
- metadata +39 -7
- metadata.gz.sig +0 -0
- data/lib/aspera/colors.rb +0 -79
- data/lib/aspera/rest_call_error.rb +0 -25
- data/lib/aspera/rest_error_analyzer.rb +0 -111
- data/lib/aspera/rest_errors_aspera.rb +0 -58
- data/lib/aspera/rest_list.rb +0 -136
- data/lib/aspera/web_auth.rb +0 -211
|
@@ -25,7 +25,7 @@ module Aspera
|
|
|
25
25
|
error = nil
|
|
26
26
|
urls.each do |base_url|
|
|
27
27
|
next unless base_url.match?(%r{^https?://})
|
|
28
|
-
api = Rest.new(base_url: base_url)
|
|
28
|
+
api = Rest::Client.new(base_url: base_url)
|
|
29
29
|
data, http = api.read(TEST_ENDPOINT, query: {format: :json}, ret: :both)
|
|
30
30
|
next unless data['remote_orchestrator_info']
|
|
31
31
|
url = http.uri.to_s
|
|
@@ -56,17 +56,11 @@ module Aspera
|
|
|
56
56
|
}
|
|
57
57
|
end
|
|
58
58
|
|
|
59
|
-
option :result, description: "Specify result value as: 'work_step:parameter'"
|
|
60
|
-
option :synchronous, description: 'Wait for completion', allowed: Type::BOOLEAN,
|
|
59
|
+
option :result, description: "Specify result value as: 'work_step:parameter'", deprecation: {last: '4.27.2', message: 'use keys `step` and `variable` of argument `execution` of `workflows start`'}
|
|
60
|
+
option :synchronous, description: 'Wait for completion', allowed: Type::BOOLEAN, deprecation: {last: '4.27.2', message: 'use key `synchronous` of argument `execution` of `workflows start`'}
|
|
61
61
|
option :ret_style, description: 'How return type is requested in api', allowed: %i[header arg ext], default: :arg
|
|
62
62
|
option :auth_style, description: 'Authentication type', allowed: %i[arg_pass head_basic apikey], default: :head_basic
|
|
63
63
|
|
|
64
|
-
def initialize(**_)
|
|
65
|
-
super
|
|
66
|
-
@api_orch = nil
|
|
67
|
-
options.parse_options!
|
|
68
|
-
end
|
|
69
|
-
|
|
70
64
|
# Call orchestrator API, it's a bit special
|
|
71
65
|
# @param endpoint [String] the endpoint to call
|
|
72
66
|
# @param ret_style [Symbol] the return style, :header, :arg, :ext(extension)
|
|
@@ -90,9 +84,9 @@ module Aspera
|
|
|
90
84
|
else Aspera.error_unexpected_value(ret_style)
|
|
91
85
|
end
|
|
92
86
|
end
|
|
93
|
-
add_query =
|
|
87
|
+
add_query = query_read_delete
|
|
94
88
|
call_args[:query].merge!(add_query.symbolize_keys) unless add_query.nil?
|
|
95
|
-
data, resp =
|
|
89
|
+
data, resp = api_orch.call(**call_args)
|
|
96
90
|
return resp if http
|
|
97
91
|
result = format.eql?('xml') ? XmlSimple.xml_in(resp.body, {'ForceArray' => xml_arrays}) : data
|
|
98
92
|
Log.dump(:data, result)
|
|
@@ -103,73 +97,73 @@ module Aspera
|
|
|
103
97
|
|
|
104
98
|
# --- DSL ---
|
|
105
99
|
|
|
106
|
-
command :health, description: 'Check Orchestrator API health'
|
|
107
|
-
command :info, description: '
|
|
108
|
-
command :processes, description: 'Show Orchestrator background process status',
|
|
109
|
-
command :monitors, description: 'Show Orchestrator monitor snapshot',
|
|
110
|
-
command :plugins, description: 'Show Orchestrator plugin versions',
|
|
111
|
-
command :workflows, description: 'Manage workflows'
|
|
112
|
-
command :workorders, description: 'Manage work orders'
|
|
113
|
-
command :workstep, description: 'Manage work steps'
|
|
100
|
+
command :health, description: 'Check Orchestrator API health'
|
|
101
|
+
command :info, description: 'Check that Orchestrator responds (ping)', action: ->(**) { Result::SingleObject.new(call_ao('remote_node_ping', format: 'xml', xml_arrays: false)) }
|
|
102
|
+
command :processes, description: 'Show Orchestrator background process status', action: ->(**) { Result::ObjectList.new(call_ao('processes_status', format: 'xml')['process']) }
|
|
103
|
+
command :monitors, description: 'Show Orchestrator monitor snapshot', action: ->(**) { Result::SingleObject.new(call_ao('monitor_snapshot')['monitor']) }
|
|
104
|
+
command :plugins, description: 'Show Orchestrator plugin versions', action: ->(**) { Result::ObjectList.new(call_ao('plugin_version')['Plugin']) }
|
|
105
|
+
command :workflows, description: 'Manage workflows'
|
|
106
|
+
command :workorders, description: 'Manage work orders'
|
|
107
|
+
command :workstep, description: 'Manage work steps'
|
|
114
108
|
|
|
115
109
|
commands_under :workflows do
|
|
116
|
-
command
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
arguments: [{name: :wf_id, type: :identifier}, {name: :external_parameters, type: Hash, mandatory: false, default: {}}]
|
|
110
|
+
command :list, description: 'List all workflows'
|
|
111
|
+
command :status, description: 'Show running status of a workflow',
|
|
112
|
+
arguments: [{name: :workflow_id, type: :identifier}],
|
|
113
|
+
action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao(workflow_id.eql?(SpecialValues::ALL) ? 'workflows_status' : "workflows_status/#{workflow_id}")['workflows']['workflow']) }
|
|
114
|
+
command :inputs, description: 'Show input specification of a workflow',
|
|
115
|
+
arguments: [{name: :workflow_id, type: :identifier}],
|
|
116
|
+
action: ->(workflow_id:, **) { Result::SingleObject.new(call_ao("workflow_inputs_spec/#{workflow_id}")['workflow_inputs_spec']) }
|
|
117
|
+
command :details, description: 'Show detailed running status of a workflow',
|
|
118
|
+
arguments: [{name: :workflow_id, type: :identifier}],
|
|
119
|
+
action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao("workflow_details/#{workflow_id}")['workflows']['workflow']['statuses']) }
|
|
120
|
+
command :start, description: 'Start a workflow: create a work order (sync or async)',
|
|
121
|
+
arguments: [
|
|
122
|
+
{name: :workflow_id, type: :identifier},
|
|
123
|
+
{name: :parameters, type: Hash, mandatory: false, default: {}},
|
|
124
|
+
{name: :execution, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowStart'}
|
|
125
|
+
]
|
|
133
126
|
command :export, description: 'Export a workflow',
|
|
134
|
-
arguments: [{name: :
|
|
135
|
-
action: ->(
|
|
136
|
-
command :workorders, description: '
|
|
137
|
-
arguments: [{name: :
|
|
138
|
-
action: ->(
|
|
139
|
-
command :outputs, description: '
|
|
140
|
-
arguments: [{name: :
|
|
141
|
-
action: ->(
|
|
127
|
+
arguments: [{name: :workflow_id, type: :identifier}],
|
|
128
|
+
action: ->(workflow_id:, **) { Result::Text.new(call_ao("export_workflow/#{workflow_id}", format: nil, http: true).body) }
|
|
129
|
+
command :workorders, description: 'List work orders of a workflow',
|
|
130
|
+
arguments: [{name: :workflow_id, type: :identifier}],
|
|
131
|
+
action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao("work_orders_list/#{workflow_id}")['work_orders']) }
|
|
132
|
+
command :outputs, description: 'Show output specification of a workflow',
|
|
133
|
+
arguments: [{name: :workflow_id, type: :identifier}],
|
|
134
|
+
action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao("workflow_outputs_spec/#{workflow_id}")['workflow_outputs_spec']['output']) }
|
|
142
135
|
end
|
|
143
136
|
|
|
144
137
|
commands_under :workorders do
|
|
145
|
-
command :status, description: '
|
|
146
|
-
arguments: [{name: :
|
|
147
|
-
action: ->(
|
|
138
|
+
command :status, description: 'Show status of a work order',
|
|
139
|
+
arguments: [{name: :workorder_id, type: :identifier}],
|
|
140
|
+
action: ->(workorder_id:, **) { Result::SingleObject.new(call_ao("work_order_status/#{workorder_id}")['work_order']) }
|
|
148
141
|
command :cancel, description: 'Cancel a work order',
|
|
149
|
-
arguments: [{name: :
|
|
150
|
-
action: ->(
|
|
142
|
+
arguments: [{name: :workorder_id, type: :identifier}],
|
|
143
|
+
action: ->(workorder_id:, **) { Result::SingleObject.new(call_ao("work_order_cancel/#{workorder_id}")['work_order']) }
|
|
151
144
|
command :reset, description: 'Reset a work order',
|
|
152
|
-
arguments: [{name: :
|
|
153
|
-
action: ->(
|
|
154
|
-
command :output, description: '
|
|
155
|
-
arguments: [{name: :
|
|
156
|
-
action: ->(
|
|
145
|
+
arguments: [{name: :workorder_id, type: :identifier}],
|
|
146
|
+
action: ->(workorder_id:, **) { Result::SingleObject.new(call_ao("work_order_reset/#{workorder_id}")['work_order']) }
|
|
147
|
+
command :output, description: 'Show output of a work order',
|
|
148
|
+
arguments: [{name: :workorder_id, type: :identifier}],
|
|
149
|
+
action: ->(workorder_id:, **) { Result::ObjectList.new(call_ao("work_order_output/#{workorder_id}", format: 'xml')['variable']) }
|
|
157
150
|
end
|
|
158
151
|
|
|
159
152
|
commands_under :workstep do
|
|
160
|
-
command :status, description: '
|
|
161
|
-
arguments: [{name: :
|
|
162
|
-
action: ->(
|
|
153
|
+
command :status, description: 'Show status of a work step',
|
|
154
|
+
arguments: [{name: :workstep_id, type: :identifier}],
|
|
155
|
+
action: ->(workstep_id:, **) { Result::SingleObject.new(call_ao("work_step_status/#{workstep_id}")) }
|
|
163
156
|
command :cancel, description: 'Cancel a work step',
|
|
164
|
-
arguments: [{name: :
|
|
165
|
-
action: ->(
|
|
157
|
+
arguments: [{name: :workstep_id, type: :identifier}],
|
|
158
|
+
action: ->(workstep_id:, **) { Result::SingleObject.new(call_ao("work_step_cancel/#{workstep_id}")) }
|
|
166
159
|
end
|
|
167
160
|
|
|
168
|
-
# ---
|
|
161
|
+
# --- API ---
|
|
169
162
|
|
|
170
|
-
#
|
|
171
|
-
# @return [
|
|
172
|
-
def
|
|
163
|
+
# Orchestrator REST API, built from CLI options on first use.
|
|
164
|
+
# @return [Rest::Client]
|
|
165
|
+
def api_orch
|
|
166
|
+
return @api_orch if @api_orch
|
|
173
167
|
auth_params =
|
|
174
168
|
case options.get_option(:auth_style, mandatory: true)
|
|
175
169
|
when :arg_pass
|
|
@@ -189,14 +183,13 @@ module Aspera
|
|
|
189
183
|
when :apikey
|
|
190
184
|
Aspera.error_not_implemented
|
|
191
185
|
end
|
|
192
|
-
@api_orch = Rest.new(
|
|
186
|
+
@api_orch = Rest::Client.new(
|
|
193
187
|
base_url: options.get_option(:url, mandatory: true),
|
|
194
188
|
auth: auth_params
|
|
195
189
|
)
|
|
196
|
-
{}
|
|
197
190
|
end
|
|
198
191
|
|
|
199
|
-
def action_health
|
|
192
|
+
def action_health(**)
|
|
200
193
|
nagios = Nagios.new
|
|
201
194
|
begin
|
|
202
195
|
info = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
|
|
@@ -209,25 +202,45 @@ module Aspera
|
|
|
209
202
|
end
|
|
210
203
|
|
|
211
204
|
# 2.1/2.2 Initiate a workorder (async / synchronous)
|
|
212
|
-
def
|
|
205
|
+
def action_workflows_list(**)
|
|
206
|
+
Result::ObjectList.new(
|
|
207
|
+
call_ao('workflows_list')['workflows']['workflow'],
|
|
208
|
+
fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification]
|
|
209
|
+
)
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Keys of argument `execution` of `workflows start`
|
|
213
|
+
WORKFLOW_START_KEYS = %w[synchronous step variable].freeze
|
|
214
|
+
private_constant :WORKFLOW_START_KEYS
|
|
215
|
+
|
|
216
|
+
def action_workflows_start(workflow_id:, parameters:, execution:, **)
|
|
217
|
+
execution = execution.transform_keys(&:to_s)
|
|
218
|
+
unknown = execution.keys - WORKFLOW_START_KEYS
|
|
219
|
+
Aspera.assert(unknown.empty?, type: Cli::BadArgument) { "Unknown keys in execution: #{unknown.join(', ')}, expected: #{WORKFLOW_START_KEYS.join(', ')}" }
|
|
220
|
+
# Deprecated options, used when key not provided
|
|
221
|
+
execution['synchronous'] = options.get_option(:synchronous) unless execution.key?('synchronous')
|
|
222
|
+
result_location = options.get_option(:result)
|
|
223
|
+
unless result_location.nil? || execution.key?('step') || execution.key?('variable')
|
|
224
|
+
fields = result_location.split(':')
|
|
225
|
+
Aspera.assert(fields.length == 2, type: Cli::BadArgument) { "Expects: work_step:result_name : #{result_location}" }
|
|
226
|
+
execution['step'], execution['variable'] = fields
|
|
227
|
+
end
|
|
213
228
|
call_params = {format: :json}
|
|
214
229
|
# get external parameters if any
|
|
215
|
-
|
|
230
|
+
parameters.each do |name, value|
|
|
216
231
|
call_params["external_parameters[#{name}]"] = value
|
|
217
232
|
end
|
|
218
|
-
|
|
219
|
-
call_params['synchronous'] = true if
|
|
233
|
+
Aspera.assert_type(execution['synchronous'], NilClass, *BoolValue::TYPES, type: Cli::BadArgument) { 'synchronous' }
|
|
234
|
+
call_params['synchronous'] = true if execution['synchronous'].eql?(true)
|
|
220
235
|
# expected result for synchro call ?
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
call_params['explicit_output_step'] = fields[0]
|
|
226
|
-
call_params['explicit_output_variable'] = fields[1]
|
|
236
|
+
if execution.key?('step') || execution.key?('variable')
|
|
237
|
+
Aspera.assert(execution['step'].is_a?(String) && execution['variable'].is_a?(String), type: Cli::BadArgument) { 'Both step and variable are required' }
|
|
238
|
+
call_params['explicit_output_step'] = execution['step']
|
|
239
|
+
call_params['explicit_output_variable'] = execution['variable']
|
|
227
240
|
# implicitly, call is synchronous
|
|
228
241
|
call_params['synchronous'] = true
|
|
229
242
|
end
|
|
230
|
-
result_data = call_ao("initiate/#{
|
|
243
|
+
result_data = call_ao("initiate/#{workflow_id}", args: call_params)
|
|
231
244
|
call_params['synchronous'] ? Result::Text.new(result_data) : Result::SingleObject.new(result_data)
|
|
232
245
|
end
|
|
233
246
|
end
|
|
@@ -18,6 +18,8 @@ require 'aspera/uri_reader'
|
|
|
18
18
|
require 'aspera/log'
|
|
19
19
|
require 'aspera/assert'
|
|
20
20
|
require 'securerandom'
|
|
21
|
+
require 'aspera/rainbow'
|
|
22
|
+
using Rainbow
|
|
21
23
|
|
|
22
24
|
module Aspera
|
|
23
25
|
module Cli
|
|
@@ -46,21 +48,19 @@ module Aspera
|
|
|
46
48
|
:LOG_LIMITER_SEC,
|
|
47
49
|
:REMOTE_ACCESS
|
|
48
50
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
# DSL option declarations (Pattern 1 and Pattern 2 - handler: Symbol resolved to self)
|
|
51
|
+
# DSL option declarations
|
|
52
52
|
option :skip_format, description: 'Skip this preview format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS
|
|
53
53
|
# TODO: use the same option as in `node` plugin
|
|
54
|
-
option :folder_reset_cache, description: 'Force detection of generated preview by refresh cache', allowed: %i[no header read],
|
|
55
|
-
option :skip_types, description: 'Skip generation for those types of files', allowed: Type::SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES
|
|
56
|
-
option :previews_folder, description: 'Preview folder in storage root',
|
|
57
|
-
option :skip_folders, description: 'List of folder to skip', allowed: Type::STRING_ARRAY
|
|
54
|
+
option :folder_reset_cache, description: 'Force detection of generated preview by refresh cache', allowed: %i[no header read], default: :no
|
|
55
|
+
option :skip_types, description: 'Skip generation for those types of files', allowed: Type::SYMBOL_ARRAY + Aspera::Preview::FileTypes::CONVERSION_TYPES
|
|
56
|
+
option :previews_folder, description: 'Preview folder in storage root', default: DEFAULT_PREVIEWS_FOLDER
|
|
57
|
+
option :skip_folders, description: 'List of folder to skip', allowed: Type::STRING_ARRAY
|
|
58
58
|
option :base, description: 'Basename of output for for test'
|
|
59
|
-
option :filter,
|
|
60
|
-
option :
|
|
61
|
-
option :overwrite, description: 'When to overwrite result file',
|
|
59
|
+
option :filter, description: 'File name filter: String (glob), Regexp, or Proc', allowed: [String, Regexp, Proc, NilClass]
|
|
60
|
+
option :detect_mime, description: 'Detect Mime type by analyzing file', allowed: Type::BOOLEAN, default: false
|
|
61
|
+
option :overwrite, description: 'When to overwrite result file', allowed: %i[always never mtime], default: :mtime
|
|
62
62
|
option :root_url, description: "How to read and write files on storage (#{REMOTE_ACCESS}, or #{UriReader.file_url('<folder>')})", default: REMOTE_ACCESS
|
|
63
|
-
# Generator-specific options (
|
|
63
|
+
# Generator-specific options (bound to @gen_options in initialize)
|
|
64
64
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
|
65
65
|
values = if opt.key?(:values)
|
|
66
66
|
opt[:values]
|
|
@@ -79,12 +79,20 @@ module Aspera
|
|
|
79
79
|
# Optional callback used to filter entries before generation.
|
|
80
80
|
@filter_block = nil
|
|
81
81
|
@access_remote = true
|
|
82
|
-
# Bind generator-specific options to @gen_options
|
|
82
|
+
# Bind generator-specific options to @gen_options
|
|
83
83
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
|
84
|
-
options.
|
|
84
|
+
options.on_set(opt[:name], @gen_options.method(:"#{opt[:name]}="))
|
|
85
85
|
end
|
|
86
|
-
|
|
86
|
+
# Values set through `on_set` callbacks are used below
|
|
87
87
|
options.parse_options!
|
|
88
|
+
@option_skip_types = options.get_option(:skip_types)
|
|
89
|
+
@option_previews_folder = options.get_option(:previews_folder)
|
|
90
|
+
@option_folder_reset_cache = options.get_option(:folder_reset_cache)
|
|
91
|
+
# Modified during scan
|
|
92
|
+
@option_skip_folders = options.get_option(:skip_folders).dup
|
|
93
|
+
@option_overwrite = options.get_option(:overwrite)
|
|
94
|
+
# Tell which tool we will use for office docs
|
|
95
|
+
Aspera::Preview::Utils.office_tool = @gen_options.office_conversion
|
|
88
96
|
Api::Node.api_options[:cache] = !@option_folder_reset_cache.eql?(:header)
|
|
89
97
|
# Start from the full supported format list, then remove any skipped format.
|
|
90
98
|
@preview_formats_to_generate = Aspera::Preview::Generator::PREVIEW_FORMATS.clone
|
|
@@ -108,7 +116,7 @@ module Aspera
|
|
|
108
116
|
events_filter['iteration_token'] = iteration_persistency.data.first unless iteration_persistency.nil?
|
|
109
117
|
begin
|
|
110
118
|
events = @api_node.read('events', events_filter)
|
|
111
|
-
rescue
|
|
119
|
+
rescue Rest::CallError => e
|
|
112
120
|
if e.message.include?('Invalid iteration_token')
|
|
113
121
|
Log.log.warn { "Retrying without iteration token: #{e}" }
|
|
114
122
|
events_filter.delete('iteration_token')
|
|
@@ -360,7 +368,7 @@ module Aspera
|
|
|
360
368
|
Log.log.debug { "processing entry #{entry_path_with_slash}" } if @periodic.trigger?
|
|
361
369
|
entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
|
|
362
370
|
if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
|
|
363
|
-
Log.log.debug { "#{entry['path']} folder (skip start)".
|
|
371
|
+
Log.log.debug { "#{entry['path']} folder (skip start)".bg(:red) }
|
|
364
372
|
next
|
|
365
373
|
end
|
|
366
374
|
Log.log.debug { "item:#{entry}" }
|
|
@@ -376,7 +384,7 @@ module Aspera
|
|
|
376
384
|
Log.log.debug('Ignoring link.')
|
|
377
385
|
when 'folder'
|
|
378
386
|
if @option_skip_folders.include?(entry['path'])
|
|
379
|
-
Log.log.debug { "#{entry['path']} folder (skip list)".
|
|
387
|
+
Log.log.debug { "#{entry['path']} folder (skip list)".bg(:red) }
|
|
380
388
|
else
|
|
381
389
|
Log.log.debug { "#{entry['path']} folder".green }
|
|
382
390
|
# get folder content
|
|
@@ -402,14 +410,16 @@ module Aspera
|
|
|
402
410
|
|
|
403
411
|
# scan, events and trevents connect to the Node API (setup: :setup_node_api).
|
|
404
412
|
# `check`, `test` and `show` work without Node API authentication.
|
|
405
|
-
command :scan, description: 'Scan a file or folder and generate previews',
|
|
406
|
-
|
|
407
|
-
command :
|
|
408
|
-
command :
|
|
409
|
-
command :
|
|
413
|
+
command :scan, description: 'Scan a file or folder and generate previews', setup: :setup_node_api,
|
|
414
|
+
arguments: [{name: :path, description: 'File or folder from which to start scanning', mandatory: false, default: '/'}]
|
|
415
|
+
command :events, description: 'Process file events and generate previews', setup: :setup_node_api, action: ->(**) { run_event_loop(:events) }
|
|
416
|
+
command :trevents, description: 'Process transfer events and generate previews', setup: :setup_node_api, action: ->(**) { run_event_loop(:trevents) }
|
|
417
|
+
command :check, description: 'Check required tools are installed', action: ->(**) { tools_cleanup { Result::Status.new('Tools validated') } }
|
|
418
|
+
command :show, description: 'Generate and display preview of a source file',
|
|
419
|
+
arguments: [{name: :source_file, description: 'Local file to preview'}]
|
|
410
420
|
command :test, description: 'Test preview generation for a source file',
|
|
411
421
|
arguments: [
|
|
412
|
-
{name: :source_file,
|
|
422
|
+
{name: :source_file, description: 'Local file to preview'},
|
|
413
423
|
{name: :format, description: 'Preview output format', allowed: Aspera::Preview::Generator::PREVIEW_FORMATS, mandatory: false, default: :png}
|
|
414
424
|
]
|
|
415
425
|
|
|
@@ -461,71 +471,47 @@ module Aspera
|
|
|
461
471
|
# --- handlers ---
|
|
462
472
|
|
|
463
473
|
def action_scan(path:, **)
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
cleanup_tmp_folder
|
|
477
|
-
end
|
|
478
|
-
|
|
479
|
-
def action_events(**)
|
|
480
|
-
check_tools_and_mimemagic
|
|
481
|
-
run_event_loop(:events, filter: options.get_option(:filter))
|
|
482
|
-
ensure
|
|
483
|
-
cleanup_tmp_folder
|
|
484
|
-
end
|
|
485
|
-
|
|
486
|
-
def action_trevents(**)
|
|
487
|
-
check_tools_and_mimemagic
|
|
488
|
-
run_event_loop(:trevents, filter: options.get_option(:filter))
|
|
489
|
-
ensure
|
|
490
|
-
cleanup_tmp_folder
|
|
474
|
+
tools_cleanup do
|
|
475
|
+
apifid =
|
|
476
|
+
if (selector = Parser.percent_selector(path))
|
|
477
|
+
Aspera.assert_values(selector[:field], ['id'], type: BadArgument) { 'file id' }
|
|
478
|
+
Api::NodeFileId.new(@api_node, selector[:value].to_s.empty? ? @access_key_self['root_file_id'] : selector[:value])
|
|
479
|
+
else
|
|
480
|
+
@api_node.resolve_api_fid(@access_key_self['root_file_id'], path)
|
|
481
|
+
end
|
|
482
|
+
@filter_block = Base.file_matcher(options.get_option(:filter))
|
|
483
|
+
scan_folder_files(@api_node.read("files/#{apifid.file_id}"))
|
|
484
|
+
Result::Status.new('scan finished')
|
|
485
|
+
end
|
|
491
486
|
end
|
|
492
487
|
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
Result::Status.new('Tools validated')
|
|
496
|
-
ensure
|
|
497
|
-
cleanup_tmp_folder
|
|
498
|
-
end
|
|
488
|
+
# Status message for generation
|
|
489
|
+
MSG_GEN_TITLE = 'Generated: '
|
|
499
490
|
|
|
500
491
|
def action_test(source_file:, format:, **)
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
cleanup_tmp_folder
|
|
492
|
+
tools_cleanup do
|
|
493
|
+
generated_file_path = preview_filename(format, options.get_option(:base))
|
|
494
|
+
Aspera::Preview::Generator.new(source_file, generated_file_path, @gen_options, @tmp_folder).generate
|
|
495
|
+
Result::Status.new("#{MSG_GEN_TITLE}#{generated_file_path}")
|
|
496
|
+
end
|
|
507
497
|
end
|
|
508
498
|
|
|
509
499
|
def action_show(source_file:, **)
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
formatter.display_status("generated: #{generated_file_path}")
|
|
500
|
+
status = action_test(source_file: source_file, format: :png)
|
|
501
|
+
formatter.display_status(status.data)
|
|
502
|
+
generated_file_path = status.data.delete_prefix(MSG_GEN_TITLE)
|
|
514
503
|
Result::Image.new(UriReader.file_url(generated_file_path))
|
|
515
|
-
ensure
|
|
516
|
-
cleanup_tmp_folder
|
|
517
504
|
end
|
|
518
505
|
|
|
519
506
|
private
|
|
520
507
|
|
|
521
|
-
# Check tools and set
|
|
522
|
-
def check_tools_and_mimemagic
|
|
523
|
-
Aspera::Preview::FileTypes.instance.use_mimemagic = options.get_option(:mimemagic, mandatory: true)
|
|
524
|
-
Aspera::Preview::Utils.check_tools(@option_skip_types)
|
|
525
|
-
end
|
|
526
|
-
|
|
508
|
+
# Check tools and set option flag.
|
|
527
509
|
# Clean up the temporary folder after each command.
|
|
528
|
-
def
|
|
510
|
+
def tools_cleanup
|
|
511
|
+
Aspera::Preview::FileTypes.instance.detect_mime = options.get_option(:detect_mime, mandatory: true)
|
|
512
|
+
Aspera::Preview::Utils.check_tools(@option_skip_types)
|
|
513
|
+
yield
|
|
514
|
+
ensure
|
|
529
515
|
Log.log.debug { "cleaning up temp folder #{@tmp_folder}" }
|
|
530
516
|
FileUtils.rm_rf(@tmp_folder)
|
|
531
517
|
end
|
|
@@ -533,23 +519,26 @@ module Aspera
|
|
|
533
519
|
# Shared event-loop body for :events and :trevents.
|
|
534
520
|
# @param command [:events, :trevents]
|
|
535
521
|
# @param filter [String, Regexp, Proc, nil] optional filter expression
|
|
536
|
-
def run_event_loop(command
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
iteration_persistency =
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
522
|
+
def run_event_loop(command)
|
|
523
|
+
tools_cleanup do
|
|
524
|
+
filter = options.get_option(:filter)
|
|
525
|
+
@filter_block = Base.file_matcher(filter)
|
|
526
|
+
iteration_persistency = nil
|
|
527
|
+
if options.get_option(:once_only, mandatory: true)
|
|
528
|
+
iteration_persistency = PersistencyActionOnce.new(
|
|
529
|
+
manager: persistency,
|
|
530
|
+
data: [],
|
|
531
|
+
id: IdGenerator.from_list(
|
|
532
|
+
'preview_iteration',
|
|
533
|
+
command.to_s,
|
|
534
|
+
options.get_option(:url, mandatory: true),
|
|
535
|
+
options.get_option(:username, mandatory: true)
|
|
536
|
+
)
|
|
548
537
|
)
|
|
549
|
-
|
|
538
|
+
end
|
|
539
|
+
send(:"process_#{command}", iteration_persistency)
|
|
540
|
+
Result::Status.new("#{command} finished")
|
|
550
541
|
end
|
|
551
|
-
send(:"process_#{command}", iteration_persistency)
|
|
552
|
-
Result::Status.new("#{command} finished")
|
|
553
542
|
end
|
|
554
543
|
end
|
|
555
544
|
end
|