aspera-cli 4.27.5 → 4.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +160 -4
- data/CONTRIBUTING.md +4 -2
- data/README.md +1 -1
- data/bin/asession +1 -1
- data/docs/README.md +1086 -331
- data/docs/test-mcp-with-ai.md +5 -3
- data/lib/aspera/agent/direct.rb +15 -4
- data/lib/aspera/agent/factory.rb +2 -1
- data/lib/aspera/agent/node.rb +1 -1
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/faspex.rb +1 -1
- data/lib/aspera/api/node.rb +6 -2
- data/lib/aspera/ascp/installation.rb +47 -11
- data/lib/aspera/ascp/management.rb +89 -43
- data/lib/aspera/cli/ascp_actions.rb +5 -19
- data/lib/aspera/cli/bootstrapper.rb +1 -2
- data/lib/aspera/cli/command_registry.rb +43 -3
- data/lib/aspera/cli/command_spec.rb +3 -0
- data/lib/aspera/cli/completion/ascli.bash +39 -5
- data/lib/aspera/cli/completion/ascli.fish +16 -8
- data/lib/aspera/cli/completion/ascli.ps1 +41 -0
- data/lib/aspera/cli/completion/ascli.zsh +20 -4
- data/lib/aspera/cli/completion_actions.rb +360 -0
- data/lib/aspera/cli/error.rb +42 -0
- data/lib/aspera/cli/extended_value.rb +30 -5
- data/lib/aspera/cli/formatter.rb +118 -56
- data/lib/aspera/cli/hints.rb +0 -1
- data/lib/aspera/cli/http.rb +6 -5
- data/lib/aspera/cli/info.rb +10 -4
- data/lib/aspera/cli/mailer.rb +1 -1
- data/lib/aspera/cli/mcp/http_servlet.rb +156 -0
- data/lib/aspera/cli/mcp/secrets.rb +143 -0
- data/lib/aspera/cli/mcp/tool.rb +231 -0
- data/lib/aspera/cli/option_value.rb +9 -1
- data/lib/aspera/cli/options.schema.yaml +390 -39
- data/lib/aspera/cli/parser.rb +14 -5
- data/lib/aspera/cli/plugins/aoc.rb +8 -11
- data/lib/aspera/cli/plugins/base.rb +36 -15
- data/lib/aspera/cli/plugins/config.rb +15 -44
- data/lib/aspera/cli/plugins/console.rb +3 -1
- data/lib/aspera/cli/plugins/factory.rb +1 -1
- data/lib/aspera/cli/plugins/faspex5.rb +9 -19
- data/lib/aspera/cli/plugins/faspio.rb +36 -5
- data/lib/aspera/cli/plugins/httpgw.rb +1 -0
- data/lib/aspera/cli/plugins/mcp.rb +177 -118
- data/lib/aspera/cli/plugins/node.rb +14 -8
- data/lib/aspera/cli/plugins/orchestrator.rb +351 -38
- data/lib/aspera/cli/plugins/server.rb +3 -1
- data/lib/aspera/cli/plugins/shares.rb +205 -35
- data/lib/aspera/cli/preset_actions.rb +15 -7
- data/lib/aspera/cli/preset_manager.rb +24 -2
- data/lib/aspera/cli/prompt.rb +3 -1
- data/lib/aspera/cli/result.rb +17 -4
- data/lib/aspera/cli/runner.rb +133 -43
- data/lib/aspera/cli/security.rb +134 -0
- data/lib/aspera/cli/table_fitter.rb +116 -0
- data/lib/aspera/cli/terminal_formatter.rb +23 -34
- data/lib/aspera/cli/transfer_agent.rb +4 -1
- data/lib/aspera/cli/vault_manager.rb +6 -5
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/web_secret.html.erb +61 -0
- data/lib/aspera/cli/web_secret.rb +220 -0
- data/lib/aspera/cli/wizard.rb +19 -9
- data/lib/aspera/command_line_builder.rb +26 -0
- data/lib/aspera/data_repository.rb +38 -24
- data/lib/aspera/duration.rb +42 -0
- data/lib/aspera/environment.rb +88 -24
- data/lib/aspera/exec_spec.rb +6 -5
- data/lib/aspera/faspex_gw.rb +98 -32
- data/lib/aspera/faspex_postproc.rb +187 -44
- data/lib/aspera/formatter_interface.rb +0 -1
- data/lib/aspera/id_generator.rb +2 -1
- data/lib/aspera/json_rpc/client.rb +51 -32
- data/lib/aspera/json_rpc/version.rb +1 -0
- data/lib/aspera/keychain/base.rb +5 -0
- data/lib/aspera/keychain/encrypted_hash.rb +6 -2
- data/lib/aspera/keychain/factory.rb +2 -3
- data/lib/aspera/keychain/hashicorp_vault.rb +7 -4
- data/lib/aspera/keychain/macos_security.rb +119 -13
- data/lib/aspera/keychain/one_password_api.rb +8 -3
- data/lib/aspera/keychain/one_password_base.rb +14 -9
- data/lib/aspera/keychain/one_password_cli.rb +4 -2
- data/lib/aspera/log.rb +3 -6
- data/lib/aspera/markdown.rb +152 -115
- data/lib/aspera/mime.rb +6 -0
- data/lib/aspera/nagios.rb +3 -1
- data/lib/aspera/node_emulator.rb +98 -101
- data/lib/aspera/oauth/{base.rb → client/base.rb} +5 -4
- data/lib/aspera/oauth/{boot.rb → client/boot.rb} +3 -2
- data/lib/aspera/oauth/{factory.rb → client/factory.rb} +7 -0
- data/lib/aspera/oauth/{generic.rb → client/generic.rb} +1 -1
- data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +1 -1
- data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +1 -1
- data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +1 -1
- data/lib/aspera/oauth/{web.rb → client/web.rb} +2 -2
- data/lib/aspera/oauth/server/authorize.html.erb +34 -0
- data/lib/aspera/oauth/server/pkce.rb +426 -0
- data/lib/aspera/orchestrator_workflow_checker.rb +561 -0
- data/lib/aspera/preview/terminal.rb +3 -2
- data/lib/aspera/rainbow.rb +27 -1
- data/lib/aspera/rest/aspera_errors.rb +51 -59
- data/lib/aspera/rest/client.rb +21 -14
- data/lib/aspera/rest/error_analyzer.rb +9 -0
- data/lib/aspera/rest/list.rb +6 -6
- data/lib/aspera/rest/util.rb +7 -1
- data/lib/aspera/rest.rb +0 -1
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -2
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4001 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +35 -35
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
- data/lib/aspera/schema/documentation.rb +50 -78
- data/lib/aspera/schema/reader.rb +39 -11
- data/lib/aspera/schema/registry.rb +8 -2
- data/lib/aspera/secret_hider.rb +94 -15
- data/lib/aspera/secret_strength.rb +114 -0
- data/lib/aspera/servlet_helpers.rb +111 -0
- data/lib/aspera/sync/args.schema.yaml +3 -1
- data/lib/aspera/sync/conf.schema.yaml +16 -6
- data/lib/aspera/sync/operations.rb +21 -1
- data/lib/aspera/temp_file_manager.rb +79 -20
- data/lib/aspera/transfer/parameters.rb +5 -0
- data/lib/aspera/transfer/spec.rb +52 -12
- data/lib/aspera/transfer/spec.schema.yaml +8 -5
- data/lib/aspera/web_auth/index.html.erb +16 -0
- data/lib/aspera/web_auth/server.rb +18 -10
- data/lib/aspera/web_page.html.erb +262 -0
- data/lib/aspera/web_page.rb +61 -0
- data/lib/aspera/web_server_simple.rb +76 -10
- data.tar.gz.sig +0 -0
- metadata +47 -32
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/mcp_tool.rb +0 -189
- data/lib/aspera/oauth.rb +0 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
- data/lib/aspera/web_auth/index.html +0 -147
|
@@ -5,6 +5,8 @@ require 'aspera/cli/special_values'
|
|
|
5
5
|
require 'aspera/nagios'
|
|
6
6
|
require 'aspera/log'
|
|
7
7
|
require 'aspera/assert'
|
|
8
|
+
require 'aspera/transfer/spec'
|
|
9
|
+
require 'aspera/schema/registry'
|
|
8
10
|
require 'xmlsimple'
|
|
9
11
|
|
|
10
12
|
module Aspera
|
|
@@ -14,7 +16,35 @@ module Aspera
|
|
|
14
16
|
class Orchestrator < BasicAuth
|
|
15
17
|
STANDARD_PATH = '/aspera/orchestrator'
|
|
16
18
|
TEST_ENDPOINT = 'api/remote_node_ping'
|
|
17
|
-
|
|
19
|
+
# Raw Markdown of the workflow authoring documents
|
|
20
|
+
DOC_BASE_URL = 'https://raw.githubusercontent.com/laurent-martin/aspera-orchestrator-plugins-doc/main/docs'
|
|
21
|
+
# Documents of command `workflows development`: name => file
|
|
22
|
+
DOC_FILES = {
|
|
23
|
+
authoring: 'workflow-authoring-guide.md',
|
|
24
|
+
templates: 'action-template-reference.md',
|
|
25
|
+
manual: 'Orchestrator_Plugin_Manual.md'
|
|
26
|
+
}.freeze
|
|
27
|
+
# Value of argument `section` of command `workflows development` for the table of contents
|
|
28
|
+
DOC_TOC = 'toc'
|
|
29
|
+
# Default maximum level of headings in the table of contents: guide sections, one entry per plugin
|
|
30
|
+
DOC_TOC_LEVELS = {
|
|
31
|
+
authoring: 3,
|
|
32
|
+
templates: 2,
|
|
33
|
+
manual: 3
|
|
34
|
+
}.freeze
|
|
35
|
+
# Sections of a plugin specification, listed by command `plugins spec`
|
|
36
|
+
SPEC_SECTIONS = %i[attributes inputs outputs].freeze
|
|
37
|
+
# Fields of the list of plugins of command `plugins spec`
|
|
38
|
+
SPEC_LIST_FIELDS = %w[name category version display_name].freeze
|
|
39
|
+
# Text value that is a YAML document (Ruby objects serialized by Orchestrator)
|
|
40
|
+
YAML_DUMP = /\A---\s/
|
|
41
|
+
# Types of monitors of `list_monitors` (`monitor_type`) and of `monitor_snapshot` (path)
|
|
42
|
+
MONITOR_TYPES = %i[workflow workorder folder node script].freeze
|
|
43
|
+
MONITOR_SNAPSHOT_TYPES = %i[workflows workorders folders nodes scripts].freeze
|
|
44
|
+
# Id segment of the path, for endpoints routed only as `api/<endpoint>/<id>` that do not use it
|
|
45
|
+
ANY_ID = '0'
|
|
46
|
+
private_constant :STANDARD_PATH, :TEST_ENDPOINT, :DOC_BASE_URL, :DOC_FILES, :DOC_TOC, :DOC_TOC_LEVELS, :SPEC_SECTIONS, :SPEC_LIST_FIELDS, :YAML_DUMP,
|
|
47
|
+
:MONITOR_TYPES, :MONITOR_SNAPSHOT_TYPES, :ANY_ID
|
|
18
48
|
|
|
19
49
|
class << self
|
|
20
50
|
# @return [Hash,NilClass]
|
|
@@ -46,10 +76,12 @@ module Aspera
|
|
|
46
76
|
# @param app_url [String] Tested URL
|
|
47
77
|
# @return [Hash] :preset_value, :test_args
|
|
48
78
|
def wizard(wizard, app_url)
|
|
79
|
+
username = options.get_option(:username, mandatory: true)
|
|
49
80
|
return {
|
|
81
|
+
preset_ids: [username],
|
|
50
82
|
preset_value: {
|
|
51
83
|
url: app_url,
|
|
52
|
-
username:
|
|
84
|
+
username: username,
|
|
53
85
|
password: options.get_option(:password, mandatory: true)
|
|
54
86
|
},
|
|
55
87
|
test_args: 'workflows list'
|
|
@@ -61,55 +93,108 @@ module Aspera
|
|
|
61
93
|
option :ret_style, description: 'Method to specify the expected response format ("Accept")', allowed: %i[header query path], default: :query
|
|
62
94
|
option :result, description: "Specify result value as: 'work_step:parameter'", deprecation: {last: '4.27.2', message: 'use keys `step` and `variable` of argument `execution` of `workflows start`'}
|
|
63
95
|
option :synchronous, description: 'Wait for completion', allowed: Type::BOOLEAN, deprecation: {last: '4.27.2', message: 'use key `synchronous` of argument `execution` of `workflows start`'}
|
|
96
|
+
option :plugin_specs, description: 'URL or path of the plugin specifications (YAML) used by `plugins spec`', default: "#{DOC_BASE_URL}/plugins.yml"
|
|
64
97
|
|
|
65
98
|
# Call orchestrator API (handles ret_style negotiation and XML parsing)
|
|
66
|
-
# @param endpoint
|
|
67
|
-
# @param body
|
|
68
|
-
# @param
|
|
69
|
-
# @param
|
|
70
|
-
# @param
|
|
71
|
-
# @param
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
99
|
+
# @param endpoint [String] the endpoint to call
|
|
100
|
+
# @param body [Hash, Array] the body to pass; also implies POST
|
|
101
|
+
# @param content_type [String] the body type (Mime::JSON or Mime::MULTIPART)
|
|
102
|
+
# @param accept [String] the response type to request (Mime::JSON or Mime::XML), or nil
|
|
103
|
+
# @param query [Hash] the arguments to pass as query parameters
|
|
104
|
+
# @param xml_opts [Hash] options of `XmlSimple.xml_in` to parse an XML response
|
|
105
|
+
# @param json_opts [Hash] options of `JSON.parse` to parse a JSON response
|
|
106
|
+
# @param http [Boolean] if true, returns the HttpResponse, else
|
|
107
|
+
def call_ao(endpoint, body: nil, content_type: Mime::JSON, accept: Mime::JSON, query: nil, xml_opts: {}, json_opts: {}, http: false)
|
|
108
|
+
call_args = {operation: body.nil? ? 'GET' : 'POST', subpath: "api/#{endpoint}", ret: :both, query: {}, json_opts: json_opts}
|
|
109
|
+
call_args.merge!(body: body, content_type: content_type) unless body.nil? # rubocop:disable Performance/RedundantMerge
|
|
75
110
|
ret_style = options.get_option(:ret_style, mandatory: true)
|
|
76
111
|
call_args[:query].merge!(query) unless query.nil?
|
|
77
|
-
unless
|
|
112
|
+
unless accept.nil?
|
|
113
|
+
# 'json' or 'xml'
|
|
114
|
+
short_type = accept.split('/').last
|
|
78
115
|
case ret_style
|
|
79
116
|
when :header
|
|
80
|
-
call_args[:headers] = {'Accept' =>
|
|
117
|
+
call_args[:headers] = {'Accept' => accept}
|
|
81
118
|
when :query
|
|
82
|
-
call_args[:query][:format] =
|
|
119
|
+
call_args[:query][:format] = short_type
|
|
83
120
|
when :path
|
|
84
|
-
call_args[:subpath] = "#{call_args[:subpath]}.#{
|
|
121
|
+
call_args[:subpath] = "#{call_args[:subpath]}.#{short_type}"
|
|
85
122
|
else Aspera.error_unexpected_value(ret_style) { 'ret_style' }
|
|
86
123
|
end
|
|
87
124
|
end
|
|
88
|
-
add_query = query_read_delete
|
|
125
|
+
add_query = query_read_delete(schema: api_query_schema(endpoint, call_args[:operation]))
|
|
89
126
|
call_args[:query].merge!(add_query.symbolize_keys) unless add_query.nil?
|
|
90
127
|
data, resp = api_orch.call(**call_args)
|
|
91
128
|
return resp if http
|
|
92
|
-
result =
|
|
129
|
+
result = accept.eql?(Mime::XML) ? XmlSimple.xml_in(resp.body, xml_opts) : data
|
|
93
130
|
Log.dump(:data, result)
|
|
94
131
|
return result
|
|
95
132
|
end
|
|
96
133
|
|
|
97
|
-
|
|
134
|
+
# Schema of the query parameters of an endpoint, in the OpenAPI description of Orchestrator (for `--query=help`)
|
|
135
|
+
# @param endpoint [String] Endpoint, with an id or not, e.g. `journal_files/journal`
|
|
136
|
+
# @param operation [String] HTTP method
|
|
137
|
+
# @return [String, nil] Schema path, or `nil` if the endpoint is not described
|
|
138
|
+
def api_query_schema(endpoint, operation)
|
|
139
|
+
name, id = endpoint.split('/', 2)
|
|
140
|
+
paths = Schema::Registry.instance.reader(Schema::Registry::ORCHESTRATOR)['paths']
|
|
141
|
+
# Orchestrator routes both `api/<name>` and `api/<name>/<id>`: prefer the form used
|
|
142
|
+
candidates = ["api/#{name}", "api/#{name}/{id}"]
|
|
143
|
+
candidates.reverse! unless id.nil?
|
|
144
|
+
path = candidates.find { |candidate| paths.dig("/#{candidate}", operation.downcase) }
|
|
145
|
+
path && Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, path, method: operation.downcase)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
private :call_ao, :api_query_schema
|
|
98
149
|
|
|
99
150
|
# --- DSL ---
|
|
100
151
|
|
|
101
152
|
command :health, description: 'Check Orchestrator API health'
|
|
102
|
-
command :info, description: 'Check that Orchestrator responds (ping)', action: ->(**) { Result::SingleObject.new(call_ao('remote_node_ping',
|
|
103
|
-
command :processes, description: 'Show Orchestrator background process status', action: ->(**) { Result::ObjectList.new(call_ao('processes_status',
|
|
104
|
-
command :monitors, description: '
|
|
153
|
+
command :info, description: 'Check that Orchestrator responds (ping)', action: ->(**) { Result::SingleObject.new(call_ao('remote_node_ping', accept: Mime::XML, xml_opts: {'ForceArray' => false})) }
|
|
154
|
+
command :processes, description: 'Show Orchestrator background process status', action: ->(**) { Result::ObjectList.new(call_ao('processes_status', accept: Mime::XML, xml_opts: {'ForceArray' => %w[node process]})['node'].flat_map { |n| n['process'] }) }
|
|
155
|
+
command :monitors, description: 'Monitors of workflows, work orders, folders, remote nodes and scripts'
|
|
156
|
+
command :queues, description: 'Managed queues'
|
|
157
|
+
command :pools, description: 'Resource pools'
|
|
158
|
+
command :journal, description: 'Journal books'
|
|
105
159
|
command :workorders, description: 'Manage work orders'
|
|
106
160
|
command :workstep, description: 'Manage work steps'
|
|
107
161
|
|
|
162
|
+
commands_under :monitors do
|
|
163
|
+
command :snapshot, description: 'Show status of monitors, of all types or of one type',
|
|
164
|
+
arguments: [{name: :type, type: Symbol, mandatory: false, allowed: MONITOR_SNAPSHOT_TYPES}],
|
|
165
|
+
action: ->(type: nil, **) { Result::SingleObject.new(call_ao(type.nil? ? 'monitor_snapshot' : "monitor_snapshot/#{type}")['monitor']) }
|
|
166
|
+
command :list, description: 'List monitors of a type: remote node monitors give the names of remote nodes',
|
|
167
|
+
arguments: [{name: :type, type: Symbol, allowed: MONITOR_TYPES}]
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
commands_under :queues do
|
|
171
|
+
command :list, description: 'List managed queues',
|
|
172
|
+
action: ->(**) { Result::ObjectList.new(call_ao('queues_list').dig('Queues', 'queue') || []) }
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
commands_under :pools do
|
|
176
|
+
command :status, description: 'Show resources of a resource pool and their status',
|
|
177
|
+
arguments: [{name: :pool_name}],
|
|
178
|
+
# Orchestrator repeats key `id` in each resource
|
|
179
|
+
action: ->(pool_name:, **) { Result::ObjectList.new(call_ao("pool_status/#{ANY_ID}", query: {name: pool_name}, json_opts: {allow_duplicate_key: true})['ManagedResource'] || []) }
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
commands_under :journal do
|
|
183
|
+
command :files, description: 'List files recorded in a journal book (filters with option query)',
|
|
184
|
+
arguments: [{name: :book}],
|
|
185
|
+
query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/journal_files/{id}')
|
|
186
|
+
end
|
|
187
|
+
|
|
108
188
|
commands_under :plugins do
|
|
109
189
|
command :list, description: 'Show Orchestrator plugin versions', action: ->(**) { Result::ObjectList.new(call_ao('plugin_version')['Plugin']) }
|
|
110
190
|
command :reload_set, description: 'Reload a set of plugins',
|
|
111
191
|
arguments: [{name: :plugin_set, type: Hash, schema: 'opts:components.schemas.OrchestratorReloadPluginSet'}],
|
|
112
192
|
action: ->(plugin_set:, **) { Result::SingleObject.new(call_ao('reload_plugin_set', body: plugin_set.transform_keys(&:to_sym))) }
|
|
193
|
+
command :spec, description: "Show specifications of action plugins, from GitHub (no access to Orchestrator): list of plugins, one plugin, or its #{SPEC_SECTIONS.join(', ')}",
|
|
194
|
+
arguments: [
|
|
195
|
+
{name: :plugin, mandatory: false},
|
|
196
|
+
{name: :section, type: Symbol, mandatory: false, allowed: SPEC_SECTIONS}
|
|
197
|
+
]
|
|
113
198
|
end
|
|
114
199
|
|
|
115
200
|
commands_under :workflows do
|
|
@@ -129,21 +214,37 @@ module Aspera
|
|
|
129
214
|
{name: :parameters, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorInitiateParameters'},
|
|
130
215
|
{name: :execution, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowStart'}
|
|
131
216
|
]
|
|
132
|
-
command :import, description: 'Import a workflow',
|
|
133
|
-
arguments: [
|
|
134
|
-
|
|
217
|
+
command :import, description: 'Import a workflow from a file created by `workflows export`, local problems are reported as warnings (see `check_import`)',
|
|
218
|
+
arguments: [
|
|
219
|
+
{name: :file_path},
|
|
220
|
+
{name: :import, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowImport'}
|
|
221
|
+
]
|
|
222
|
+
command :check_import, description: 'Check a workflow file (no access to Orchestrator): rules that Orchestrator does not check on import, but that make the workflow fail to publish or to run',
|
|
223
|
+
arguments: [{name: :file_path}]
|
|
224
|
+
command :find_constraints, description: 'List the conflicts and missing plugins of a workflow file present on the Orchestrator host, before `import_with_constraints`',
|
|
225
|
+
arguments: [
|
|
226
|
+
{name: :filename},
|
|
227
|
+
{name: :add_as_revision, type: :identifier, mandatory: false}
|
|
228
|
+
]
|
|
135
229
|
command :publish, description: 'Publish a workflow',
|
|
136
230
|
arguments: [{name: :workflow_id, type: :identifier}],
|
|
137
|
-
action: ->(workflow_id:, **) { Result::
|
|
138
|
-
command :import_with_constraints, description: 'Import a workflow with
|
|
139
|
-
arguments: [{name: :payload, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWithConstraints'}]
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
231
|
+
action: ->(workflow_id:, **) { Result::Status.new(call_ao('publish_workflow', body: {id: workflow_id}).eql?(true) ? 'published' : 'not published') }
|
|
232
|
+
command :import_with_constraints, description: 'Import a workflow file present on the Orchestrator host, with resolution of conflicts',
|
|
233
|
+
arguments: [{name: :payload, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWithConstraints'}]
|
|
234
|
+
command :export, description: 'Export a workflow: display it, or save it with its dependencies',
|
|
235
|
+
arguments: [
|
|
236
|
+
{name: :workflow_id, type: :identifier},
|
|
237
|
+
{name: :export, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowExport'}
|
|
238
|
+
]
|
|
144
239
|
command :workorders, description: 'List work orders of a workflow',
|
|
145
240
|
arguments: [{name: :workflow_id, type: :identifier}],
|
|
146
241
|
action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao("work_orders_list/#{workflow_id}")['work_orders']) }
|
|
242
|
+
command :development, description: "Show documentation for workflow development, from GitHub (no access to Orchestrator): whole document, its table of contents (#{DOC_TOC}, headings up to level #{DOC_TOC_LEVELS.map { |d, l| "#{l} (#{d})" }.join(', ')}, or argument level), or one section (anchor)",
|
|
243
|
+
arguments: [
|
|
244
|
+
{name: :document, type: Symbol, mandatory: false, default: :authoring, allowed: DOC_FILES.keys},
|
|
245
|
+
{name: :section, mandatory: false},
|
|
246
|
+
{name: :level, type: Integer, mandatory: false}
|
|
247
|
+
]
|
|
147
248
|
command :outputs, description: 'Show output specification of a workflow',
|
|
148
249
|
arguments: [{name: :workflow_id, type: :identifier}],
|
|
149
250
|
action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao("workflow_outputs_spec/#{workflow_id}")['workflow_outputs_spec']['output']) }
|
|
@@ -161,13 +262,12 @@ module Aspera
|
|
|
161
262
|
action: ->(workorder_id:, **) { Result::SingleObject.new(call_ao("work_order_reset/#{workorder_id}")['work_order']) }
|
|
162
263
|
command :output, description: 'Show output of a work order',
|
|
163
264
|
arguments: [{name: :workorder_id, type: :identifier}],
|
|
164
|
-
action: ->(workorder_id:, **) { Result::ObjectList.new(call_ao("work_order_output/#{workorder_id}",
|
|
265
|
+
action: ->(workorder_id:, **) { Result::ObjectList.new(call_ao("work_order_output/#{workorder_id}", accept: Mime::XML, xml_opts: {'ForceArray' => %w[variable], 'SuppressEmpty' => nil})['variable']) }
|
|
165
266
|
end
|
|
166
267
|
|
|
167
268
|
commands_under :workstep do
|
|
168
269
|
command :status, description: 'Show status of a work step',
|
|
169
|
-
arguments: [{name: :workstep_id, type: :identifier}]
|
|
170
|
-
action: ->(workstep_id:, **) { Result::SingleObject.new(call_ao("work_step_status/#{workstep_id}")) }
|
|
270
|
+
arguments: [{name: :workstep_id, type: :identifier}]
|
|
171
271
|
command :cancel, description: 'Cancel a work step',
|
|
172
272
|
arguments: [{name: :workstep_id, type: :identifier}],
|
|
173
273
|
action: ->(workstep_id:, **) { Result::SingleObject.new(call_ao("work_step_cancel/#{workstep_id}")) }
|
|
@@ -222,7 +322,7 @@ module Aspera
|
|
|
222
322
|
def action_health(**)
|
|
223
323
|
nagios = Nagios.new
|
|
224
324
|
begin
|
|
225
|
-
info = call_ao('remote_node_ping',
|
|
325
|
+
info = call_ao('remote_node_ping', accept: Mime::XML, xml_opts: {'ForceArray' => false})
|
|
226
326
|
nagios.add_ok('api', 'accessible')
|
|
227
327
|
nagios.check_product_version('api', 'orchestrator', info['orchestrator-version'])
|
|
228
328
|
rescue StandardError => e
|
|
@@ -231,14 +331,227 @@ module Aspera
|
|
|
231
331
|
Result::ObjectList.new(nagios.status_list)
|
|
232
332
|
end
|
|
233
333
|
|
|
334
|
+
# Monitors of work orders are wrapped in `monitor-workorder`
|
|
335
|
+
def action_monitors_list(type:, **)
|
|
336
|
+
monitors = call_ao("list_monitors/#{ANY_ID}", query: {monitor_type: type})
|
|
337
|
+
Result::ObjectList.new(monitors.map { |monitor| monitor.is_a?(Hash) && monitor.size.eql?(1) && monitor.key?('monitor-workorder') ? monitor['monitor-workorder'] : monitor })
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# Files of the packages, and files without package, with the name of their package
|
|
341
|
+
def action_journal_files(book:, **)
|
|
342
|
+
report = call_ao("journal_files/#{book}")
|
|
343
|
+
files = Array(report['Packages']).flat_map { |package| Array(package['files']).map { |file| {'package' => package['name']}.merge(file) } }
|
|
344
|
+
Result::ObjectList.new(files + Array(report['Files']))
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# Fields of the work step that Orchestrator returns as YAML dumps of its Ruby objects (`prerequisites`, `work_outputs`...)
|
|
348
|
+
# are removed: they are not readable, and the inputs and outputs are also given as `inputs` and `outputs`.
|
|
349
|
+
def action_workstep_status(workstep_id:, **)
|
|
350
|
+
status = call_ao("work_step_status/#{workstep_id}")
|
|
351
|
+
status['work_step']&.reject! { |_name, value| value.is_a?(String) && value.match?(YAML_DUMP) }
|
|
352
|
+
Result::SingleObject.new(status)
|
|
353
|
+
end
|
|
354
|
+
|
|
234
355
|
# 2.1/2.2 Initiate a workorder (async / synchronous)
|
|
235
356
|
def action_workflows_list(**)
|
|
236
357
|
Result::ObjectList.new(
|
|
237
358
|
call_ao('workflows_list')['workflows']['workflow'],
|
|
238
|
-
fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification]
|
|
359
|
+
fields: %w[id portable_id name folder_id published_status published_revision_id latest_revision_id last_modification]
|
|
239
360
|
)
|
|
240
361
|
end
|
|
241
362
|
|
|
363
|
+
# The document is read from GitHub: does not require access to Orchestrator
|
|
364
|
+
# @param level [Integer, nil] Maximum level of headings of the table of contents (default per document)
|
|
365
|
+
def action_workflows_development(document: :authoring, section: nil, level: nil, **)
|
|
366
|
+
require 'aspera/markdown'
|
|
367
|
+
require 'aspera/uri_reader'
|
|
368
|
+
Aspera.assert(level.nil? || section.eql?(DOC_TOC), type: Cli::BadArgument) { "argument level is only for #{DOC_TOC}" }
|
|
369
|
+
content = UriReader.read("#{DOC_BASE_URL}/#{DOC_FILES[document]}")
|
|
370
|
+
return Result::Text.new(content) if section.nil?
|
|
371
|
+
return Result::ObjectList.new(Markdown.toc(content, max_level: level || DOC_TOC_LEVELS[document]), fields: %w[level title anchor]) if section.eql?(DOC_TOC)
|
|
372
|
+
text = Markdown.extract_section(content, section)
|
|
373
|
+
Aspera.assert(!text.nil?, type: Cli::Error) { "Section not found: #{section}, use #{DOC_TOC} to list anchors" }
|
|
374
|
+
Result::Text.new(text)
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
# Specifications of action plugins, read from GitHub, or from option `plugin_specs`: does not require access to Orchestrator
|
|
378
|
+
# @return [Hash] Class name of plugin => attributes, inputs, outputs
|
|
379
|
+
def plugin_specs
|
|
380
|
+
return @plugin_specs if @plugin_specs
|
|
381
|
+
require 'aspera/uri_reader'
|
|
382
|
+
require 'aspera/yaml'
|
|
383
|
+
specs = Yaml.safe_load(UriReader.read(options.get_option(:plugin_specs, mandatory: true)))
|
|
384
|
+
Aspera.assert_type(specs, Hash, type: Cli::Error) { 'plugin specifications' }
|
|
385
|
+
@plugin_specs = specs
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
# @param plugin [String, nil] Class name of the plugin (case insensitive), or nil to list plugins
|
|
389
|
+
# @param section [Symbol, nil] One of SPEC_SECTIONS, or nil for the whole specification
|
|
390
|
+
def action_plugins_spec(plugin: nil, section: nil, **)
|
|
391
|
+
specs = plugin_specs
|
|
392
|
+
return Result::ObjectList.new(specs.map { |name, spec| {'name' => name}.merge(spec.except(*SPEC_SECTIONS.map(&:to_s), 'notes')) }, fields: SPEC_LIST_FIELDS) if plugin.nil?
|
|
393
|
+
name = specs.keys.find { |key| key.casecmp?(plugin) }
|
|
394
|
+
if name.nil?
|
|
395
|
+
similar = specs.keys.select { |key| key.downcase.include?(plugin.downcase) }
|
|
396
|
+
raise Cli::BadArgument, "Unknown plugin: #{plugin}#{", similar: #{similar.join(', ')}" unless similar.empty?}, use `plugins spec` to list plugins"
|
|
397
|
+
end
|
|
398
|
+
return Result::SingleObject.new({'name' => name}.merge(specs[name])) if section.nil?
|
|
399
|
+
Result::ObjectList.new(specs[name][section.to_s].map { |key, value| {'name' => key}.merge(value) })
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
# Without dependencies, the workflow file (YAML) is displayed.
|
|
403
|
+
# With dependencies, the package (binary) is saved in the destination folder.
|
|
404
|
+
def action_workflows_export(workflow_id:, export:, **)
|
|
405
|
+
dependencies = export.transform_keys(&:to_s)['dependencies'].eql?(true)
|
|
406
|
+
# Orchestrator exports dependencies for any value of the parameter
|
|
407
|
+
begin
|
|
408
|
+
resp = call_ao("export_workflow/#{workflow_id}", accept: nil, query: dependencies ? {export_with_dependencies: true} : nil, http: true)
|
|
409
|
+
rescue Rest::CallError
|
|
410
|
+
# Orchestrator 4.1.6 fails to build the package of some workflows (it is written in the folder it packs)
|
|
411
|
+
Log.log.warn('Export with dependencies failed: export the workflow and each sub-workflow without dependencies instead') if dependencies
|
|
412
|
+
raise
|
|
413
|
+
end
|
|
414
|
+
return Result::Text.new(resp.body) unless dependencies
|
|
415
|
+
disposition = resp['Content-Disposition']
|
|
416
|
+
file_name = disposition && Rest.parse_header(disposition)[:parameters][:filename]
|
|
417
|
+
file_path = File.join(transfer.destination_folder(Transfer::Spec::DIRECTION_RECEIVE), File.basename(file_name || "Workflow_#{workflow_id}.wkf"))
|
|
418
|
+
File.binwrite(file_path, resp.body)
|
|
419
|
+
Result::Status.new("Saved to: #{file_path}")
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
# Check of a workflow file, for problems that Orchestrator does not detect on import: it imports an invalid workflow, which then fails to publish or to run.
|
|
423
|
+
# This is the only place that knows how the check is done: it is done locally, because Orchestrator has no such API.
|
|
424
|
+
# When it has one (same request as `import_workflow`), replace the body by a call, e.g.
|
|
425
|
+
# `call_ao('check_workflow', body: [['import_file', content, {filename: file_name}], ['import_file_name', file_name]], content_type: Mime::MULTIPART)`
|
|
426
|
+
# The response is a Hash, see {OrchestratorWorkflowChecker}.
|
|
427
|
+
# Templates and inputs of steps are checked with the specifications of plugins (see `plugins spec`), when they can be read.
|
|
428
|
+
# @param file_path [String] Workflow file (`.yml` or `.wkf`)
|
|
429
|
+
# @return [Hash] `valid` (Boolean), `file_name`, `issues` (list of Hash: `severity`, `code`, `file`, `path`, `message`)
|
|
430
|
+
def check_workflow(file_path)
|
|
431
|
+
require 'aspera/orchestrator_workflow_checker'
|
|
432
|
+
specs = nil
|
|
433
|
+
specs_error = nil
|
|
434
|
+
begin
|
|
435
|
+
specs = plugin_specs
|
|
436
|
+
rescue StandardError => e
|
|
437
|
+
specs_error = e.message
|
|
438
|
+
end
|
|
439
|
+
response = OrchestratorWorkflowChecker.check(file_name: File.basename(file_path), content: File.binread(file_path), specs: specs)
|
|
440
|
+
unless specs_error.nil?
|
|
441
|
+
response['issues'].push({
|
|
442
|
+
'severity' => 'warning',
|
|
443
|
+
'code' => 'plugins.specs_unavailable',
|
|
444
|
+
'path' => 'templates',
|
|
445
|
+
'message' => "plugin specifications not read (#{specs_error}): templates and inputs of steps are not checked"
|
|
446
|
+
})
|
|
447
|
+
end
|
|
448
|
+
response
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
# @param file_path [String] Workflow file
|
|
452
|
+
# @return [Result] The issues found, or a status if none
|
|
453
|
+
def action_workflows_check_import(file_path:, **)
|
|
454
|
+
response = check_workflow(file_path)
|
|
455
|
+
return Result::Status.new('no problem found') if response['issues'].empty?
|
|
456
|
+
Result::ObjectList.new(response['issues'], fields: %w[severity code file path message])
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
# Conflicts and missing plugins found by Orchestrator, as one row per name to resolve.
|
|
460
|
+
# Orchestrator does not check that the file exists: for a wrong path, it returns the same empty result as for a file without conflict.
|
|
461
|
+
# @return [Hash{String=>Object}] Items returned by Orchestrator, by name (`subwf constraints`, `missing plugins`...)
|
|
462
|
+
def server_constraints(filename, add_as_revision = nil)
|
|
463
|
+
list = call_ao('find_constraints', query: {file: filename, wf_id: add_as_revision}.compact)
|
|
464
|
+
Aspera.assert_type(list, Array, type: Cli::Error) { 'response of find_constraints' }
|
|
465
|
+
list.grep(Hash).reduce({}, :merge)
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
# Keys of the conflicts in Orchestrator response => keys of the argument of `import_with_constraints`
|
|
469
|
+
CONSTRAINT_KEYS = {
|
|
470
|
+
'subwf constraints' => 'subwf_constraints',
|
|
471
|
+
'action template constraints' => 'action_template_constraints',
|
|
472
|
+
'remote node constraints' => 'remote_node_constraints'
|
|
473
|
+
}.freeze
|
|
474
|
+
private_constant :CONSTRAINT_KEYS
|
|
475
|
+
|
|
476
|
+
def action_workflows_find_constraints(filename:, add_as_revision: nil, **)
|
|
477
|
+
found = server_constraints(filename, add_as_revision)
|
|
478
|
+
rows = CONSTRAINT_KEYS.flat_map do |api_key, key|
|
|
479
|
+
(found[api_key] || {}).map { |name, choices| {'type' => key, 'name' => name, 'choices' => choices} }
|
|
480
|
+
end
|
|
481
|
+
Array(found['missing plugins']).each do |name|
|
|
482
|
+
rows.push({'type' => 'missing_plugins', 'name' => name, 'choices' => 'auto_enable_missing_plugins: true'})
|
|
483
|
+
end
|
|
484
|
+
return Result::Status.new('no conflict found (Orchestrator gives the same answer if the file does not exist)') if rows.empty?
|
|
485
|
+
Result::ObjectList.new(rows, fields: %w[type name choices])
|
|
486
|
+
end
|
|
487
|
+
|
|
488
|
+
# Orchestrator fails or decides silently in these cases, which the web interface prevents:
|
|
489
|
+
# - plugins to enable but `auto_enable_missing_plugins` not true: the import fails with an internal error
|
|
490
|
+
# - resolution given for some conflicts only: the others are imported as copies, without error
|
|
491
|
+
# @param payload [Hash] Argument of `import_with_constraints`, with string keys
|
|
492
|
+
def check_import_constraints(payload)
|
|
493
|
+
found = server_constraints(payload['filename'], payload['add_as_revision'])
|
|
494
|
+
problems = []
|
|
495
|
+
missing = Array(found['missing plugins'])
|
|
496
|
+
problems.push("plugins are not enabled: #{missing.join(', ')}, set auto_enable_missing_plugins to true") unless missing.empty? || payload['auto_enable_missing_plugins'].eql?(true)
|
|
497
|
+
CONSTRAINT_KEYS.each do |api_key, key|
|
|
498
|
+
chosen = payload[key]
|
|
499
|
+
next if chosen.nil? || chosen.empty?
|
|
500
|
+
unresolved = (found[api_key] || {}).keys - chosen.keys.map(&:to_s)
|
|
501
|
+
problems.push("#{key} not resolved for: #{unresolved.join(', ')} (see `workflows find_constraints`)") unless unresolved.empty?
|
|
502
|
+
end
|
|
503
|
+
raise Cli::BadArgument, "Import refused: #{problems.join('; ')}" unless problems.empty?
|
|
504
|
+
end
|
|
505
|
+
|
|
506
|
+
# The file is uploaded as a multipart form
|
|
507
|
+
# @param import [Hash] Options: `folder_id`, `publish`, `link_to_subworkflows`
|
|
508
|
+
def action_workflows_import(file_path:, import: {}, **)
|
|
509
|
+
import = import.transform_keys(&:to_s)
|
|
510
|
+
check_workflow(file_path)['issues'].each do |issue|
|
|
511
|
+
Log.log.warn { "#{issue['severity']}: #{[issue['file'], issue['path']].compact.join(' ')}: #{issue['message']} (#{issue['code']})" }
|
|
512
|
+
end
|
|
513
|
+
file_name = File.basename(file_path)
|
|
514
|
+
form = [
|
|
515
|
+
['import_file', File.binread(file_path), {filename: file_name}],
|
|
516
|
+
['import_file_name', file_name]
|
|
517
|
+
]
|
|
518
|
+
form.push(['folder_id', import['folder_id'].to_s]) if import.key?('folder_id')
|
|
519
|
+
# `false` overrides the server option `publish_api_imported_workflow`
|
|
520
|
+
form.push(['publish_workflow', import['publish'].to_s]) if import.key?('publish')
|
|
521
|
+
# Orchestrator takes any value as true
|
|
522
|
+
form.push(%w[link_to_subworkflows true]) if import['link_to_subworkflows'].eql?(true)
|
|
523
|
+
result = call_ao('import_workflow', body: form, content_type: Mime::MULTIPART)
|
|
524
|
+
# Other responses: nothing was imported
|
|
525
|
+
if result.key?('plugins')
|
|
526
|
+
raise Cli::Error, "Plugins could not be enabled: #{Array(result['plugins']).join(', ')}" \
|
|
527
|
+
"#{" (missing dependencies: #{Array(result['missing_deps']).join(', ')})" unless Array(result['missing_deps']).empty?}"
|
|
528
|
+
end
|
|
529
|
+
if result.key?('dependencies')
|
|
530
|
+
dependencies = result['dependencies'].values.flatten.filter_map { |d| "#{d['entity']} #{d['id_value']}" if d.is_a?(Hash) }
|
|
531
|
+
raise Cli::Error, "Workflow dependencies are not included in the file (export with dependencies to a .wkf file): #{dependencies.uniq.join(', ')}"
|
|
532
|
+
end
|
|
533
|
+
Result::SingleObject.new(result['workflow'])
|
|
534
|
+
end
|
|
535
|
+
|
|
536
|
+
# Items of the array expected by `import_with_constraints`, in this order: argument key => API key
|
|
537
|
+
IMPORT_CONSTRAINT_ITEMS = {
|
|
538
|
+
'filename' => 'filename',
|
|
539
|
+
'add_as_revision' => 'add as revision',
|
|
540
|
+
'subwf_constraints' => 'subwf constraints',
|
|
541
|
+
'action_template_constraints' => 'action template constraints',
|
|
542
|
+
'remote_node_constraints' => 'remote node constraints',
|
|
543
|
+
'auto_enable_missing_plugins' => 'Auto-enable missing plugins?'
|
|
544
|
+
}.freeze
|
|
545
|
+
private_constant :IMPORT_CONSTRAINT_ITEMS
|
|
546
|
+
|
|
547
|
+
# The API expects an array of single-key objects, in a fixed order
|
|
548
|
+
def action_workflows_import_with_constraints(payload:, **)
|
|
549
|
+
payload = payload.transform_keys(&:to_s)
|
|
550
|
+
check_import_constraints(payload)
|
|
551
|
+
body = IMPORT_CONSTRAINT_ITEMS.map { |key, api_key| {api_key => payload.fetch(key) { key.end_with?('_constraints') ? {} : nil }} }
|
|
552
|
+
Result::SingleObject.new(call_ao('import_with_constraints', body: body)['workflow'])
|
|
553
|
+
end
|
|
554
|
+
|
|
242
555
|
# Keys of argument `execution` of `workflows start`
|
|
243
556
|
WORKFLOW_START_KEYS = %w[synchronous step variable].freeze
|
|
244
557
|
private_constant :WORKFLOW_START_KEYS
|
|
@@ -269,8 +582,8 @@ module Aspera
|
|
|
269
582
|
# implicitly, call is synchronous
|
|
270
583
|
query_params[:synchronous] = true
|
|
271
584
|
end
|
|
272
|
-
|
|
273
|
-
|
|
585
|
+
# Work order information, or value of the explicit output (any JSON value)
|
|
586
|
+
Result.auto(call_ao('initiate', body: json_body, query: query_params.empty? ? nil : query_params))
|
|
274
587
|
end
|
|
275
588
|
end
|
|
276
589
|
end
|
|
@@ -66,10 +66,12 @@ module Aspera
|
|
|
66
66
|
# @param app_url [String] Tested URL
|
|
67
67
|
# @return [Hash] :preset_value, :test_args
|
|
68
68
|
def wizard(wizard, app_url)
|
|
69
|
+
username = options.get_option(:username, mandatory: true)
|
|
69
70
|
return {
|
|
71
|
+
preset_ids: [username],
|
|
70
72
|
preset_value: {
|
|
71
73
|
url: app_url,
|
|
72
|
-
username:
|
|
74
|
+
username: username,
|
|
73
75
|
password: options.get_option(:password, mandatory: true)
|
|
74
76
|
},
|
|
75
77
|
test_args: 'browse /'
|