aspera-cli 4.27.5 → 4.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +160 -4
  4. data/CONTRIBUTING.md +4 -2
  5. data/README.md +1 -1
  6. data/bin/asession +1 -1
  7. data/docs/README.md +1086 -331
  8. data/docs/test-mcp-with-ai.md +5 -3
  9. data/lib/aspera/agent/direct.rb +15 -4
  10. data/lib/aspera/agent/factory.rb +2 -1
  11. data/lib/aspera/agent/node.rb +1 -1
  12. data/lib/aspera/api/cos_node.rb +1 -1
  13. data/lib/aspera/api/faspex.rb +1 -1
  14. data/lib/aspera/api/node.rb +6 -2
  15. data/lib/aspera/ascp/installation.rb +47 -11
  16. data/lib/aspera/ascp/management.rb +89 -43
  17. data/lib/aspera/cli/ascp_actions.rb +5 -19
  18. data/lib/aspera/cli/bootstrapper.rb +1 -2
  19. data/lib/aspera/cli/command_registry.rb +43 -3
  20. data/lib/aspera/cli/command_spec.rb +3 -0
  21. data/lib/aspera/cli/completion/ascli.bash +39 -5
  22. data/lib/aspera/cli/completion/ascli.fish +16 -8
  23. data/lib/aspera/cli/completion/ascli.ps1 +41 -0
  24. data/lib/aspera/cli/completion/ascli.zsh +20 -4
  25. data/lib/aspera/cli/completion_actions.rb +360 -0
  26. data/lib/aspera/cli/error.rb +42 -0
  27. data/lib/aspera/cli/extended_value.rb +30 -5
  28. data/lib/aspera/cli/formatter.rb +118 -56
  29. data/lib/aspera/cli/hints.rb +0 -1
  30. data/lib/aspera/cli/http.rb +6 -5
  31. data/lib/aspera/cli/info.rb +10 -4
  32. data/lib/aspera/cli/mailer.rb +1 -1
  33. data/lib/aspera/cli/mcp/http_servlet.rb +156 -0
  34. data/lib/aspera/cli/mcp/secrets.rb +143 -0
  35. data/lib/aspera/cli/mcp/tool.rb +231 -0
  36. data/lib/aspera/cli/option_value.rb +9 -1
  37. data/lib/aspera/cli/options.schema.yaml +390 -39
  38. data/lib/aspera/cli/parser.rb +14 -5
  39. data/lib/aspera/cli/plugins/aoc.rb +8 -11
  40. data/lib/aspera/cli/plugins/base.rb +36 -15
  41. data/lib/aspera/cli/plugins/config.rb +15 -44
  42. data/lib/aspera/cli/plugins/console.rb +3 -1
  43. data/lib/aspera/cli/plugins/factory.rb +1 -1
  44. data/lib/aspera/cli/plugins/faspex5.rb +9 -19
  45. data/lib/aspera/cli/plugins/faspio.rb +36 -5
  46. data/lib/aspera/cli/plugins/httpgw.rb +1 -0
  47. data/lib/aspera/cli/plugins/mcp.rb +177 -118
  48. data/lib/aspera/cli/plugins/node.rb +14 -8
  49. data/lib/aspera/cli/plugins/orchestrator.rb +351 -38
  50. data/lib/aspera/cli/plugins/server.rb +3 -1
  51. data/lib/aspera/cli/plugins/shares.rb +205 -35
  52. data/lib/aspera/cli/preset_actions.rb +15 -7
  53. data/lib/aspera/cli/preset_manager.rb +24 -2
  54. data/lib/aspera/cli/prompt.rb +3 -1
  55. data/lib/aspera/cli/result.rb +17 -4
  56. data/lib/aspera/cli/runner.rb +133 -43
  57. data/lib/aspera/cli/security.rb +134 -0
  58. data/lib/aspera/cli/table_fitter.rb +116 -0
  59. data/lib/aspera/cli/terminal_formatter.rb +23 -34
  60. data/lib/aspera/cli/transfer_agent.rb +4 -1
  61. data/lib/aspera/cli/vault_manager.rb +6 -5
  62. data/lib/aspera/cli/version.rb +1 -1
  63. data/lib/aspera/cli/web_secret.html.erb +61 -0
  64. data/lib/aspera/cli/web_secret.rb +220 -0
  65. data/lib/aspera/cli/wizard.rb +19 -9
  66. data/lib/aspera/command_line_builder.rb +26 -0
  67. data/lib/aspera/data_repository.rb +38 -24
  68. data/lib/aspera/duration.rb +42 -0
  69. data/lib/aspera/environment.rb +88 -24
  70. data/lib/aspera/exec_spec.rb +6 -5
  71. data/lib/aspera/faspex_gw.rb +98 -32
  72. data/lib/aspera/faspex_postproc.rb +187 -44
  73. data/lib/aspera/formatter_interface.rb +0 -1
  74. data/lib/aspera/id_generator.rb +2 -1
  75. data/lib/aspera/json_rpc/client.rb +51 -32
  76. data/lib/aspera/json_rpc/version.rb +1 -0
  77. data/lib/aspera/keychain/base.rb +5 -0
  78. data/lib/aspera/keychain/encrypted_hash.rb +6 -2
  79. data/lib/aspera/keychain/factory.rb +2 -3
  80. data/lib/aspera/keychain/hashicorp_vault.rb +7 -4
  81. data/lib/aspera/keychain/macos_security.rb +119 -13
  82. data/lib/aspera/keychain/one_password_api.rb +8 -3
  83. data/lib/aspera/keychain/one_password_base.rb +14 -9
  84. data/lib/aspera/keychain/one_password_cli.rb +4 -2
  85. data/lib/aspera/log.rb +3 -6
  86. data/lib/aspera/markdown.rb +152 -115
  87. data/lib/aspera/mime.rb +6 -0
  88. data/lib/aspera/nagios.rb +3 -1
  89. data/lib/aspera/node_emulator.rb +98 -101
  90. data/lib/aspera/oauth/{base.rb → client/base.rb} +5 -4
  91. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +3 -2
  92. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +7 -0
  93. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +1 -1
  94. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +1 -1
  95. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +1 -1
  96. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +1 -1
  97. data/lib/aspera/oauth/{web.rb → client/web.rb} +2 -2
  98. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  99. data/lib/aspera/oauth/server/pkce.rb +426 -0
  100. data/lib/aspera/orchestrator_workflow_checker.rb +561 -0
  101. data/lib/aspera/preview/terminal.rb +3 -2
  102. data/lib/aspera/rainbow.rb +27 -1
  103. data/lib/aspera/rest/aspera_errors.rb +51 -59
  104. data/lib/aspera/rest/client.rb +21 -14
  105. data/lib/aspera/rest/error_analyzer.rb +9 -0
  106. data/lib/aspera/rest/list.rb +6 -6
  107. data/lib/aspera/rest/util.rb +7 -1
  108. data/lib/aspera/rest.rb +0 -1
  109. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -2
  110. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4001 -0
  111. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  112. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +35 -35
  113. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  114. data/lib/aspera/schema/documentation.rb +50 -78
  115. data/lib/aspera/schema/reader.rb +39 -11
  116. data/lib/aspera/schema/registry.rb +8 -2
  117. data/lib/aspera/secret_hider.rb +94 -15
  118. data/lib/aspera/secret_strength.rb +114 -0
  119. data/lib/aspera/servlet_helpers.rb +111 -0
  120. data/lib/aspera/sync/args.schema.yaml +3 -1
  121. data/lib/aspera/sync/conf.schema.yaml +16 -6
  122. data/lib/aspera/sync/operations.rb +21 -1
  123. data/lib/aspera/temp_file_manager.rb +79 -20
  124. data/lib/aspera/transfer/parameters.rb +5 -0
  125. data/lib/aspera/transfer/spec.rb +52 -12
  126. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  127. data/lib/aspera/web_auth/index.html.erb +16 -0
  128. data/lib/aspera/web_auth/server.rb +18 -10
  129. data/lib/aspera/web_page.html.erb +262 -0
  130. data/lib/aspera/web_page.rb +61 -0
  131. data/lib/aspera/web_server_simple.rb +76 -10
  132. data.tar.gz.sig +0 -0
  133. metadata +47 -32
  134. metadata.gz.sig +0 -0
  135. data/lib/aspera/cli/mcp_tool.rb +0 -189
  136. data/lib/aspera/oauth.rb +0 -9
  137. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  138. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  139. data/lib/aspera/web_auth/index.html +0 -147
@@ -5,6 +5,8 @@ require 'aspera/cli/special_values'
5
5
  require 'aspera/nagios'
6
6
  require 'aspera/log'
7
7
  require 'aspera/assert'
8
+ require 'aspera/transfer/spec'
9
+ require 'aspera/schema/registry'
8
10
  require 'xmlsimple'
9
11
 
10
12
  module Aspera
@@ -14,7 +16,35 @@ module Aspera
14
16
  class Orchestrator < BasicAuth
15
17
  STANDARD_PATH = '/aspera/orchestrator'
16
18
  TEST_ENDPOINT = 'api/remote_node_ping'
17
- private_constant :STANDARD_PATH, :TEST_ENDPOINT
19
+ # Raw Markdown of the workflow authoring documents
20
+ DOC_BASE_URL = 'https://raw.githubusercontent.com/laurent-martin/aspera-orchestrator-plugins-doc/main/docs'
21
+ # Documents of command `workflows development`: name => file
22
+ DOC_FILES = {
23
+ authoring: 'workflow-authoring-guide.md',
24
+ templates: 'action-template-reference.md',
25
+ manual: 'Orchestrator_Plugin_Manual.md'
26
+ }.freeze
27
+ # Value of argument `section` of command `workflows development` for the table of contents
28
+ DOC_TOC = 'toc'
29
+ # Default maximum level of headings in the table of contents: guide sections, one entry per plugin
30
+ DOC_TOC_LEVELS = {
31
+ authoring: 3,
32
+ templates: 2,
33
+ manual: 3
34
+ }.freeze
35
+ # Sections of a plugin specification, listed by command `plugins spec`
36
+ SPEC_SECTIONS = %i[attributes inputs outputs].freeze
37
+ # Fields of the list of plugins of command `plugins spec`
38
+ SPEC_LIST_FIELDS = %w[name category version display_name].freeze
39
+ # Text value that is a YAML document (Ruby objects serialized by Orchestrator)
40
+ YAML_DUMP = /\A---\s/
41
+ # Types of monitors of `list_monitors` (`monitor_type`) and of `monitor_snapshot` (path)
42
+ MONITOR_TYPES = %i[workflow workorder folder node script].freeze
43
+ MONITOR_SNAPSHOT_TYPES = %i[workflows workorders folders nodes scripts].freeze
44
+ # Id segment of the path, for endpoints routed only as `api/<endpoint>/<id>` that do not use it
45
+ ANY_ID = '0'
46
+ private_constant :STANDARD_PATH, :TEST_ENDPOINT, :DOC_BASE_URL, :DOC_FILES, :DOC_TOC, :DOC_TOC_LEVELS, :SPEC_SECTIONS, :SPEC_LIST_FIELDS, :YAML_DUMP,
47
+ :MONITOR_TYPES, :MONITOR_SNAPSHOT_TYPES, :ANY_ID
18
48
 
19
49
  class << self
20
50
  # @return [Hash,NilClass]
@@ -46,10 +76,12 @@ module Aspera
46
76
  # @param app_url [String] Tested URL
47
77
  # @return [Hash] :preset_value, :test_args
48
78
  def wizard(wizard, app_url)
79
+ username = options.get_option(:username, mandatory: true)
49
80
  return {
81
+ preset_ids: [username],
50
82
  preset_value: {
51
83
  url: app_url,
52
- username: options.get_option(:username, mandatory: true),
84
+ username: username,
53
85
  password: options.get_option(:password, mandatory: true)
54
86
  },
55
87
  test_args: 'workflows list'
@@ -61,55 +93,108 @@ module Aspera
61
93
  option :ret_style, description: 'Method to specify the expected response format ("Accept")', allowed: %i[header query path], default: :query
62
94
  option :result, description: "Specify result value as: 'work_step:parameter'", deprecation: {last: '4.27.2', message: 'use keys `step` and `variable` of argument `execution` of `workflows start`'}
63
95
  option :synchronous, description: 'Wait for completion', allowed: Type::BOOLEAN, deprecation: {last: '4.27.2', message: 'use key `synchronous` of argument `execution` of `workflows start`'}
96
+ option :plugin_specs, description: 'URL or path of the plugin specifications (YAML) used by `plugins spec`', default: "#{DOC_BASE_URL}/plugins.yml"
64
97
 
65
98
  # Call orchestrator API (handles ret_style negotiation and XML parsing)
66
- # @param endpoint [String] the endpoint to call
67
- # @param body [Hash] the body to pass (JSON); also implies POST
68
- # @param format [String] the format to request, 'json', 'xml', nil
69
- # @param query [Hash] the arguments to pass as query parameters
70
- # @param xml_arrays [Boolean] if true, force arrays in xml parsing
71
- # @param http [Boolean] if true, returns the HttpResponse, else
72
- def call_ao(endpoint, body: nil, format: 'json', query: nil, xml_arrays: true, http: false)
73
- call_args = {operation: body.nil? ? 'GET' : 'POST', subpath: "api/#{endpoint}", ret: :both, query: {}}
74
- call_args.merge!(body: body, content_type: Mime::JSON) unless body.nil? # rubocop:disable Performance/RedundantMerge
99
+ # @param endpoint [String] the endpoint to call
100
+ # @param body [Hash, Array] the body to pass; also implies POST
101
+ # @param content_type [String] the body type (Mime::JSON or Mime::MULTIPART)
102
+ # @param accept [String] the response type to request (Mime::JSON or Mime::XML), or nil
103
+ # @param query [Hash] the arguments to pass as query parameters
104
+ # @param xml_opts [Hash] options of `XmlSimple.xml_in` to parse an XML response
105
+ # @param json_opts [Hash] options of `JSON.parse` to parse a JSON response
106
+ # @param http [Boolean] if true, returns the HttpResponse, else
107
+ def call_ao(endpoint, body: nil, content_type: Mime::JSON, accept: Mime::JSON, query: nil, xml_opts: {}, json_opts: {}, http: false)
108
+ call_args = {operation: body.nil? ? 'GET' : 'POST', subpath: "api/#{endpoint}", ret: :both, query: {}, json_opts: json_opts}
109
+ call_args.merge!(body: body, content_type: content_type) unless body.nil? # rubocop:disable Performance/RedundantMerge
75
110
  ret_style = options.get_option(:ret_style, mandatory: true)
76
111
  call_args[:query].merge!(query) unless query.nil?
77
- unless format.nil?
112
+ unless accept.nil?
113
+ # 'json' or 'xml'
114
+ short_type = accept.split('/').last
78
115
  case ret_style
79
116
  when :header
80
- call_args[:headers] = {'Accept' => "application/#{format}"}
117
+ call_args[:headers] = {'Accept' => accept}
81
118
  when :query
82
- call_args[:query][:format] = format
119
+ call_args[:query][:format] = short_type
83
120
  when :path
84
- call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
121
+ call_args[:subpath] = "#{call_args[:subpath]}.#{short_type}"
85
122
  else Aspera.error_unexpected_value(ret_style) { 'ret_style' }
86
123
  end
87
124
  end
88
- add_query = query_read_delete
125
+ add_query = query_read_delete(schema: api_query_schema(endpoint, call_args[:operation]))
89
126
  call_args[:query].merge!(add_query.symbolize_keys) unless add_query.nil?
90
127
  data, resp = api_orch.call(**call_args)
91
128
  return resp if http
92
- result = format.eql?('xml') ? XmlSimple.xml_in(resp.body, {'ForceArray' => xml_arrays}) : data
129
+ result = accept.eql?(Mime::XML) ? XmlSimple.xml_in(resp.body, xml_opts) : data
93
130
  Log.dump(:data, result)
94
131
  return result
95
132
  end
96
133
 
97
- private :call_ao
134
+ # Schema of the query parameters of an endpoint, in the OpenAPI description of Orchestrator (for `--query=help`)
135
+ # @param endpoint [String] Endpoint, with an id or not, e.g. `journal_files/journal`
136
+ # @param operation [String] HTTP method
137
+ # @return [String, nil] Schema path, or `nil` if the endpoint is not described
138
+ def api_query_schema(endpoint, operation)
139
+ name, id = endpoint.split('/', 2)
140
+ paths = Schema::Registry.instance.reader(Schema::Registry::ORCHESTRATOR)['paths']
141
+ # Orchestrator routes both `api/<name>` and `api/<name>/<id>`: prefer the form used
142
+ candidates = ["api/#{name}", "api/#{name}/{id}"]
143
+ candidates.reverse! unless id.nil?
144
+ path = candidates.find { |candidate| paths.dig("/#{candidate}", operation.downcase) }
145
+ path && Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, path, method: operation.downcase)
146
+ end
147
+
148
+ private :call_ao, :api_query_schema
98
149
 
99
150
  # --- DSL ---
100
151
 
101
152
  command :health, description: 'Check Orchestrator API health'
102
- command :info, description: 'Check that Orchestrator responds (ping)', action: ->(**) { Result::SingleObject.new(call_ao('remote_node_ping', format: 'xml', xml_arrays: false)) }
103
- command :processes, description: 'Show Orchestrator background process status', action: ->(**) { Result::ObjectList.new(call_ao('processes_status', format: 'xml')['node'].flat_map { |n| n['process'] }) }
104
- command :monitors, description: 'Show Orchestrator monitor snapshot', action: ->(**) { Result::SingleObject.new(call_ao('monitor_snapshot')['monitor']) }
153
+ command :info, description: 'Check that Orchestrator responds (ping)', action: ->(**) { Result::SingleObject.new(call_ao('remote_node_ping', accept: Mime::XML, xml_opts: {'ForceArray' => false})) }
154
+ command :processes, description: 'Show Orchestrator background process status', action: ->(**) { Result::ObjectList.new(call_ao('processes_status', accept: Mime::XML, xml_opts: {'ForceArray' => %w[node process]})['node'].flat_map { |n| n['process'] }) }
155
+ command :monitors, description: 'Monitors of workflows, work orders, folders, remote nodes and scripts'
156
+ command :queues, description: 'Managed queues'
157
+ command :pools, description: 'Resource pools'
158
+ command :journal, description: 'Journal books'
105
159
  command :workorders, description: 'Manage work orders'
106
160
  command :workstep, description: 'Manage work steps'
107
161
 
162
+ commands_under :monitors do
163
+ command :snapshot, description: 'Show status of monitors, of all types or of one type',
164
+ arguments: [{name: :type, type: Symbol, mandatory: false, allowed: MONITOR_SNAPSHOT_TYPES}],
165
+ action: ->(type: nil, **) { Result::SingleObject.new(call_ao(type.nil? ? 'monitor_snapshot' : "monitor_snapshot/#{type}")['monitor']) }
166
+ command :list, description: 'List monitors of a type: remote node monitors give the names of remote nodes',
167
+ arguments: [{name: :type, type: Symbol, allowed: MONITOR_TYPES}]
168
+ end
169
+
170
+ commands_under :queues do
171
+ command :list, description: 'List managed queues',
172
+ action: ->(**) { Result::ObjectList.new(call_ao('queues_list').dig('Queues', 'queue') || []) }
173
+ end
174
+
175
+ commands_under :pools do
176
+ command :status, description: 'Show resources of a resource pool and their status',
177
+ arguments: [{name: :pool_name}],
178
+ # Orchestrator repeats key `id` in each resource
179
+ action: ->(pool_name:, **) { Result::ObjectList.new(call_ao("pool_status/#{ANY_ID}", query: {name: pool_name}, json_opts: {allow_duplicate_key: true})['ManagedResource'] || []) }
180
+ end
181
+
182
+ commands_under :journal do
183
+ command :files, description: 'List files recorded in a journal book (filters with option query)',
184
+ arguments: [{name: :book}],
185
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/journal_files/{id}')
186
+ end
187
+
108
188
  commands_under :plugins do
109
189
  command :list, description: 'Show Orchestrator plugin versions', action: ->(**) { Result::ObjectList.new(call_ao('plugin_version')['Plugin']) }
110
190
  command :reload_set, description: 'Reload a set of plugins',
111
191
  arguments: [{name: :plugin_set, type: Hash, schema: 'opts:components.schemas.OrchestratorReloadPluginSet'}],
112
192
  action: ->(plugin_set:, **) { Result::SingleObject.new(call_ao('reload_plugin_set', body: plugin_set.transform_keys(&:to_sym))) }
193
+ command :spec, description: "Show specifications of action plugins, from GitHub (no access to Orchestrator): list of plugins, one plugin, or its #{SPEC_SECTIONS.join(', ')}",
194
+ arguments: [
195
+ {name: :plugin, mandatory: false},
196
+ {name: :section, type: Symbol, mandatory: false, allowed: SPEC_SECTIONS}
197
+ ]
113
198
  end
114
199
 
115
200
  commands_under :workflows do
@@ -129,21 +214,37 @@ module Aspera
129
214
  {name: :parameters, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorInitiateParameters'},
130
215
  {name: :execution, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowStart'}
131
216
  ]
132
- command :import, description: 'Import a workflow',
133
- arguments: [{name: :workflow, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWorkflow'}],
134
- action: ->(workflow:, **) { Result::SingleObject.new(call_ao('import_workflow', body: workflow.transform_keys(&:to_sym))) }
217
+ command :import, description: 'Import a workflow from a file created by `workflows export`, local problems are reported as warnings (see `check_import`)',
218
+ arguments: [
219
+ {name: :file_path},
220
+ {name: :import, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowImport'}
221
+ ]
222
+ command :check_import, description: 'Check a workflow file (no access to Orchestrator): rules that Orchestrator does not check on import, but that make the workflow fail to publish or to run',
223
+ arguments: [{name: :file_path}]
224
+ command :find_constraints, description: 'List the conflicts and missing plugins of a workflow file present on the Orchestrator host, before `import_with_constraints`',
225
+ arguments: [
226
+ {name: :filename},
227
+ {name: :add_as_revision, type: :identifier, mandatory: false}
228
+ ]
135
229
  command :publish, description: 'Publish a workflow',
136
230
  arguments: [{name: :workflow_id, type: :identifier}],
137
- action: ->(workflow_id:, **) { Result::SingleObject.new(call_ao('publish_workflow', body: {workflow_id: workflow_id.to_i})) }
138
- command :import_with_constraints, description: 'Import a workflow with constraint resolution',
139
- arguments: [{name: :payload, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWithConstraints'}],
140
- action: ->(payload:, **) { Result::SingleObject.new(call_ao('import_with_constraints', body: payload.transform_keys(&:to_sym))) }
141
- command :export, description: 'Export a workflow',
142
- arguments: [{name: :workflow_id, type: :identifier}],
143
- action: ->(workflow_id:, **) { Result::Text.new(call_ao("export_workflow/#{workflow_id}", format: nil, http: true).body) }
231
+ action: ->(workflow_id:, **) { Result::Status.new(call_ao('publish_workflow', body: {id: workflow_id}).eql?(true) ? 'published' : 'not published') }
232
+ command :import_with_constraints, description: 'Import a workflow file present on the Orchestrator host, with resolution of conflicts',
233
+ arguments: [{name: :payload, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWithConstraints'}]
234
+ command :export, description: 'Export a workflow: display it, or save it with its dependencies',
235
+ arguments: [
236
+ {name: :workflow_id, type: :identifier},
237
+ {name: :export, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowExport'}
238
+ ]
144
239
  command :workorders, description: 'List work orders of a workflow',
145
240
  arguments: [{name: :workflow_id, type: :identifier}],
146
241
  action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao("work_orders_list/#{workflow_id}")['work_orders']) }
242
+ command :development, description: "Show documentation for workflow development, from GitHub (no access to Orchestrator): whole document, its table of contents (#{DOC_TOC}, headings up to level #{DOC_TOC_LEVELS.map { |d, l| "#{l} (#{d})" }.join(', ')}, or argument level), or one section (anchor)",
243
+ arguments: [
244
+ {name: :document, type: Symbol, mandatory: false, default: :authoring, allowed: DOC_FILES.keys},
245
+ {name: :section, mandatory: false},
246
+ {name: :level, type: Integer, mandatory: false}
247
+ ]
147
248
  command :outputs, description: 'Show output specification of a workflow',
148
249
  arguments: [{name: :workflow_id, type: :identifier}],
149
250
  action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao("workflow_outputs_spec/#{workflow_id}")['workflow_outputs_spec']['output']) }
@@ -161,13 +262,12 @@ module Aspera
161
262
  action: ->(workorder_id:, **) { Result::SingleObject.new(call_ao("work_order_reset/#{workorder_id}")['work_order']) }
162
263
  command :output, description: 'Show output of a work order',
163
264
  arguments: [{name: :workorder_id, type: :identifier}],
164
- action: ->(workorder_id:, **) { Result::ObjectList.new(call_ao("work_order_output/#{workorder_id}", format: 'xml')['variable']) }
265
+ action: ->(workorder_id:, **) { Result::ObjectList.new(call_ao("work_order_output/#{workorder_id}", accept: Mime::XML, xml_opts: {'ForceArray' => %w[variable], 'SuppressEmpty' => nil})['variable']) }
165
266
  end
166
267
 
167
268
  commands_under :workstep do
168
269
  command :status, description: 'Show status of a work step',
169
- arguments: [{name: :workstep_id, type: :identifier}],
170
- action: ->(workstep_id:, **) { Result::SingleObject.new(call_ao("work_step_status/#{workstep_id}")) }
270
+ arguments: [{name: :workstep_id, type: :identifier}]
171
271
  command :cancel, description: 'Cancel a work step',
172
272
  arguments: [{name: :workstep_id, type: :identifier}],
173
273
  action: ->(workstep_id:, **) { Result::SingleObject.new(call_ao("work_step_cancel/#{workstep_id}")) }
@@ -222,7 +322,7 @@ module Aspera
222
322
  def action_health(**)
223
323
  nagios = Nagios.new
224
324
  begin
225
- info = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
325
+ info = call_ao('remote_node_ping', accept: Mime::XML, xml_opts: {'ForceArray' => false})
226
326
  nagios.add_ok('api', 'accessible')
227
327
  nagios.check_product_version('api', 'orchestrator', info['orchestrator-version'])
228
328
  rescue StandardError => e
@@ -231,14 +331,227 @@ module Aspera
231
331
  Result::ObjectList.new(nagios.status_list)
232
332
  end
233
333
 
334
+ # Monitors of work orders are wrapped in `monitor-workorder`
335
+ def action_monitors_list(type:, **)
336
+ monitors = call_ao("list_monitors/#{ANY_ID}", query: {monitor_type: type})
337
+ Result::ObjectList.new(monitors.map { |monitor| monitor.is_a?(Hash) && monitor.size.eql?(1) && monitor.key?('monitor-workorder') ? monitor['monitor-workorder'] : monitor })
338
+ end
339
+
340
+ # Files of the packages, and files without package, with the name of their package
341
+ def action_journal_files(book:, **)
342
+ report = call_ao("journal_files/#{book}")
343
+ files = Array(report['Packages']).flat_map { |package| Array(package['files']).map { |file| {'package' => package['name']}.merge(file) } }
344
+ Result::ObjectList.new(files + Array(report['Files']))
345
+ end
346
+
347
+ # Fields of the work step that Orchestrator returns as YAML dumps of its Ruby objects (`prerequisites`, `work_outputs`...)
348
+ # are removed: they are not readable, and the inputs and outputs are also given as `inputs` and `outputs`.
349
+ def action_workstep_status(workstep_id:, **)
350
+ status = call_ao("work_step_status/#{workstep_id}")
351
+ status['work_step']&.reject! { |_name, value| value.is_a?(String) && value.match?(YAML_DUMP) }
352
+ Result::SingleObject.new(status)
353
+ end
354
+
234
355
  # 2.1/2.2 Initiate a workorder (async / synchronous)
235
356
  def action_workflows_list(**)
236
357
  Result::ObjectList.new(
237
358
  call_ao('workflows_list')['workflows']['workflow'],
238
- fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification]
359
+ fields: %w[id portable_id name folder_id published_status published_revision_id latest_revision_id last_modification]
239
360
  )
240
361
  end
241
362
 
363
+ # The document is read from GitHub: does not require access to Orchestrator
364
+ # @param level [Integer, nil] Maximum level of headings of the table of contents (default per document)
365
+ def action_workflows_development(document: :authoring, section: nil, level: nil, **)
366
+ require 'aspera/markdown'
367
+ require 'aspera/uri_reader'
368
+ Aspera.assert(level.nil? || section.eql?(DOC_TOC), type: Cli::BadArgument) { "argument level is only for #{DOC_TOC}" }
369
+ content = UriReader.read("#{DOC_BASE_URL}/#{DOC_FILES[document]}")
370
+ return Result::Text.new(content) if section.nil?
371
+ return Result::ObjectList.new(Markdown.toc(content, max_level: level || DOC_TOC_LEVELS[document]), fields: %w[level title anchor]) if section.eql?(DOC_TOC)
372
+ text = Markdown.extract_section(content, section)
373
+ Aspera.assert(!text.nil?, type: Cli::Error) { "Section not found: #{section}, use #{DOC_TOC} to list anchors" }
374
+ Result::Text.new(text)
375
+ end
376
+
377
+ # Specifications of action plugins, read from GitHub, or from option `plugin_specs`: does not require access to Orchestrator
378
+ # @return [Hash] Class name of plugin => attributes, inputs, outputs
379
+ def plugin_specs
380
+ return @plugin_specs if @plugin_specs
381
+ require 'aspera/uri_reader'
382
+ require 'aspera/yaml'
383
+ specs = Yaml.safe_load(UriReader.read(options.get_option(:plugin_specs, mandatory: true)))
384
+ Aspera.assert_type(specs, Hash, type: Cli::Error) { 'plugin specifications' }
385
+ @plugin_specs = specs
386
+ end
387
+
388
+ # @param plugin [String, nil] Class name of the plugin (case insensitive), or nil to list plugins
389
+ # @param section [Symbol, nil] One of SPEC_SECTIONS, or nil for the whole specification
390
+ def action_plugins_spec(plugin: nil, section: nil, **)
391
+ specs = plugin_specs
392
+ return Result::ObjectList.new(specs.map { |name, spec| {'name' => name}.merge(spec.except(*SPEC_SECTIONS.map(&:to_s), 'notes')) }, fields: SPEC_LIST_FIELDS) if plugin.nil?
393
+ name = specs.keys.find { |key| key.casecmp?(plugin) }
394
+ if name.nil?
395
+ similar = specs.keys.select { |key| key.downcase.include?(plugin.downcase) }
396
+ raise Cli::BadArgument, "Unknown plugin: #{plugin}#{", similar: #{similar.join(', ')}" unless similar.empty?}, use `plugins spec` to list plugins"
397
+ end
398
+ return Result::SingleObject.new({'name' => name}.merge(specs[name])) if section.nil?
399
+ Result::ObjectList.new(specs[name][section.to_s].map { |key, value| {'name' => key}.merge(value) })
400
+ end
401
+
402
+ # Without dependencies, the workflow file (YAML) is displayed.
403
+ # With dependencies, the package (binary) is saved in the destination folder.
404
+ def action_workflows_export(workflow_id:, export:, **)
405
+ dependencies = export.transform_keys(&:to_s)['dependencies'].eql?(true)
406
+ # Orchestrator exports dependencies for any value of the parameter
407
+ begin
408
+ resp = call_ao("export_workflow/#{workflow_id}", accept: nil, query: dependencies ? {export_with_dependencies: true} : nil, http: true)
409
+ rescue Rest::CallError
410
+ # Orchestrator 4.1.6 fails to build the package of some workflows (it is written in the folder it packs)
411
+ Log.log.warn('Export with dependencies failed: export the workflow and each sub-workflow without dependencies instead') if dependencies
412
+ raise
413
+ end
414
+ return Result::Text.new(resp.body) unless dependencies
415
+ disposition = resp['Content-Disposition']
416
+ file_name = disposition && Rest.parse_header(disposition)[:parameters][:filename]
417
+ file_path = File.join(transfer.destination_folder(Transfer::Spec::DIRECTION_RECEIVE), File.basename(file_name || "Workflow_#{workflow_id}.wkf"))
418
+ File.binwrite(file_path, resp.body)
419
+ Result::Status.new("Saved to: #{file_path}")
420
+ end
421
+
422
+ # Check of a workflow file, for problems that Orchestrator does not detect on import: it imports an invalid workflow, which then fails to publish or to run.
423
+ # This is the only place that knows how the check is done: it is done locally, because Orchestrator has no such API.
424
+ # When it has one (same request as `import_workflow`), replace the body by a call, e.g.
425
+ # `call_ao('check_workflow', body: [['import_file', content, {filename: file_name}], ['import_file_name', file_name]], content_type: Mime::MULTIPART)`
426
+ # The response is a Hash, see {OrchestratorWorkflowChecker}.
427
+ # Templates and inputs of steps are checked with the specifications of plugins (see `plugins spec`), when they can be read.
428
+ # @param file_path [String] Workflow file (`.yml` or `.wkf`)
429
+ # @return [Hash] `valid` (Boolean), `file_name`, `issues` (list of Hash: `severity`, `code`, `file`, `path`, `message`)
430
+ def check_workflow(file_path)
431
+ require 'aspera/orchestrator_workflow_checker'
432
+ specs = nil
433
+ specs_error = nil
434
+ begin
435
+ specs = plugin_specs
436
+ rescue StandardError => e
437
+ specs_error = e.message
438
+ end
439
+ response = OrchestratorWorkflowChecker.check(file_name: File.basename(file_path), content: File.binread(file_path), specs: specs)
440
+ unless specs_error.nil?
441
+ response['issues'].push({
442
+ 'severity' => 'warning',
443
+ 'code' => 'plugins.specs_unavailable',
444
+ 'path' => 'templates',
445
+ 'message' => "plugin specifications not read (#{specs_error}): templates and inputs of steps are not checked"
446
+ })
447
+ end
448
+ response
449
+ end
450
+
451
+ # @param file_path [String] Workflow file
452
+ # @return [Result] The issues found, or a status if none
453
+ def action_workflows_check_import(file_path:, **)
454
+ response = check_workflow(file_path)
455
+ return Result::Status.new('no problem found') if response['issues'].empty?
456
+ Result::ObjectList.new(response['issues'], fields: %w[severity code file path message])
457
+ end
458
+
459
+ # Conflicts and missing plugins found by Orchestrator, as one row per name to resolve.
460
+ # Orchestrator does not check that the file exists: for a wrong path, it returns the same empty result as for a file without conflict.
461
+ # @return [Hash{String=>Object}] Items returned by Orchestrator, by name (`subwf constraints`, `missing plugins`...)
462
+ def server_constraints(filename, add_as_revision = nil)
463
+ list = call_ao('find_constraints', query: {file: filename, wf_id: add_as_revision}.compact)
464
+ Aspera.assert_type(list, Array, type: Cli::Error) { 'response of find_constraints' }
465
+ list.grep(Hash).reduce({}, :merge)
466
+ end
467
+
468
+ # Keys of the conflicts in Orchestrator response => keys of the argument of `import_with_constraints`
469
+ CONSTRAINT_KEYS = {
470
+ 'subwf constraints' => 'subwf_constraints',
471
+ 'action template constraints' => 'action_template_constraints',
472
+ 'remote node constraints' => 'remote_node_constraints'
473
+ }.freeze
474
+ private_constant :CONSTRAINT_KEYS
475
+
476
+ def action_workflows_find_constraints(filename:, add_as_revision: nil, **)
477
+ found = server_constraints(filename, add_as_revision)
478
+ rows = CONSTRAINT_KEYS.flat_map do |api_key, key|
479
+ (found[api_key] || {}).map { |name, choices| {'type' => key, 'name' => name, 'choices' => choices} }
480
+ end
481
+ Array(found['missing plugins']).each do |name|
482
+ rows.push({'type' => 'missing_plugins', 'name' => name, 'choices' => 'auto_enable_missing_plugins: true'})
483
+ end
484
+ return Result::Status.new('no conflict found (Orchestrator gives the same answer if the file does not exist)') if rows.empty?
485
+ Result::ObjectList.new(rows, fields: %w[type name choices])
486
+ end
487
+
488
+ # Orchestrator fails or decides silently in these cases, which the web interface prevents:
489
+ # - plugins to enable but `auto_enable_missing_plugins` not true: the import fails with an internal error
490
+ # - resolution given for some conflicts only: the others are imported as copies, without error
491
+ # @param payload [Hash] Argument of `import_with_constraints`, with string keys
492
+ def check_import_constraints(payload)
493
+ found = server_constraints(payload['filename'], payload['add_as_revision'])
494
+ problems = []
495
+ missing = Array(found['missing plugins'])
496
+ problems.push("plugins are not enabled: #{missing.join(', ')}, set auto_enable_missing_plugins to true") unless missing.empty? || payload['auto_enable_missing_plugins'].eql?(true)
497
+ CONSTRAINT_KEYS.each do |api_key, key|
498
+ chosen = payload[key]
499
+ next if chosen.nil? || chosen.empty?
500
+ unresolved = (found[api_key] || {}).keys - chosen.keys.map(&:to_s)
501
+ problems.push("#{key} not resolved for: #{unresolved.join(', ')} (see `workflows find_constraints`)") unless unresolved.empty?
502
+ end
503
+ raise Cli::BadArgument, "Import refused: #{problems.join('; ')}" unless problems.empty?
504
+ end
505
+
506
+ # The file is uploaded as a multipart form
507
+ # @param import [Hash] Options: `folder_id`, `publish`, `link_to_subworkflows`
508
+ def action_workflows_import(file_path:, import: {}, **)
509
+ import = import.transform_keys(&:to_s)
510
+ check_workflow(file_path)['issues'].each do |issue|
511
+ Log.log.warn { "#{issue['severity']}: #{[issue['file'], issue['path']].compact.join(' ')}: #{issue['message']} (#{issue['code']})" }
512
+ end
513
+ file_name = File.basename(file_path)
514
+ form = [
515
+ ['import_file', File.binread(file_path), {filename: file_name}],
516
+ ['import_file_name', file_name]
517
+ ]
518
+ form.push(['folder_id', import['folder_id'].to_s]) if import.key?('folder_id')
519
+ # `false` overrides the server option `publish_api_imported_workflow`
520
+ form.push(['publish_workflow', import['publish'].to_s]) if import.key?('publish')
521
+ # Orchestrator takes any value as true
522
+ form.push(%w[link_to_subworkflows true]) if import['link_to_subworkflows'].eql?(true)
523
+ result = call_ao('import_workflow', body: form, content_type: Mime::MULTIPART)
524
+ # Other responses: nothing was imported
525
+ if result.key?('plugins')
526
+ raise Cli::Error, "Plugins could not be enabled: #{Array(result['plugins']).join(', ')}" \
527
+ "#{" (missing dependencies: #{Array(result['missing_deps']).join(', ')})" unless Array(result['missing_deps']).empty?}"
528
+ end
529
+ if result.key?('dependencies')
530
+ dependencies = result['dependencies'].values.flatten.filter_map { |d| "#{d['entity']} #{d['id_value']}" if d.is_a?(Hash) }
531
+ raise Cli::Error, "Workflow dependencies are not included in the file (export with dependencies to a .wkf file): #{dependencies.uniq.join(', ')}"
532
+ end
533
+ Result::SingleObject.new(result['workflow'])
534
+ end
535
+
536
+ # Items of the array expected by `import_with_constraints`, in this order: argument key => API key
537
+ IMPORT_CONSTRAINT_ITEMS = {
538
+ 'filename' => 'filename',
539
+ 'add_as_revision' => 'add as revision',
540
+ 'subwf_constraints' => 'subwf constraints',
541
+ 'action_template_constraints' => 'action template constraints',
542
+ 'remote_node_constraints' => 'remote node constraints',
543
+ 'auto_enable_missing_plugins' => 'Auto-enable missing plugins?'
544
+ }.freeze
545
+ private_constant :IMPORT_CONSTRAINT_ITEMS
546
+
547
+ # The API expects an array of single-key objects, in a fixed order
548
+ def action_workflows_import_with_constraints(payload:, **)
549
+ payload = payload.transform_keys(&:to_s)
550
+ check_import_constraints(payload)
551
+ body = IMPORT_CONSTRAINT_ITEMS.map { |key, api_key| {api_key => payload.fetch(key) { key.end_with?('_constraints') ? {} : nil }} }
552
+ Result::SingleObject.new(call_ao('import_with_constraints', body: body)['workflow'])
553
+ end
554
+
242
555
  # Keys of argument `execution` of `workflows start`
243
556
  WORKFLOW_START_KEYS = %w[synchronous step variable].freeze
244
557
  private_constant :WORKFLOW_START_KEYS
@@ -269,8 +582,8 @@ module Aspera
269
582
  # implicitly, call is synchronous
270
583
  query_params[:synchronous] = true
271
584
  end
272
- result_data = call_ao('initiate', body: json_body, query: query_params.empty? ? nil : query_params)
273
- query_params[:synchronous] ? Result::Text.new(result_data) : Result::SingleObject.new(result_data)
585
+ # Work order information, or value of the explicit output (any JSON value)
586
+ Result.auto(call_ao('initiate', body: json_body, query: query_params.empty? ? nil : query_params))
274
587
  end
275
588
  end
276
589
  end
@@ -66,10 +66,12 @@ module Aspera
66
66
  # @param app_url [String] Tested URL
67
67
  # @return [Hash] :preset_value, :test_args
68
68
  def wizard(wizard, app_url)
69
+ username = options.get_option(:username, mandatory: true)
69
70
  return {
71
+ preset_ids: [username],
70
72
  preset_value: {
71
73
  url: app_url,
72
- username: options.get_option(:username, mandatory: true),
74
+ username: username,
73
75
  password: options.get_option(:password, mandatory: true)
74
76
  },
75
77
  test_args: 'browse /'