aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -1,10 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/cli/plugins/basic_auth'
4
- require 'aspera/cli/special_values'
5
- require 'aspera/nagios'
6
- require 'aspera/log'
7
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'basic_auth'
7
+ require_relative '../special_values'
8
+ require_relative '../../nagios'
9
+ require_relative '../../log'
10
+ require_relative '../../assert'
11
+ require_relative '../../transfer/spec'
12
+ require_relative '../../schema/registry'
8
13
  require 'xmlsimple'
9
14
 
10
15
  module Aspera
@@ -14,7 +19,54 @@ module Aspera
14
19
  class Orchestrator < BasicAuth
15
20
  STANDARD_PATH = '/aspera/orchestrator'
16
21
  TEST_ENDPOINT = 'api/remote_node_ping'
17
- private_constant :STANDARD_PATH, :TEST_ENDPOINT
22
+ # Raw Markdown of the workflow authoring documents
23
+ DOC_BASE_URL = 'https://raw.githubusercontent.com/laurent-martin/aspera-orchestrator-plugins-doc/main/docs'
24
+ # Short link to the development guide: tried first, so that the location of the document can change
25
+ DOC_GUIDE_LINK = 'https://ibm.biz/aspera-orchestrator-book'
26
+ # Documents of command `workflows development`: name => URLs, tried in order
27
+ DOC_URLS = {
28
+ guide: [DOC_GUIDE_LINK, "#{DOC_BASE_URL}/orchestrator-development-guide.md"].freeze,
29
+ templates: ["#{DOC_BASE_URL}/action-template-reference.md"].freeze,
30
+ manual: ["#{DOC_BASE_URL}/Orchestrator_Plugin_Manual.md"].freeze
31
+ }.freeze
32
+ # Description of the documents of command `workflows development`
33
+ DOC_DESCRIPTIONS = {
34
+ guide: 'Aspera Orchestrator Development Guide: concepts, designer, workflow files, recipes, plugins and portlets (large: use `toc` and `section`)',
35
+ templates: 'Action Template Reference: template attributes, inputs and outputs of each action plugin',
36
+ manual: 'Orchestrator Plugin Manual: functional description and setup of each plugin (large: use `toc` and `section`)'
37
+ }.freeze
38
+ # An unknown short link ends on a HTML page, but the documents are Markdown (they may start with a comment)
39
+ HTML_PAGE = /\A\s*<(!doctype|html)/i
40
+ # Default maximum level of headings in the table of contents: guide sections, one entry per plugin
41
+ DOC_TOC_LEVELS = {
42
+ guide: 3,
43
+ templates: 2,
44
+ manual: 3
45
+ }.freeze
46
+ # Sections of a plugin specification, listed by command `plugins spec show`
47
+ SPEC_SECTIONS = %i[attributes inputs outputs].freeze
48
+ # Fields of the list of plugins of command `plugins spec toc`
49
+ SPEC_LIST_FIELDS = %w[name category version display_name manual_anchor].freeze
50
+ # Text value that is a YAML document (Ruby objects serialized by Orchestrator)
51
+ YAML_DUMP = /\A---\s/
52
+ # Types of monitors of `list_monitors` (`monitor_type`) and of `monitor_snapshot` (path)
53
+ MONITOR_TYPES = %i[workflow workorder folder node script].freeze
54
+ MONITOR_SNAPSHOT_TYPES = %i[workflows workorders folders nodes scripts].freeze
55
+ # Processes and commands of `control_process`
56
+ PROCESS_TYPES = %i[engine monitor worker].freeze
57
+ PROCESS_COMMANDS = %i[start stop kill].freeze
58
+ # XML options for the status of processes (`processes_status` and `control_process`)
59
+ PROCESSES_XML_OPTS = {'ForceArray' => %w[node process]}.freeze
60
+ # Moves of `reorder_queued_item` and `bulk_reorder_queue`
61
+ QUEUE_MOVES = %i[up down first last].freeze
62
+ # `ManagedResource` of Orchestrator repeats key `id` in each resource
63
+ RESOURCE_JSON_OPTS = {allow_duplicate_key: true}.freeze
64
+ # Id segment of the path, for endpoints routed only as `api/<endpoint>/<id>` that do not use it
65
+ ANY_ID = '0'
66
+ # Login of the current user, for `list_tasks`
67
+ SELF_USER = 'self'
68
+ private_constant :STANDARD_PATH, :TEST_ENDPOINT, :DOC_BASE_URL, :DOC_GUIDE_LINK, :DOC_URLS, :DOC_DESCRIPTIONS, :HTML_PAGE, :DOC_TOC_LEVELS, :SPEC_SECTIONS, :SPEC_LIST_FIELDS, :YAML_DUMP,
69
+ :MONITOR_TYPES, :MONITOR_SNAPSHOT_TYPES, :PROCESS_TYPES, :PROCESS_COMMANDS, :PROCESSES_XML_OPTS, :QUEUE_MOVES, :RESOURCE_JSON_OPTS, :ANY_ID, :SELF_USER
18
70
 
19
71
  class << self
20
72
  # @return [Hash,NilClass]
@@ -46,70 +98,232 @@ module Aspera
46
98
  # @param app_url [String] Tested URL
47
99
  # @return [Hash] :preset_value, :test_args
48
100
  def wizard(wizard, app_url)
101
+ username = options.get_option(:username, mandatory: true)
49
102
  return {
103
+ preset_ids: [username],
50
104
  preset_value: {
51
105
  url: app_url,
52
- username: options.get_option(:username, mandatory: true),
106
+ username: username,
53
107
  password: options.get_option(:password, mandatory: true)
54
108
  },
55
109
  test_args: 'workflows list'
56
110
  }
57
111
  end
58
112
 
59
- option :apikey, description: 'API key (exclusive with username/password)'
113
+ option :apikey, description: 'API key of the user (exclusive with password), requires username'
60
114
  option :auth_style, description: 'Authentication style', allowed: %i[basic query token], default: :token
61
115
  option :ret_style, description: 'Method to specify the expected response format ("Accept")', allowed: %i[header query path], default: :query
62
116
  option :result, description: "Specify result value as: 'work_step:parameter'", deprecation: {last: '4.27.2', message: 'use keys `step` and `variable` of argument `execution` of `workflows start`'}
63
117
  option :synchronous, description: 'Wait for completion', allowed: Type::BOOLEAN, deprecation: {last: '4.27.2', message: 'use key `synchronous` of argument `execution` of `workflows start`'}
118
+ option :plugin_specs, description: 'URL or path of the plugin specifications (YAML) used by `plugins spec`', default: "#{DOC_BASE_URL}/plugins.yml"
64
119
 
65
120
  # Call orchestrator API (handles ret_style negotiation and XML parsing)
66
- # @param endpoint [String] the endpoint to call
67
- # @param body [Hash] the body to pass (JSON); also implies POST
68
- # @param format [String] the format to request, 'json', 'xml', nil
69
- # @param query [Hash] the arguments to pass as query parameters
70
- # @param xml_arrays [Boolean] if true, force arrays in xml parsing
71
- # @param http [Boolean] if true, returns the HttpResponse, else
72
- def call_ao(endpoint, body: nil, format: 'json', query: nil, xml_arrays: true, http: false)
73
- call_args = {operation: body.nil? ? 'GET' : 'POST', subpath: "api/#{endpoint}", ret: :both, query: {}}
74
- call_args.merge!(body: body, content_type: Mime::JSON) unless body.nil? # rubocop:disable Performance/RedundantMerge
121
+ # @param endpoint [String] the endpoint to call
122
+ # @param body [Hash, Array] the body to pass; also implies POST
123
+ # @param content_type [String] the body type (Mime::JSON or Mime::MULTIPART)
124
+ # @param accept [String] the response type to request (Mime::JSON or Mime::XML), or nil
125
+ # @param query [Hash] the arguments to pass as query parameters
126
+ # @param xml_opts [Hash] options of `XmlSimple.xml_in` to parse an XML response
127
+ # @param json_opts [Hash] options of `JSON.parse` to parse a JSON response
128
+ # @param http [Boolean] if true, returns the HttpResponse, else
129
+ def call_ao(endpoint, body: nil, content_type: Mime::JSON, accept: Mime::JSON, query: nil, xml_opts: {}, json_opts: {}, http: false)
130
+ call_args = {operation: body.nil? ? 'GET' : 'POST', subpath: "api/#{endpoint}", ret: :both, query: {}, json_opts: json_opts}
131
+ call_args.merge!(body: body, content_type: content_type) unless body.nil? # rubocop:disable Performance/RedundantMerge
75
132
  ret_style = options.get_option(:ret_style, mandatory: true)
76
133
  call_args[:query].merge!(query) unless query.nil?
77
- unless format.nil?
134
+ unless accept.nil?
135
+ # 'json' or 'xml'
136
+ short_type = accept.split('/').last
78
137
  case ret_style
79
138
  when :header
80
- call_args[:headers] = {'Accept' => "application/#{format}"}
139
+ call_args[:headers] = {'Accept' => accept}
81
140
  when :query
82
- call_args[:query][:format] = format
141
+ call_args[:query][:format] = short_type
83
142
  when :path
84
- call_args[:subpath] = "#{call_args[:subpath]}.#{format}"
143
+ call_args[:subpath] = "#{call_args[:subpath]}.#{short_type}"
85
144
  else Aspera.error_unexpected_value(ret_style) { 'ret_style' }
86
145
  end
87
146
  end
88
- add_query = query_read_delete
147
+ add_query = query_read_delete(schema: api_query_schema(endpoint, call_args[:operation]))
89
148
  call_args[:query].merge!(add_query.symbolize_keys) unless add_query.nil?
90
149
  data, resp = api_orch.call(**call_args)
91
150
  return resp if http
92
- result = format.eql?('xml') ? XmlSimple.xml_in(resp.body, {'ForceArray' => xml_arrays}) : data
151
+ result = accept.eql?(Mime::XML) ? XmlSimple.xml_in(resp.body, xml_opts) : data
93
152
  Log.dump(:data, result)
94
153
  return result
95
154
  end
96
155
 
97
- private :call_ao
156
+ # Schema of the query parameters of an endpoint, in the OpenAPI description of Orchestrator (for `--query=help`)
157
+ # @param endpoint [String] Endpoint, with an id or not, e.g. `journal_files/journal`
158
+ # @param operation [String] HTTP method
159
+ # @return [String, nil] Schema path, or `nil` if the endpoint is not described
160
+ def api_query_schema(endpoint, operation)
161
+ name, id = endpoint.split('/', 2)
162
+ paths = Schema::Registry.instance.reader(Schema::Registry::ORCHESTRATOR)['paths']
163
+ # Orchestrator routes both `api/<name>` and `api/<name>/<id>`: prefer the form used
164
+ candidates = ["api/#{name}", "api/#{name}/{id}"]
165
+ candidates.reverse! unless id.nil?
166
+ path = candidates.find { |candidate| paths.dig("/#{candidate}", operation.downcase) }
167
+ path && Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, path, method: operation.downcase)
168
+ end
169
+
170
+ private :call_ao, :api_query_schema
98
171
 
99
172
  # --- DSL ---
100
173
 
101
174
  command :health, description: 'Check Orchestrator API health'
102
- command :info, description: 'Check that Orchestrator responds (ping)', action: ->(**) { Result::SingleObject.new(call_ao('remote_node_ping', format: 'xml', xml_arrays: false)) }
103
- command :processes, description: 'Show Orchestrator background process status', action: ->(**) { Result::ObjectList.new(call_ao('processes_status', format: 'xml')['node'].flat_map { |n| n['process'] }) }
104
- command :monitors, description: 'Show Orchestrator monitor snapshot', action: ->(**) { Result::SingleObject.new(call_ao('monitor_snapshot')['monitor']) }
175
+ command :info, description: 'Check that Orchestrator responds (ping)', action: ->(**) { Result::SingleObject.new(call_ao('remote_node_ping', accept: Mime::XML, xml_opts: {'ForceArray' => false})) }
176
+ command :processes, description: 'Show Orchestrator background process status', action: ->(**) { processes_result(call_ao('processes_status', accept: Mime::XML, xml_opts: PROCESSES_XML_OPTS)) }
177
+ command :control, description: 'Start, stop or kill the engine, the monitor or workers of Orchestrator, and show the process status',
178
+ arguments: [
179
+ {name: :process, type: Symbol, allowed: PROCESS_TYPES},
180
+ {name: :operation, type: Symbol, allowed: PROCESS_COMMANDS},
181
+ {name: :worker_ids, mandatory: false}
182
+ ],
183
+ action: ->(process:, operation:, worker_ids: nil, **) { processes_result(call_ao("control_process/#{process}", query: {command: operation, worker_id: worker_ids}.compact, accept: Mime::XML, xml_opts: PROCESSES_XML_OPTS)) }
184
+ command :users, description: 'Manage users'
185
+ command :alerts, description: 'Alerts of the categories the user is subscribed to'
186
+ command :tasks, description: 'Tasks: user inputs waiting for a user'
187
+ command :monitors, description: 'Monitors of workflows, work orders, folders, remote nodes and scripts'
188
+ command :queues, description: 'Managed queues'
189
+ command :portlets, description: 'Portlets of the web interface'
190
+ command :pools, description: 'Resource pools'
191
+ command :journal, description: 'Journal books'
105
192
  command :workorders, description: 'Manage work orders'
106
- command :workstep, description: 'Manage work steps'
193
+ command :worksteps, description: 'Manage work steps'
194
+
195
+ commands_under :alerts do
196
+ command :list, description: 'List alerts (option query: `category`, `count_only`)',
197
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/alerts/{id}'),
198
+ action: ->(**) { Result.auto(call_ao("alerts/#{ANY_ID}")) }
199
+ end
200
+
201
+ commands_under :tasks do
202
+ command :list, description: 'List tasks waiting for a user (only an administrator can list the tasks of another user)',
203
+ arguments: [{name: :user, mandatory: false, default: SELF_USER}],
204
+ action: ->(user:, **) { Result::ObjectList.new(call_ao("list_tasks/#{URI.encode_uri_component(user)}")['tasks'] || []) }
205
+ command :show, description: 'Show a task: its user inputs and its parameters. Id is a task id, or WO<work order id> with the step name when the work order has several',
206
+ arguments: [
207
+ {name: :task_id},
208
+ {name: :step_name, mandatory: false}
209
+ ],
210
+ action: ->(task_id:, step_name: nil, **) { Result::SingleObject.new(call_ao("task_details/#{task_id}", query: {step_name: step_name, Step_name: step_name}.compact)) }
211
+ command :submit, description: 'Complete a task with values of its user inputs (all required inputs must be given)',
212
+ arguments: [
213
+ {name: :task_id, type: :identifier},
214
+ {name: :inputs, type: Hash, schema: 'orchestrator:components.schemas.TaskUserInputs'}
215
+ ],
216
+ action: ->(task_id:, inputs:, **) { Result::SingleObject.new(call_ao("submit_task/#{task_id}", query: inputs.transform_keys { |name| :"outputs[#{name}]" })) }
217
+ end
218
+
219
+ commands_under :monitors do
220
+ command :snapshot, description: 'Show status of monitors, of all types or of one type',
221
+ arguments: [{name: :type, type: Symbol, mandatory: false, allowed: MONITOR_SNAPSHOT_TYPES}],
222
+ action: ->(type: nil, **) { Result::SingleObject.new(call_ao(type.nil? ? 'monitor_snapshot' : "monitor_snapshot/#{type}")['monitor']) }
223
+ command :list, description: 'List monitors of a type: remote node monitors give the names of remote nodes',
224
+ arguments: [{name: :type, type: Symbol, allowed: MONITOR_TYPES}]
225
+ command :activate, description: 'Activate a monitor',
226
+ arguments: [{name: :type, type: Symbol, allowed: MONITOR_TYPES}, {name: :monitor_id, type: :identifier}],
227
+ action: ->(type:, monitor_id:, **) { Result::SingleObject.new(call_ao("activate_monitor/#{monitor_id}", query: {monitor_type: type})) }
228
+ command :deactivate, description: 'Deactivate a monitor',
229
+ arguments: [{name: :type, type: Symbol, allowed: MONITOR_TYPES}, {name: :monitor_id, type: :identifier}],
230
+ action: ->(type:, monitor_id:, **) { Result::SingleObject.new(call_ao("deactivate_monitor/#{monitor_id}", query: {monitor_type: type})) }
231
+ end
232
+
233
+ commands_under :portlets do
234
+ command :list, description: 'List portlets with their version',
235
+ action: ->(**) { Result::ObjectList.new(call_ao('portlet_version')['Portlet'] || []) }
236
+ command :show, description: 'Show the version of a portlet',
237
+ arguments: [{name: :portlet}],
238
+ action: ->(portlet:, **) { Result::ObjectList.new(call_ao("portlet_version/#{URI.encode_uri_component(portlet)}")['Portlet'] || []) }
239
+ end
240
+
241
+ commands_under :users do
242
+ command :create, description: 'Create a user',
243
+ arguments: [{name: :user, type: Hash, schema: Schema::Registry.req_body(Schema::Registry::ORCHESTRATOR, 'api/create_user/{id}.post')}],
244
+ action: ->(user:, **) { Result::Status.new(call_ao("create_user/#{ANY_ID}", body: user).dig('create_user', 'success').to_s) }
245
+ end
246
+
247
+ commands_under :queues do
248
+ command :stats, description: 'Statistics of the execution queue of the engine',
249
+ action: ->(**) { Result::SingleObject.new(call_ao("execution_queue_stats/#{ANY_ID}")) }
250
+ command :list, description: 'List managed queues',
251
+ action: ->(**) { Result::ObjectList.new(call_ao('queues_list').dig('Queues', 'queue') || []) }
252
+ command :items, description: 'List items of a queue',
253
+ arguments: [{name: :queue}],
254
+ action: ->(queue:, **) { Result::ObjectList.new(call_ao("queued_items/#{URI.encode_uri_component(queue)}")['queued_items'] || []) }
255
+ command :lookup, description: 'Find queued items, with option query: by item id, by item or by description (optionally in a queue)',
256
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/lookup_queued_item'),
257
+ action: ->(**) { Result::ObjectList.new(call_ao('lookup_queued_item')['queued_items'] || []) }
258
+ command :add, description: 'Add an item to a queue (priority, weight, description and tags with option query)',
259
+ arguments: [{name: :queue}, {name: :item}],
260
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/queue_item/{id}'),
261
+ action: ->(queue:, item:, **) { Result::SingleObject.new(call_ao("queue_item/#{URI.encode_uri_component(queue)}", query: {queued_item: item})) }
262
+ command :move, description: "Move a queued item: #{QUEUE_MOVES.join(', ')}",
263
+ arguments: [{name: :operation, type: Symbol, allowed: QUEUE_MOVES}, {name: :item_id, type: :identifier}],
264
+ action: ->(operation:, item_id:, **) { Result::ObjectList.new(call_ao('reorder_queued_item', query: {operation: operation, queued_item_id: item_id})['queued_items'] || []) }
265
+ command :bulk_move, description: "Move several queued items (comma-separated ids): #{QUEUE_MOVES.join(', ')}",
266
+ arguments: [{name: :operation, type: Symbol, allowed: QUEUE_MOVES}, {name: :item_ids}],
267
+ action: ->(operation:, item_ids:, **) { Result::ObjectList.new(call_ao('bulk_reorder_queue', query: {operation: operation, queued_item_ids: item_ids})['queued_items'].map { |entry| entry['queued_item'] }) }
268
+ # Asks for XML: on success, the JSON response of this endpoint is a plain text that is not valid JSON
269
+ command :delete, description: 'Delete a queued item',
270
+ arguments: [{name: :queue}, {name: :item_id, type: :identifier}],
271
+ action: ->(queue:, item_id:, **) { Result::Status.new(call_ao('delete_queued_item', query: {queue_name: queue, queued_item_id: item_id}, accept: Mime::XML, xml_opts: {'ForceArray' => false}).fetch('Success')) }
272
+ end
273
+
274
+ commands_under :pools do
275
+ command :status, description: 'Show resources of a resource pool and their status',
276
+ arguments: [{name: :pool_name}],
277
+ action: ->(pool_name:, **) { Result::ObjectList.new(call_ao("pool_status/#{ANY_ID}", query: {name: pool_name}, json_opts: RESOURCE_JSON_OPTS)['ManagedResource'] || []) }
278
+ command :add_resource, description: 'Add a resource to a resource pool (capacity and tags with option query)',
279
+ arguments: [{name: :pool_name}, {name: :resource_name}],
280
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/add_resource/{id}'),
281
+ action: ->(pool_name:, resource_name:, **) { Result::SingleObject.new(call_ao("add_resource/#{ANY_ID}", query: {pool_name: pool_name, resource_name: resource_name}, json_opts: RESOURCE_JSON_OPTS)['ManagedResource']) }
282
+ command :resource_capacity, description: 'Change the capacity of a resource',
283
+ arguments: [{name: :pool_name}, {name: :resource_name}, {name: :capacity, type: Integer}],
284
+ action: ->(pool_name:, resource_name:, capacity:, **) { Result::SingleObject.new(call_ao("edit_resource_capacity/#{capacity}", query: {pool_name: pool_name, resource_name: resource_name}, json_opts: RESOURCE_JSON_OPTS)['ManagedResource']) }
285
+ command :capacity, description: 'Change the capacity of a resource pool',
286
+ arguments: [{name: :pool_name}, {name: :capacity, type: Integer}],
287
+ action: ->(pool_name:, capacity:, **) { Result::ObjectList.new(call_ao("edit_pool_capacity/#{capacity}", query: {pool_name: pool_name}, json_opts: RESOURCE_JSON_OPTS)['ManagedResource']) }
288
+ command :online, description: 'Take a resource online',
289
+ arguments: [{name: :pool_name}, {name: :resource_name}],
290
+ action: ->(pool_name:, resource_name:, **) { Result::SingleObject.new(call_ao("resource_online/#{ANY_ID}", query: {pool_name: pool_name, resource_name: resource_name})['ManagedResource']) }
291
+ command :offline, description: 'Take a resource offline',
292
+ arguments: [{name: :pool_name}, {name: :resource_name}],
293
+ action: ->(pool_name:, resource_name:, **) { Result::SingleObject.new(call_ao("resource_offline/#{ANY_ID}", query: {pool_name: pool_name, resource_name: resource_name})['ManagedResource']) }
294
+ end
295
+
296
+ commands_under :journal do
297
+ command :files, description: 'List files recorded in a journal book (filters with option query)',
298
+ arguments: [{name: :book}],
299
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/journal_files/{id}')
300
+ end
107
301
 
108
302
  commands_under :plugins do
109
- command :list, description: 'Show Orchestrator plugin versions', action: ->(**) { Result::ObjectList.new(call_ao('plugin_version')['Plugin']) }
303
+ command :list, description: 'List plugins with their version', action: ->(**) { Result::ObjectList.new(call_ao('plugin_version')['Plugin']) }
304
+ command :reload, description: 'Reload a plugin (class name), or all enabled plugins',
305
+ arguments: [{name: :plugin, mandatory: false}],
306
+ action: ->(plugin: nil, **) { Result::SingleObject.new(call_ao('reload_plugin', query: {action_type: plugin}.compact)) }
307
+ command :enable, description: 'Enable a plugin (class name), or all plugins. Orchestrator 4.1.6: never for a plugin imported with `import`, it deletes its folder',
308
+ arguments: [{name: :plugin, mandatory: false}],
309
+ action: ->(plugin: nil, **) { Result::SingleObject.new(call_ao("enable_plugin/#{ANY_ID}", query: {action_type: plugin}.compact)) }
310
+ command :import, description: 'Import and enable a plugin from an archive present on the Orchestrator host, readable by aspweb. Orchestrator 4.1.6: its migration files stay in the load folder, and prevent the next start',
311
+ arguments: [{name: :file_path}],
312
+ action: ->(file_path:, **) { Result::SingleObject.new(call_ao("import_plugin/#{ANY_ID}", query: {file: file_path})) }
110
313
  command :reload_set, description: 'Reload a set of plugins',
111
314
  arguments: [{name: :plugin_set, type: Hash, schema: 'opts:components.schemas.OrchestratorReloadPluginSet'}],
112
315
  action: ->(plugin_set:, **) { Result::SingleObject.new(call_ao('reload_plugin_set', body: plugin_set.transform_keys(&:to_sym))) }
316
+ # Specifications of action plugins, read from GitHub (no access to Orchestrator)
317
+ commands_under :spec, description: 'Specifications of action plugins, from GitHub (no access to Orchestrator)' do
318
+ command :toc, description: 'List the plugins: name, category, version, display name, anchor of the section in the Plugin Manual (filter with option select)',
319
+ action: ->(**) { plugin_spec_toc }
320
+ command :show, description: "Show the specification of a plugin, with `manual_anchor` for `workflows development manual section`: whole, or one section (#{SPEC_SECTIONS.join(', ')})",
321
+ arguments: [
322
+ {name: :plugin},
323
+ {name: :section, type: Symbol, mandatory: false, allowed: SPEC_SECTIONS}
324
+ ],
325
+ action: ->(plugin:, section: nil, **) { plugin_spec_show(plugin, section) }
326
+ end
113
327
  end
114
328
 
115
329
  commands_under :workflows do
@@ -120,8 +334,9 @@ module Aspera
120
334
  command :inputs, description: 'Show input specification of a workflow',
121
335
  arguments: [{name: :workflow_id, type: :identifier}],
122
336
  action: ->(workflow_id:, **) { Result::SingleObject.new(call_ao("workflow_inputs_spec/#{workflow_id}")['workflow_inputs_spec']) }
123
- command :details, description: 'Show detailed running status of a workflow',
337
+ command :details, description: 'Show detailed running status of a workflow (period with option query: `start_date`, `end_date`)',
124
338
  arguments: [{name: :workflow_id, type: :identifier}],
339
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/workflow_details/{id}'),
125
340
  action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao("workflow_details/#{workflow_id}")['workflows']['workflow']['statuses']) }
126
341
  command :start, description: 'Start a workflow: create a work order (sync or async)',
127
342
  arguments: [
@@ -129,48 +344,109 @@ module Aspera
129
344
  {name: :parameters, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorInitiateParameters'},
130
345
  {name: :execution, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowStart'}
131
346
  ]
132
- command :import, description: 'Import a workflow',
133
- arguments: [{name: :workflow, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWorkflow'}],
134
- action: ->(workflow:, **) { Result::SingleObject.new(call_ao('import_workflow', body: workflow.transform_keys(&:to_sym))) }
347
+ command :import, description: 'Import a workflow from a file created by `workflows export`, local problems are reported as warnings (see `check_import`)',
348
+ arguments: [
349
+ {name: :file_path, complete: :file},
350
+ {name: :import, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowImport'}
351
+ ]
352
+ command :check_import, description: 'Check a workflow file (no access to Orchestrator): rules that Orchestrator does not check on import, but that make the workflow fail to publish or to run',
353
+ arguments: [{name: :file_path, complete: :file}]
354
+ command :find_constraints, description: 'List the conflicts and missing plugins of a workflow file present on the Orchestrator host, before `import_with_constraints`',
355
+ arguments: [
356
+ {name: :filename},
357
+ {name: :add_as_revision, type: :identifier, mandatory: false}
358
+ ]
135
359
  command :publish, description: 'Publish a workflow',
136
360
  arguments: [{name: :workflow_id, type: :identifier}],
137
- action: ->(workflow_id:, **) { Result::SingleObject.new(call_ao('publish_workflow', body: {workflow_id: workflow_id.to_i})) }
138
- command :import_with_constraints, description: 'Import a workflow with constraint resolution',
139
- arguments: [{name: :payload, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWithConstraints'}],
140
- action: ->(payload:, **) { Result::SingleObject.new(call_ao('import_with_constraints', body: payload.transform_keys(&:to_sym))) }
141
- command :export, description: 'Export a workflow',
142
- arguments: [{name: :workflow_id, type: :identifier}],
143
- action: ->(workflow_id:, **) { Result::Text.new(call_ao("export_workflow/#{workflow_id}", format: nil, http: true).body) }
144
- command :workorders, description: 'List work orders of a workflow',
361
+ action: ->(workflow_id:, **) { Result::Status.new(call_ao('publish_workflow', body: {id: workflow_id}).eql?(true) ? 'published' : 'not published') }
362
+ command :import_with_constraints, description: 'Import a workflow file present on the Orchestrator host, with resolution of conflicts',
363
+ arguments: [{name: :payload, type: Hash, schema: 'opts:components.schemas.OrchestratorImportWithConstraints'}]
364
+ command :export, description: 'Export a workflow: display it, or save it with its dependencies',
365
+ arguments: [
366
+ {name: :workflow_id, type: :identifier},
367
+ {name: :export, type: Hash, mandatory: false, default: {}, schema: 'opts:components.schemas.OrchestratorWorkflowExport'}
368
+ ]
369
+ command :workorders, description: 'List work orders of a workflow (filters with option query)',
145
370
  arguments: [{name: :workflow_id, type: :identifier}],
371
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/work_orders_list/{id}'),
146
372
  action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao("work_orders_list/#{workflow_id}")['work_orders']) }
373
+ # Documentation for workflow development, read from GitHub (no access to Orchestrator)
374
+ commands_under :development, description: 'Documentation for workflow development, from GitHub (no access to Orchestrator): choose a document' do
375
+ DOC_URLS.each_key do |document|
376
+ command document, description: DOC_DESCRIPTIONS[document]
377
+ commands_under document do
378
+ command :toc, description: "List the sections of the document: level, title, anchor (headings up to level #{DOC_TOC_LEVELS[document]}, or argument level)",
379
+ arguments: [{name: :level, type: Integer, mandatory: false}],
380
+ action: ->(level: nil, **) { development_toc(document, level) }
381
+ command :full, description: 'Display the whole document',
382
+ action: ->(**) { Result::Text.new(development_document(document)) }
383
+ command :section, description: 'Display one section of the document (anchor given by `toc`)',
384
+ arguments: [{name: :name}],
385
+ action: ->(name:, **) { development_section(document, name) }
386
+ end
387
+ end
388
+ end
147
389
  command :outputs, description: 'Show output specification of a workflow',
148
390
  arguments: [{name: :workflow_id, type: :identifier}],
149
391
  action: ->(workflow_id:, **) { Result::ObjectList.new(call_ao("workflow_outputs_spec/#{workflow_id}")['workflow_outputs_spec']['output']) }
392
+ command :parents, description: 'List published workflows that use a sub-workflow',
393
+ arguments: [{name: :workflow_id, type: :identifier}],
394
+ action: ->(workflow_id:, **) { Result::ValueList.new(call_ao("list_by_subworkflow/#{workflow_id}")['parent_workflows'] || [], name: 'workflow') }
395
+ command :steps, description: 'List steps of a type (plugin class, e.g. CustomRuby) in published workflows, with their work orders',
396
+ arguments: [{name: :step_type}],
397
+ action: ->(step_type:, **) { Result::ObjectList.new(call_ao("retrieve_steps/#{ANY_ID}", query: {type: step_type})) }
398
+ command :test_step, description: 'Launch the test cases of workflow steps: returns the id of the test run (`synchronous`, `step`, `case` with option query)',
399
+ arguments: [{name: :workflow_id, type: :identifier}],
400
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/test_step/{id}'),
401
+ action: ->(workflow_id:, **) { Result::SingleObject.new(call_ao("test_step/#{workflow_id}")) }
402
+ command :test_status, description: 'Show the status of a test run launched by `test_step`',
403
+ arguments: [{name: :test_run_id}],
404
+ action: ->(test_run_id:, **) { Result::SingleObject.new(call_ao("test_step_status/#{test_run_id}")) }
150
405
  end
151
406
 
152
407
  commands_under :workorders do
153
408
  command :status, description: 'Show status of a work order',
154
409
  arguments: [{name: :workorder_id, type: :identifier}],
155
410
  action: ->(workorder_id:, **) { Result::SingleObject.new(call_ao("work_order_status/#{workorder_id}")['work_order']) }
156
- command :cancel, description: 'Cancel a work order',
411
+ command :worksteps, description: 'List work steps of a work order (not the steps of its sub-workflows)',
412
+ arguments: [{name: :workorder_id, type: :identifier}]
413
+ command :cancel, description: 'Cancel a work order (option query: `force`)',
157
414
  arguments: [{name: :workorder_id, type: :identifier}],
415
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/work_order_cancel/{id}'),
158
416
  action: ->(workorder_id:, **) { Result::SingleObject.new(call_ao("work_order_cancel/#{workorder_id}")['work_order']) }
159
417
  command :reset, description: 'Reset a work order',
160
418
  arguments: [{name: :workorder_id, type: :identifier}],
161
419
  action: ->(workorder_id:, **) { Result::SingleObject.new(call_ao("work_order_reset/#{workorder_id}")['work_order']) }
162
- command :output, description: 'Show output of a work order',
420
+ command :output, description: 'Show output of a work order (filters with option query: `step_name`, `variable_name`)',
421
+ arguments: [{name: :workorder_id, type: :identifier}],
422
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/work_order_output/{id}'),
423
+ action: ->(workorder_id:, **) { Result::ObjectList.new(call_ao("work_order_output/#{workorder_id}", accept: Mime::XML, xml_opts: {'ForceArray' => %w[variable], 'SuppressEmpty' => nil})['variable']) }
424
+ command :delete, description: 'Delete a work order and its dependents',
163
425
  arguments: [{name: :workorder_id, type: :identifier}],
164
- action: ->(workorder_id:, **) { Result::ObjectList.new(call_ao("work_order_output/#{workorder_id}", format: 'xml')['variable']) }
426
+ action: ->(workorder_id:, **) { Result::Status.new(call_ao("work_order_destroy/#{workorder_id}").strip) }
427
+ command :stats, description: 'Statistics of work orders: of the last 30 days, or of one day (filters `workflow_id`, `label`, `date` with option query)',
428
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/work_order_stats/{id}'),
429
+ action: ->(**) { statistics_result(call_ao("work_order_stats/#{ANY_ID}")) }
165
430
  end
166
431
 
167
- commands_under :workstep do
432
+ commands_under :worksteps do
168
433
  command :status, description: 'Show status of a work step',
169
- arguments: [{name: :workstep_id, type: :identifier}],
170
- action: ->(workstep_id:, **) { Result::SingleObject.new(call_ao("work_step_status/#{workstep_id}")) }
434
+ arguments: [{name: :workstep_id, type: :identifier}]
171
435
  command :cancel, description: 'Cancel a work step',
172
436
  arguments: [{name: :workstep_id, type: :identifier}],
173
437
  action: ->(workstep_id:, **) { Result::SingleObject.new(call_ao("work_step_cancel/#{workstep_id}")) }
438
+ command :reset, description: 'Restart the work order from a work step (several work steps: comma-separated ids)',
439
+ arguments: [{name: :workstep_id, type: :identifier}],
440
+ action: ->(workstep_id:, **) { Result.auto(call_ao("workstep_reset/#{workstep_id}")) }
441
+ command :stats, description: 'Statistics of work steps: of the last 30 days, or of one day (filters `workflow_id`, `step_id`, `step_type`, `date` with option query)',
442
+ query_schema: Schema::Registry.query_params(Schema::Registry::ORCHESTRATOR, 'api/work_step_stats/{id}'),
443
+ action: ->(**) { statistics_result(call_ao("work_step_stats/#{ANY_ID}")) }
444
+ end
445
+
446
+ # Statistics are a list, or a message in `text/plain` when there is none
447
+ # @param stats [Array<Hash>, String]
448
+ def statistics_result(stats)
449
+ stats.is_a?(String) ? Result::Status.new(stats) : Result::ObjectList.new(stats)
174
450
  end
175
451
 
176
452
  # --- API ---
@@ -184,8 +460,8 @@ module Aspera
184
460
  apikey = options.get_option(:apikey)
185
461
  username = options.get_option(:username)
186
462
  password = options.get_option(:password)
187
- Aspera.assert(apikey.nil? || (username.nil? && password.nil?), type: Cli::BadArgument) { 'apikey and username/password are mutually exclusive' }
188
- Aspera.assert(!apikey.nil? || (!username.nil? && !password.nil?), type: Cli::BadArgument) { 'provide either apikey or username and password' }
463
+ Aspera.assert(apikey.nil? || password.nil?, type: Cli::BadArgument) { 'apikey and password are mutually exclusive' }
464
+ Aspera.assert(!username.nil? && !(apikey.nil? && password.nil?), type: Cli::BadArgument) { 'provide username with either password or apikey' }
189
465
 
190
466
  auth_params =
191
467
  case style
@@ -197,11 +473,7 @@ module Aspera
197
473
  password: password
198
474
  }
199
475
  when :query
200
- if apikey
201
- {type: :url, url_query: {'apikey' => apikey}}
202
- else
203
- {type: :url, url_query: {'login' => username, 'password' => password}}
204
- end
476
+ {type: :url, url_query: {'login' => username}.merge(apikey ? {'api_key' => apikey} : {'password' => password})}
205
477
  when :token
206
478
  {
207
479
  type: :oauth2,
@@ -209,7 +481,7 @@ module Aspera
209
481
  base_url: base_url,
210
482
  path_token: 'api/login',
211
483
  token_field: 'token',
212
- json: apikey ? {apikey: apikey} : {username: username, password: password}
484
+ json: {username: username}.merge(apikey ? {api_key: apikey} : {password: password})
213
485
  }
214
486
  else Aspera.error_unexpected_value(style)
215
487
  end
@@ -222,7 +494,7 @@ module Aspera
222
494
  def action_health(**)
223
495
  nagios = Nagios.new
224
496
  begin
225
- info = call_ao('remote_node_ping', format: 'xml', xml_arrays: false)
497
+ info = call_ao('remote_node_ping', accept: Mime::XML, xml_opts: {'ForceArray' => false})
226
498
  nagios.add_ok('api', 'accessible')
227
499
  nagios.check_product_version('api', 'orchestrator', info['orchestrator-version'])
228
500
  rescue StandardError => e
@@ -231,14 +503,255 @@ module Aspera
231
503
  Result::ObjectList.new(nagios.status_list)
232
504
  end
233
505
 
506
+ # Processes of all nodes
507
+ # @param status [Hash] Response of `processes_status` or `control_process` (XML)
508
+ def processes_result(status)
509
+ Result::ObjectList.new(status['node'].flat_map { |node| node['process'] })
510
+ end
511
+
512
+ # Monitors of work orders are wrapped in `monitor-workorder`
513
+ def action_monitors_list(type:, **)
514
+ monitors = call_ao("list_monitors/#{ANY_ID}", query: {monitor_type: type})
515
+ Result::ObjectList.new(monitors.map { |monitor| monitor.is_a?(Hash) && monitor.size.eql?(1) && monitor.key?('monitor-workorder') ? monitor['monitor-workorder'] : monitor })
516
+ end
517
+
518
+ # Files of the packages, and files without package, with the name of their package
519
+ def action_journal_files(book:, **)
520
+ report = call_ao("journal_files/#{URI.encode_uri_component(book)}")
521
+ files = Array(report['Packages']).flat_map { |package| Array(package['files']).map { |file| {'package' => package['name']}.merge(file) } }
522
+ Result::ObjectList.new(files + Array(report['Files']))
523
+ end
524
+
525
+ # Fields of the work step that Orchestrator returns as YAML dumps of its Ruby objects (`prerequisites`, `work_outputs`...)
526
+ # are removed: they are not readable, and the inputs and outputs are also given as `inputs` and `outputs`.
527
+ def action_worksteps_status(workstep_id:, **)
528
+ status = call_ao("work_step_status/#{workstep_id}")
529
+ status['work_step']&.reject! { |_name, value| value.is_a?(String) && value.match?(YAML_DUMP) }
530
+ Result::SingleObject.new(status)
531
+ end
532
+
533
+ # Work steps of a work order, without the fields that are YAML dumps (see `action_worksteps_status`).
534
+ # Empty until the work order has created its steps.
535
+ def action_workorders_worksteps(workorder_id:, **)
536
+ steps = Array(call_ao("work_order_status/#{workorder_id}")['work_steps'])
537
+ steps.each { |step| step.reject! { |_name, value| value.is_a?(String) && value.match?(YAML_DUMP) } }
538
+ Result::ObjectList.new(steps)
539
+ end
540
+
234
541
  # 2.1/2.2 Initiate a workorder (async / synchronous)
235
542
  def action_workflows_list(**)
236
543
  Result::ObjectList.new(
237
544
  call_ao('workflows_list')['workflows']['workflow'],
238
- fields: %w[id portable_id name published_status published_revision_id latest_revision_id last_modification]
545
+ fields: %w[id portable_id name folder_id published_status published_revision_id latest_revision_id last_modification]
239
546
  )
240
547
  end
241
548
 
549
+ # The documents of `workflows development` are read from GitHub: does not require access to Orchestrator
550
+ # @param document [Symbol] One of DOC_URLS keys
551
+ # @return [String] Markdown
552
+ def development_document(document)
553
+ require_relative '../../uri_reader'
554
+ UriReader.read_first(DOC_URLS[document], what: "document #{document}") do |text|
555
+ Aspera.assert(!HTML_PAGE.match?(text), type: Cli::Error) { 'HTML page instead of Markdown' }
556
+ text
557
+ end
558
+ end
559
+
560
+ # @param level [Integer, nil] Maximum level of headings of the table of contents (default per document)
561
+ def development_toc(document, level)
562
+ require_relative '../../markdown'
563
+ Result::ObjectList.new(Markdown.toc(development_document(document), max_level: level || DOC_TOC_LEVELS[document]), fields: %w[level title anchor])
564
+ end
565
+
566
+ # @param anchor [String] Anchor of the section, see `toc`
567
+ def development_section(document, anchor)
568
+ require_relative '../../markdown'
569
+ text = Markdown.extract_section(development_document(document), anchor)
570
+ Aspera.assert(!text.nil?, type: Cli::Error) { "Section not found: #{anchor}, use `toc` to list anchors" }
571
+ Result::Text.new(text)
572
+ end
573
+
574
+ # Specifications of action plugins, read from GitHub, or from option `plugin_specs`: does not require access to Orchestrator
575
+ # @return [Hash] Class name of plugin => attributes, inputs, outputs
576
+ def plugin_specs
577
+ return @plugin_specs if @plugin_specs
578
+ require_relative '../../uri_reader'
579
+ require_relative '../../yaml'
580
+ specs = Yaml.safe_load(UriReader.read(options.get_option(:plugin_specs, mandatory: true)))
581
+ Aspera.assert_type(specs, Hash, type: Cli::Error) { 'plugin specifications' }
582
+ @plugin_specs = specs
583
+ end
584
+
585
+ def plugin_spec_toc
586
+ Result::ObjectList.new(plugin_specs.map { |name, spec| {'name' => name}.merge(spec.except(*SPEC_SECTIONS.map(&:to_s), 'notes')) }, fields: SPEC_LIST_FIELDS)
587
+ end
588
+
589
+ # @param plugin [String] Class name of the plugin (case insensitive)
590
+ # @param section [Symbol, nil] One of SPEC_SECTIONS, or nil for the whole specification
591
+ def plugin_spec_show(plugin, section)
592
+ specs = plugin_specs
593
+ name = specs.keys.find { |key| key.casecmp?(plugin) }
594
+ if name.nil?
595
+ similar = specs.keys.select { |key| key.downcase.include?(plugin.downcase) }
596
+ raise Cli::BadArgument, "Unknown plugin: #{plugin}#{", similar: #{similar.join(', ')}" unless similar.empty?}, use `plugins spec toc` to list plugins"
597
+ end
598
+ return Result::SingleObject.new({'name' => name}.merge(specs[name])) if section.nil?
599
+ Result::ObjectList.new(specs[name][section.to_s].map { |key, value| {'name' => key}.merge(value) })
600
+ end
601
+
602
+ # Without dependencies, the workflow file (YAML) is displayed.
603
+ # With dependencies, the package (binary) is saved in the destination folder.
604
+ def action_workflows_export(workflow_id:, export:, **)
605
+ dependencies = export.transform_keys(&:to_s)['dependencies'].eql?(true)
606
+ # Orchestrator exports dependencies for any value of the parameter
607
+ begin
608
+ resp = call_ao("export_workflow/#{workflow_id}", accept: nil, query: dependencies ? {export_with_dependencies: true} : nil, http: true)
609
+ rescue Rest::CallError
610
+ # Orchestrator 4.1.6 fails to build the package of some workflows (it is written in the folder it packs)
611
+ Log.log.warn('Export with dependencies failed: export the workflow and each sub-workflow without dependencies instead') if dependencies
612
+ raise
613
+ end
614
+ return Result::Text.new(resp.body) unless dependencies
615
+ disposition = resp['Content-Disposition']
616
+ file_name = disposition && Rest.parse_header(disposition)[:parameters][:filename]
617
+ file_path = File.join(transfer.destination_folder(Transfer::Spec::DIRECTION_RECEIVE), File.basename(file_name || "Workflow_#{workflow_id}.wkf"))
618
+ File.binwrite(file_path, resp.body)
619
+ Result::Status.new("Saved to: #{file_path}")
620
+ end
621
+
622
+ # Check of a workflow file, for problems that Orchestrator does not detect on import: it imports an invalid workflow, which then fails to publish or to run.
623
+ # This is the only place that knows how the check is done: it is done locally, because Orchestrator has no such API.
624
+ # When it has one (same request as `import_workflow`), replace the body by a call, e.g.
625
+ # `call_ao('check_workflow', body: [['import_file', content, {filename: file_name}], ['import_file_name', file_name]], content_type: Mime::MULTIPART)`
626
+ # The response is a Hash, see {OrchestratorWorkflowChecker}.
627
+ # Templates and inputs of steps are checked with the specifications of plugins (see `plugins spec show`), when they can be read.
628
+ # @param file_path [String] Workflow file (`.yml` or `.wkf`)
629
+ # @return [Hash] `valid` (Boolean), `file_name`, `issues` (list of Hash: `severity`, `code`, `file`, `path`, `message`)
630
+ def check_workflow(file_path)
631
+ require_relative '../../orchestrator_workflow_checker'
632
+ specs = nil
633
+ specs_error = nil
634
+ begin
635
+ specs = plugin_specs
636
+ rescue StandardError => e
637
+ specs_error = e.message
638
+ end
639
+ response = OrchestratorWorkflowChecker.check(file_name: File.basename(file_path), content: File.binread(file_path), specs: specs)
640
+ unless specs_error.nil?
641
+ response['issues'].push({
642
+ 'severity' => 'warning',
643
+ 'code' => 'plugins.specs_unavailable',
644
+ 'path' => 'templates',
645
+ 'message' => "plugin specifications not read (#{specs_error}): templates and inputs of steps are not checked"
646
+ })
647
+ end
648
+ response
649
+ end
650
+
651
+ # @param file_path [String] Workflow file
652
+ # @return [Result] The issues found, or a status if none
653
+ def action_workflows_check_import(file_path:, **)
654
+ response = check_workflow(file_path)
655
+ return Result::Status.new('no problem found') if response['issues'].empty?
656
+ Result::ObjectList.new(response['issues'], fields: %w[severity code file path message])
657
+ end
658
+
659
+ # Conflicts and missing plugins found by Orchestrator, as one row per name to resolve.
660
+ # Orchestrator does not check that the file exists: for a wrong path, it returns the same empty result as for a file without conflict.
661
+ # @return [Hash{String=>Object}] Items returned by Orchestrator, by name (`subwf constraints`, `missing plugins`...)
662
+ def server_constraints(filename, add_as_revision = nil)
663
+ list = call_ao('find_constraints', query: {file: filename, wf_id: add_as_revision}.compact)
664
+ Aspera.assert_type(list, Array, type: Cli::Error) { 'response of find_constraints' }
665
+ list.grep(Hash).reduce({}, :merge)
666
+ end
667
+
668
+ # Keys of the conflicts in Orchestrator response => keys of the argument of `import_with_constraints`
669
+ CONSTRAINT_KEYS = {
670
+ 'subwf constraints' => 'subwf_constraints',
671
+ 'action template constraints' => 'action_template_constraints',
672
+ 'remote node constraints' => 'remote_node_constraints'
673
+ }.freeze
674
+ private_constant :CONSTRAINT_KEYS
675
+
676
+ def action_workflows_find_constraints(filename:, add_as_revision: nil, **)
677
+ found = server_constraints(filename, add_as_revision)
678
+ rows = CONSTRAINT_KEYS.flat_map do |api_key, key|
679
+ (found[api_key] || {}).map { |name, choices| {'type' => key, 'name' => name, 'choices' => choices} }
680
+ end
681
+ Array(found['missing plugins']).each do |name|
682
+ rows.push({'type' => 'missing_plugins', 'name' => name, 'choices' => 'auto_enable_missing_plugins: true'})
683
+ end
684
+ return Result::Status.new('no conflict found (Orchestrator gives the same answer if the file does not exist)') if rows.empty?
685
+ Result::ObjectList.new(rows, fields: %w[type name choices])
686
+ end
687
+
688
+ # Orchestrator fails or decides silently in these cases, which the web interface prevents:
689
+ # - plugins to enable but `auto_enable_missing_plugins` not true: the import fails with an internal error
690
+ # - resolution given for some conflicts only: the others are imported as copies, without error
691
+ # @param payload [Hash] Argument of `import_with_constraints`, with string keys
692
+ def check_import_constraints(payload)
693
+ found = server_constraints(payload['filename'], payload['add_as_revision'])
694
+ problems = []
695
+ missing = Array(found['missing plugins'])
696
+ problems.push("plugins are not enabled: #{missing.join(', ')}, set auto_enable_missing_plugins to true") unless missing.empty? || payload['auto_enable_missing_plugins'].eql?(true)
697
+ CONSTRAINT_KEYS.each do |api_key, key|
698
+ chosen = payload[key]
699
+ next if chosen.nil? || chosen.empty?
700
+ unresolved = (found[api_key] || {}).keys - chosen.keys.map(&:to_s)
701
+ problems.push("#{key} not resolved for: #{unresolved.join(', ')} (see `workflows find_constraints`)") unless unresolved.empty?
702
+ end
703
+ raise Cli::BadArgument, "Import refused: #{problems.join('; ')}" unless problems.empty?
704
+ end
705
+
706
+ # The file is uploaded as a multipart form
707
+ # @param import [Hash] Options: `folder_id`, `publish`, `link_to_subworkflows`
708
+ def action_workflows_import(file_path:, import: {}, **)
709
+ import = import.transform_keys(&:to_s)
710
+ check_workflow(file_path)['issues'].each do |issue|
711
+ Log.log.warn { "#{issue['severity']}: #{[issue['file'], issue['path']].compact.join(' ')}: #{issue['message']} (#{issue['code']})" }
712
+ end
713
+ file_name = File.basename(file_path)
714
+ form = [
715
+ ['import_file', File.binread(file_path), {filename: file_name}],
716
+ ['import_file_name', file_name]
717
+ ]
718
+ form.push(['folder_id', import['folder_id'].to_s]) if import.key?('folder_id')
719
+ # `false` overrides the server option `publish_api_imported_workflow`
720
+ form.push(['publish_workflow', import['publish'].to_s]) if import.key?('publish')
721
+ # Orchestrator takes any value as true
722
+ form.push(%w[link_to_subworkflows true]) if import['link_to_subworkflows'].eql?(true)
723
+ result = call_ao('import_workflow', body: form, content_type: Mime::MULTIPART)
724
+ # Other responses: nothing was imported
725
+ if result.key?('plugins')
726
+ raise Cli::Error, "Plugins could not be enabled: #{Array(result['plugins']).join(', ')}" \
727
+ "#{" (missing dependencies: #{Array(result['missing_deps']).join(', ')})" unless Array(result['missing_deps']).empty?}"
728
+ end
729
+ if result.key?('dependencies')
730
+ dependencies = result['dependencies'].values.flatten.filter_map { |d| "#{d['entity']} #{d['id_value']}" if d.is_a?(Hash) }
731
+ raise Cli::Error, "Workflow dependencies are not included in the file (export with dependencies to a .wkf file): #{dependencies.uniq.join(', ')}"
732
+ end
733
+ Result::SingleObject.new(result['workflow'])
734
+ end
735
+
736
+ # Items of the array expected by `import_with_constraints`, in this order: argument key => API key
737
+ IMPORT_CONSTRAINT_ITEMS = {
738
+ 'filename' => 'filename',
739
+ 'add_as_revision' => 'add as revision',
740
+ 'subwf_constraints' => 'subwf constraints',
741
+ 'action_template_constraints' => 'action template constraints',
742
+ 'remote_node_constraints' => 'remote node constraints',
743
+ 'auto_enable_missing_plugins' => 'Auto-enable missing plugins?'
744
+ }.freeze
745
+ private_constant :IMPORT_CONSTRAINT_ITEMS
746
+
747
+ # The API expects an array of single-key objects, in a fixed order
748
+ def action_workflows_import_with_constraints(payload:, **)
749
+ payload = payload.transform_keys(&:to_s)
750
+ check_import_constraints(payload)
751
+ body = IMPORT_CONSTRAINT_ITEMS.map { |key, api_key| {api_key => payload.fetch(key) { key.end_with?('_constraints') ? {} : nil }} }
752
+ Result::SingleObject.new(call_ao('import_with_constraints', body: body)['workflow'])
753
+ end
754
+
242
755
  # Keys of argument `execution` of `workflows start`
243
756
  WORKFLOW_START_KEYS = %w[synchronous step variable].freeze
244
757
  private_constant :WORKFLOW_START_KEYS
@@ -269,8 +782,8 @@ module Aspera
269
782
  # implicitly, call is synchronous
270
783
  query_params[:synchronous] = true
271
784
  end
272
- result_data = call_ao('initiate', body: json_body, query: query_params.empty? ? nil : query_params)
273
- query_params[:synchronous] ? Result::Text.new(result_data) : Result::SingleObject.new(result_data)
785
+ # Work order information, or value of the explicit output (any JSON value)
786
+ Result.auto(call_ao('initiate', body: json_body, query: query_params.empty? ? nil : query_params))
274
787
  end
275
788
  end
276
789
  end