aspera-cli 4.27.5 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +301 -4
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +139 -58
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +10 -7
  13. data/docs/README.md +2551 -772
  14. data/docs/test-mcp-with-ai.md +13 -11
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +34 -22
  19. data/lib/aspera/agent/factory.rb +8 -4
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +8 -5
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +23 -19
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +84 -42
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +102 -48
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +16 -37
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -34
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +49 -4
  50. data/lib/aspera/cli/command_spec.rb +34 -4
  51. data/lib/aspera/cli/completion/ascli.bash +53 -5
  52. data/lib/aspera/cli/completion/ascli.fish +27 -8
  53. data/lib/aspera/cli/completion/ascli.ps1 +63 -0
  54. data/lib/aspera/cli/completion/ascli.zsh +36 -4
  55. data/lib/aspera/cli/completion_actions.rb +594 -0
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +45 -0
  59. data/lib/aspera/cli/extended_value.rb +83 -24
  60. data/lib/aspera/cli/formatter.rb +142 -73
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -11
  63. data/lib/aspera/cli/http.rb +17 -13
  64. data/lib/aspera/cli/info.rb +17 -6
  65. data/lib/aspera/cli/mailer.rb +7 -4
  66. data/lib/aspera/cli/mcp/http_servlet.rb +155 -0
  67. data/lib/aspera/cli/mcp/secrets.rb +146 -0
  68. data/lib/aspera/cli/mcp/tool.rb +232 -0
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +72 -11
  73. data/lib/aspera/cli/options.schema.yaml +856 -81
  74. data/lib/aspera/cli/parser.rb +64 -31
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +250 -44
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +51 -25
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +207 -141
  81. data/lib/aspera/cli/plugins/console.rb +9 -4
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +22 -8
  84. data/lib/aspera/cli/plugins/faspex5.rb +37 -44
  85. data/lib/aspera/cli/plugins/faspio.rb +41 -8
  86. data/lib/aspera/cli/plugins/httpgw.rb +8 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +186 -121
  88. data/lib/aspera/cli/plugins/node.rb +41 -34
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +572 -59
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +66 -12
  93. data/lib/aspera/cli/plugins/shares.rb +288 -48
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +155 -33
  96. data/lib/aspera/cli/preset_manager.rb +72 -16
  97. data/lib/aspera/cli/prompt.rb +7 -2
  98. data/lib/aspera/cli/result.rb +28 -11
  99. data/lib/aspera/cli/runner.rb +193 -80
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +204 -0
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +119 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +32 -40
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +31 -19
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +197 -50
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.html.erb +61 -0
  113. data/lib/aspera/cli/web_secret.rb +222 -0
  114. data/lib/aspera/cli/wizard.rb +58 -45
  115. data/lib/aspera/command_line_builder.rb +35 -5
  116. data/lib/aspera/command_line_converter.rb +4 -1
  117. data/lib/aspera/coverage.rb +103 -33
  118. data/lib/aspera/data_repository.rb +42 -25
  119. data/lib/aspera/dot_container.rb +4 -1
  120. data/lib/aspera/duration.rb +45 -0
  121. data/lib/aspera/environment.rb +106 -33
  122. data/lib/aspera/exec_spec.rb +9 -5
  123. data/lib/aspera/faspex_gw.rb +105 -35
  124. data/lib/aspera/faspex_postproc.rb +193 -47
  125. data/lib/aspera/formatter_interface.rb +4 -2
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +6 -3
  129. data/lib/aspera/json_rpc/client.rb +55 -35
  130. data/lib/aspera/json_rpc/version.rb +4 -0
  131. data/lib/aspera/line_logger.rb +4 -1
  132. data/lib/aspera/link_header.rb +4 -1
  133. data/lib/aspera/log.rb +20 -10
  134. data/lib/aspera/markdown.rb +155 -115
  135. data/lib/aspera/mime.rb +9 -0
  136. data/lib/aspera/nagios.rb +9 -5
  137. data/lib/aspera/node_emulator.rb +111 -112
  138. data/lib/aspera/oauth/client/base.rb +163 -0
  139. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +8 -4
  140. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +13 -3
  141. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +5 -1
  142. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +4 -1
  143. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +6 -3
  144. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  145. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +4 -1
  146. data/lib/aspera/oauth/client/web.rb +84 -0
  147. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  148. data/lib/aspera/oauth/server/pkce.rb +427 -0
  149. data/lib/aspera/orchestrator_workflow_checker.rb +679 -0
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +11 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +4 -1
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +30 -1
  165. data/lib/aspera/rest/aspera_errors.rb +55 -60
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +42 -42
  168. data/lib/aspera/rest/error_analyzer.rb +14 -2
  169. data/lib/aspera/rest/list.rb +14 -10
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +13 -4
  172. data/lib/aspera/rest.rb +9 -7
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -11
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4089 -0
  175. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  176. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  177. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  178. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  179. data/lib/aspera/schema/async_tables.yaml +11 -10
  180. data/lib/aspera/schema/documentation.rb +56 -78
  181. data/lib/aspera/schema/reader.rb +46 -11
  182. data/lib/aspera/schema/registry.rb +45 -24
  183. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  184. data/lib/aspera/schema/validator.rb +4 -1
  185. data/lib/aspera/sealed_hash.rb +166 -0
  186. data/lib/aspera/secret/hider.rb +201 -0
  187. data/lib/aspera/secret/reference.rb +150 -0
  188. data/lib/aspera/secret/resolver.rb +212 -0
  189. data/lib/aspera/secret/store/base.rb +153 -0
  190. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  191. data/lib/aspera/secret/store/factory.rb +103 -0
  192. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  193. data/lib/aspera/secret/store/keeper.rb +141 -0
  194. data/lib/aspera/secret/store/macos_security.rb +284 -0
  195. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  196. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  197. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  198. data/lib/aspera/secret/store/secret_service.rb +125 -0
  199. data/lib/aspera/secret/strength.rb +119 -0
  200. data/lib/aspera/servlet_helpers.rb +114 -0
  201. data/lib/aspera/ssh.rb +10 -7
  202. data/lib/aspera/ssl.rb +5 -2
  203. data/lib/aspera/string_ext.rb +3 -0
  204. data/lib/aspera/sync/args.schema.yaml +6 -1
  205. data/lib/aspera/sync/conf.schema.yaml +25 -10
  206. data/lib/aspera/sync/database.rb +3 -0
  207. data/lib/aspera/sync/operations.rb +34 -12
  208. data/lib/aspera/temp_file_manager.rb +85 -22
  209. data/lib/aspera/timer_limiter.rb +3 -0
  210. data/lib/aspera/transfer/error.rb +4 -1
  211. data/lib/aspera/transfer/faux_file.rb +4 -1
  212. data/lib/aspera/transfer/parameters.rb +22 -14
  213. data/lib/aspera/transfer/result.rb +3 -0
  214. data/lib/aspera/transfer/resumer.rb +6 -4
  215. data/lib/aspera/transfer/spec.rb +45 -18
  216. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  217. data/lib/aspera/transfer/units.rb +75 -0
  218. data/lib/aspera/uri_reader.rb +57 -7
  219. data/lib/aspera/web_auth/index.html.erb +20 -0
  220. data/lib/aspera/web_auth/paste.rb +109 -0
  221. data/lib/aspera/web_auth/receiver.rb +50 -0
  222. data/lib/aspera/web_auth/relay.rb +76 -0
  223. data/lib/aspera/web_auth/reply.html.erb +150 -0
  224. data/lib/aspera/web_auth/server.rb +59 -14
  225. data/lib/aspera/web_page.html.erb +262 -0
  226. data/lib/aspera/web_page.rb +66 -0
  227. data/lib/aspera/web_server_simple.rb +82 -13
  228. data/lib/aspera/yaml.rb +5 -1
  229. data/llms.txt +81 -0
  230. data/skills/ascli/SKILL.md +88 -0
  231. data.tar.gz.sig +0 -0
  232. metadata +116 -48
  233. metadata.gz.sig +0 -0
  234. data/lib/aspera/cli/mcp_tool.rb +0 -189
  235. data/lib/aspera/keychain/base.rb +0 -25
  236. data/lib/aspera/keychain/encrypted_hash.rb +0 -188
  237. data/lib/aspera/keychain/factory.rb +0 -59
  238. data/lib/aspera/keychain/hashicorp_vault.rb +0 -74
  239. data/lib/aspera/keychain/macos_security.rb +0 -171
  240. data/lib/aspera/keychain/one_password_api.rb +0 -86
  241. data/lib/aspera/keychain/one_password_base.rb +0 -34
  242. data/lib/aspera/keychain/one_password_cli.rb +0 -98
  243. data/lib/aspera/oauth/base.rb +0 -141
  244. data/lib/aspera/oauth/web.rb +0 -57
  245. data/lib/aspera/oauth.rb +0 -9
  246. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  247. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  248. data/lib/aspera/secret_hider.rb +0 -117
  249. data/lib/aspera/web_auth/index.html +0 -147
@@ -0,0 +1,679 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'psych'
7
+ require 'rexml/document'
8
+ require 'stringio'
9
+ require 'zlib'
10
+ require 'rubygems/package'
11
+
12
+ module Aspera
13
+ # Check of an Orchestrator workflow file (`.yml`) or package (`.wkf`), before import.
14
+ #
15
+ # Orchestrator 4.1.6 does not validate the graph on import (`import_workflow`, `import_with_constraints`): an invalid workflow is imported,
16
+ # then fails to publish, or at run time. These checks follow the rules of the designer (`Whiteboard#parse_workflow`).
17
+ # With the specifications of plugins (`plugins.yml` of the plugin documentation), action templates and inputs of steps are also checked:
18
+ # attributes and their values, required inputs, inputs and outputs that the plugin does not have with its template.
19
+ # Not checked: availability of plugins on the server.
20
+ #
21
+ # The interface is that of a REST operation, so that it can be replaced by a call to Orchestrator when one exists:
22
+ # - request: the fields of `import_workflow`: `file_name` (`import_file_name`) and `content` (`import_file`, bytes),
23
+ # and the specifications of plugins (`specs`), that Orchestrator would not need
24
+ # - response: a Hash (JSON compatible)
25
+ # `valid`: `true` if there is no issue with severity `error`
26
+ # `file_name`: name of the checked file
27
+ # `issues`: list of Hash
28
+ # `severity`: `error` (the workflow will not import, publish or run correctly) or `warning`
29
+ # `code`: stable identifier of the rule, for programs, e.g. `graph.step_orphan`
30
+ # `file`: file of the package where the issue is (absent for a single `.yml` file)
31
+ # `path`: where in the file, e.g. `graph/Step[Say hello]`
32
+ # `message`: description for humans
33
+ #
34
+ # The file is parsed without instantiating Ruby objects.
35
+ class OrchestratorWorkflowChecker
36
+ ERROR = 'error'
37
+ WARNING = 'warning'
38
+ # Columns of tables `workflows` and `whiteboards` (Orchestrator 4.1.6), and attributes ignored by the import
39
+ WORKFLOW_ATTRIBUTES = %w[
40
+ id name portable_id created_by folder_id generated_from revision_id label run_as source_revision comments description tags templates
41
+ max_running cleanup_after_days purge_after_days user_lock locked_by created_at updated_at global_variables hide_workflow
42
+ ].freeze
43
+ WHITEBOARD_ATTRIBUTES = %w[id created_by revision_id workflow_id error_free revisions_log xml_export created_at updated_at].freeze
44
+ VALUE_TYPES = %w[string int float flag date array hash object pwd attachment].freeze
45
+ # Element 6 (plugin versions) is informational: Orchestrator 4.1.6 only reads elements 1 to 5
46
+ ELEMENT_COUNTS = [5, 6].freeze
47
+ TRUE_VALUES = %w[true 1 t].freeze
48
+ # SimpleParameterLookup: separator of the columns of the table, input that returns the table indexed by a column
49
+ LOOKUP_DELIMITER = ':::'
50
+ LOOKUP_INDEX_INPUT = 'KeyForTableIndexing'
51
+ UNNAMED = ['', 'undefined', 'null'].freeze
52
+ SUCCESS_STATUSES = %w[complete true].freeze
53
+ BOOLEAN_TEXT = %w[true false].freeze
54
+ STEP_CHILDREN = %w[Inputs Outputs].freeze
55
+ PARAMETER_MAPPINGS = %w[parameter global].freeze
56
+ # Columns of every action template, besides those of the plugin
57
+ TEMPLATE_COLUMNS = %w[id name comments created_at updated_at].freeze
58
+ # Texts that the database layer (ActiveRecord) reads as false
59
+ FALSE_VALUES = %w[false 0 f off].freeze
60
+ INTEGER_TYPES = %w[integer bigint].freeze
61
+ # Template attribute with Ruby code that computes the inputs or the outputs: they are then unknown
62
+ SPEC_CODE = {inputs: 'inputs_spec_code', outputs: 'outputs_spec_code'}.freeze
63
+ # Note of a plugin specification: some inputs or outputs come from the template, other than through declared lists
64
+ PARTLY_DEFINED = {inputs: /\AInputs are partly defined/, outputs: /\AOutputs are partly defined/}.freeze
65
+ # Placeholder that creates a required input: <%= variable %>
66
+ PLACEHOLDER = /<%=\s*([A-Za-z_]\w*)\s*%>/
67
+ # Value of a template attribute that is a single name (of an input or an output): no line break, no structure (type map, list)
68
+ SINGLE_NAME = /\A[^\r\n{}\[\],:=]+\z/
69
+ # Column separator of a table (attribute `parameters_associations` of `SimpleParameterLookup`), whose first line holds the column names
70
+ TABLE_SEPARATOR = ':::'
71
+ # Elements of a step that a `Map_to` of type `output` gives as attributes (in lower case)
72
+ STEP_IDENTITY = %w[Step_name Step_type Action_id].freeze
73
+ private_constant :WORKFLOW_ATTRIBUTES, :WHITEBOARD_ATTRIBUTES, :VALUE_TYPES, :ELEMENT_COUNTS, :TRUE_VALUES, :UNNAMED, :SUCCESS_STATUSES,
74
+ :BOOLEAN_TEXT, :STEP_CHILDREN, :PARAMETER_MAPPINGS, :TEMPLATE_COLUMNS, :FALSE_VALUES, :INTEGER_TYPES, :SPEC_CODE, :PARTLY_DEFINED, :PLACEHOLDER,
75
+ :STEP_IDENTITY, :SINGLE_NAME, :TABLE_SEPARATOR, :LOOKUP_DELIMITER, :LOOKUP_INDEX_INPUT
76
+
77
+ class << self
78
+ # @param file_name [String] Name of the file, ending with `yml` or `wkf`
79
+ # @param content [String] Bytes of the file
80
+ # @param specs [Hash, nil] Specifications of plugins (`plugins.yml`), to check templates and inputs of steps, or nil
81
+ # @return [Hash] Response, see class description
82
+ def check(file_name:, content:, specs: nil)
83
+ checker = new(file_name, specs)
84
+ checker.run(content)
85
+ {
86
+ 'valid' => checker.issues.none? { |issue| issue['severity'].eql?(ERROR) },
87
+ 'file_name' => file_name,
88
+ 'issues' => checker.issues
89
+ }
90
+ end
91
+ end
92
+
93
+ attr_reader :issues
94
+
95
+ # @param file_name [String] Name of the file
96
+ # @param specs [Hash, nil] Specifications of plugins: class name => attributes, inputs, outputs
97
+ def initialize(file_name, specs = nil)
98
+ @file_name = file_name
99
+ @specs = specs
100
+ @issues = []
101
+ # file of the package being checked
102
+ @member = nil
103
+ # attributes of the action templates of the workflow being checked, by key <StepType>__<N>
104
+ @template_attributes = {}
105
+ end
106
+
107
+ # @param content [String] Bytes of the file
108
+ def run(content)
109
+ case File.extname(@file_name)
110
+ when '.yml' then check_yaml(content)
111
+ when '.wkf' then check_package(content)
112
+ else add(ERROR, 'file.extension', 'file', "extension must be yml or wkf: #{@file_name}")
113
+ end
114
+ end
115
+
116
+ private
117
+
118
+ def add(severity, code, path, message)
119
+ @issues.push({'severity' => severity, 'code' => code, 'file' => @member, 'path' => path, 'message' => message}.compact)
120
+ end
121
+
122
+ # @param content [String] Bytes of a `.wkf` file: gzipped tar with a manifest (list of files, the first one is the main workflow)
123
+ def check_package(content)
124
+ files = {}
125
+ begin
126
+ Zlib::GzipReader.wrap(StringIO.new(content)) do |gzip|
127
+ Gem::Package::TarReader.new(gzip) do |tar|
128
+ tar.each { |entry| files[entry.full_name] = entry.read if entry.file? }
129
+ end
130
+ end
131
+ rescue Zlib::Error, Gem::Package::Error => e
132
+ return add(ERROR, 'package.archive', 'file', "not a gzipped tar archive: #{e.message}")
133
+ end
134
+ manifest = files.keys.find { |name| File.basename(name).match?(/\AWorkflow_.*\.txt\z/) }
135
+ return add(ERROR, 'package.manifest_missing', 'file', 'no manifest file (Workflow_*.txt) in package') if manifest.nil?
136
+ list = Psych.safe_load(files[manifest])
137
+ return add(ERROR, 'package.manifest_invalid', manifest, 'manifest must be a list of file names') unless list.is_a?(Array) && !list.empty? && list.all?(String)
138
+ list.each_with_index do |name, index|
139
+ member = files.find { |key, _| File.basename(key).eql?(File.basename(name)) }&.last
140
+ next add(ERROR, 'package.file_missing', manifest, "file listed but not in package: #{name}") if member.nil?
141
+ # sub-workflows are checked as the main workflow; remote nodes and nested packages are not
142
+ next unless name.end_with?('.yml') && (index.zero? || File.basename(name).start_with?('Workflow_'))
143
+ @member = File.basename(name)
144
+ check_yaml(member)
145
+ end
146
+ @member = nil
147
+ rescue Psych::Exception => e
148
+ add(ERROR, 'package.manifest_invalid', manifest.to_s, "invalid manifest: #{e.message}")
149
+ end
150
+
151
+ # @param content [String] Bytes of a workflow file
152
+ def check_yaml(content)
153
+ text = content.dup.force_encoding(Encoding::UTF_8)
154
+ return add(ERROR, 'yaml.encoding', 'file', 'file is not UTF-8') unless text.valid_encoding?
155
+ begin
156
+ root = Psych.parse_stream(text).children.first&.root
157
+ rescue Psych::SyntaxError => e
158
+ return add(ERROR, 'yaml.invalid', 'file', "invalid YAML: #{e.message}")
159
+ end
160
+ return add(ERROR, 'yaml.empty', 'file', 'empty file') if root.nil?
161
+ return add(ERROR, 'yaml.not_list', 'file', 'a workflow file is a YAML list of elements') unless root.is_a?(Psych::Nodes::Sequence)
162
+ @plain = text.lines.none? { |line| line.match?(/^ *concise_attributes:/) }
163
+ @template_attributes = {}
164
+ check_layout(text) if @plain
165
+ elements = root.children
166
+ add(ERROR, 'yaml.element_count', 'file', "expected #{ELEMENT_COUNTS.join(' or ')} elements (workflow, whiteboard, templates, dependencies, packed dependencies, plugin versions), found #{elements.size}") unless ELEMENT_COUNTS.include?(elements.size)
167
+ return if elements.size < 2
168
+ workflow = check_attributes(elements[0], 'Workflow', 'workflow', WORKFLOW_ATTRIBUTES)
169
+ add(ERROR, 'workflow.name_missing', 'workflow', 'attribute name is missing') if workflow['name'].to_s.strip.empty?
170
+ whiteboard = check_attributes(elements[1], 'Whiteboard', 'whiteboard', WHITEBOARD_ATTRIBUTES)
171
+ add(ERROR, 'whiteboard.error_free', 'whiteboard', 'error_free is not true: the workflow cannot be published') unless TRUE_VALUES.include?(whiteboard['error_free'].to_s.downcase)
172
+ templates = elements.size > 2 ? check_templates(elements[2]) : []
173
+ elements[3..4]&.each_with_index do |element, index|
174
+ add(ERROR, 'dependencies.not_map', "element #{index + 4}", 'must be a map') unless element.is_a?(Psych::Nodes::Mapping)
175
+ end
176
+ if whiteboard['xml_export'].to_s.strip.empty?
177
+ add(ERROR, 'whiteboard.xml_missing', 'whiteboard', 'attribute xml_export is missing')
178
+ else
179
+ check_graph(whiteboard['xml_export'], templates)
180
+ end
181
+ check_dependencies(elements[3], elements[4]) if elements.size > 4
182
+ end
183
+
184
+ # Plain format is converted by Orchestrator line by line: each element starts with a dash at column 0
185
+ def check_layout(text)
186
+ count = text.lines.count { |line| line.match?(/^- /) }
187
+ return if ELEMENT_COUNTS.include?(count)
188
+ add(ERROR, 'yaml.layout', 'file', "plain format: each element must start with '- ' at column 0 (#{ELEMENT_COUNTS.join(' or ')} expected), found #{count} such lines")
189
+ end
190
+
191
+ # @return [Hash{String=>Psych::Nodes::Node}] Keys and values of a mapping
192
+ def pairs(node)
193
+ return {} unless node.is_a?(Psych::Nodes::Mapping)
194
+ node.children.each_slice(2).to_h { |key, value| [key.respond_to?(:value) ? key.value : nil, value] }
195
+ end
196
+
197
+ def scalar(node)
198
+ node.is_a?(Psych::Nodes::Scalar) ? node.value : nil
199
+ end
200
+
201
+ # @return [Hash{String=>String}] Attributes of a workflow, a whiteboard or a template, in plain or Rails format
202
+ def attributes(node)
203
+ content = pairs(node)
204
+ if content.key?('concise_attributes')
205
+ content['concise_attributes'].children.to_h do |item|
206
+ item_pairs = pairs(item)
207
+ [scalar(item_pairs['name']), scalar(item_pairs['value_before_type_cast'])]
208
+ end
209
+ else
210
+ inner = content['attributes']
211
+ (inner.is_a?(Psych::Nodes::Mapping) ? pairs(inner) : content.except('attributes', 'new_record', 'active_record_yaml_version')).transform_values { |value| scalar(value) }
212
+ end
213
+ end
214
+
215
+ def check_attributes(node, klass, path, known)
216
+ unless node.is_a?(Psych::Nodes::Mapping) && node.tag.eql?("!ruby/object:#{klass}")
217
+ add(ERROR, "#{path}.type", path, "element must be a map tagged !ruby/object:#{klass}")
218
+ return {}
219
+ end
220
+ attributes(node).tap do |attrs|
221
+ attrs.each_key { |name| add(WARNING, "#{path}.attribute_unknown", path, "unknown attribute: #{name} (the import may fail)") unless known.include?(name) }
222
+ end
223
+ end
224
+
225
+ # @return [Array<String>] Keys of the action templates
226
+ def check_templates(node)
227
+ return [] unless node.is_a?(Psych::Nodes::Mapping)
228
+ pairs(node).filter_map do |key, template|
229
+ path = "templates/#{key}"
230
+ match = key.to_s.match(/\A(.+)__(\d+)\z/)
231
+ if match.nil?
232
+ add(ERROR, 'template.key', path, "key must be <StepType>__<N>: #{key}")
233
+ elsif !template.respond_to?(:tag) || !template.tag.eql?("!ruby/object:#{match[1]}")
234
+ add(ERROR, 'template.tag', path, "value must be tagged !ruby/object:#{match[1]}")
235
+ else
236
+ check_boolean_values(path, template) if @plain
237
+ @template_attributes[key] = attributes(template)
238
+ check_template_attributes(path, match[1], @template_attributes[key]) unless @specs.nil?
239
+ end
240
+ key
241
+ end
242
+ end
243
+
244
+ # In plain format, a blank attribute is dropped, and false is blank
245
+ def check_boolean_values(path, template)
246
+ attributes(template).each do |name, value|
247
+ add(WARNING, 'template.boolean', path, "#{name} is #{value}: write 1 or 0, false is dropped in plain format") if BOOLEAN_TEXT.include?(value)
248
+ end
249
+ end
250
+
251
+ def check_dependencies(listed, packed)
252
+ return unless listed.is_a?(Psych::Nodes::Mapping) && packed.is_a?(Psych::Nodes::Mapping)
253
+ external = pairs(listed).values.flat_map { |list| list.is_a?(Psych::Nodes::Sequence) ? list.children : [] }
254
+ return if external.empty? || !packed.children.empty?
255
+ names = external.map do |dep|
256
+ fields = pairs(dep)
257
+ "#{scalar(fields[':entity'] || fields['entity'])} #{scalar(fields[':id_value'] || fields['id_value'])}".strip
258
+ end
259
+ add(WARNING, 'dependencies.external', 'dependencies', "uses entities not included in the file, they must exist on the server: #{names.uniq.join(', ')}")
260
+ end
261
+
262
+ def text_of(element, name)
263
+ element.elements[name]&.text&.strip
264
+ end
265
+
266
+ def check_graph(xml, templates)
267
+ begin
268
+ root = REXML::Document.new(xml).root
269
+ rescue REXML::ParseException => e
270
+ return add(ERROR, 'graph.xml_invalid', 'xml_export', "invalid XML: #{e.message.lines.first&.strip}")
271
+ end
272
+ return add(ERROR, 'graph.xml_invalid', 'xml_export', 'no XML root element') if root.nil?
273
+ terminals = %w[Start End Fail Error].to_h { |name| [name, root.get_elements(name)] }
274
+ terminals.each do |name, nodes|
275
+ add(ERROR, 'graph.terminal_count', "graph/#{name}", "#{name} must be present once, found #{nodes.size}") unless nodes.size.eql?(1)
276
+ end
277
+ steps = root.get_elements('Steps/Step')
278
+ add(ERROR, 'graph.no_step', 'graph', 'at least one Step is required') if steps.empty?
279
+ ids = (terminals.values.flatten + steps).map { |node| node.attributes['id'] }
280
+ (ids.tally.select { |_, count| count > 1 }.keys + ids.compact.select(&:empty?)).each do |id|
281
+ add(ERROR, 'graph.id_not_unique', 'graph', "node id must be unique and not empty: '#{id}'")
282
+ end
283
+ check_steps(steps, templates)
284
+ closing = terminals.values_at('End', 'Fail', 'Error').flatten
285
+ (closing + steps).each { |node| check_node(node, ids) }
286
+ check_branches(closing, steps)
287
+ # starvation is computed on a graph without loop
288
+ check_starvation(terminals['Start'].first&.attributes&.[]('id'), closing, steps) unless check_loops(closing, steps)
289
+ check_parameters(root, steps)
290
+ check_routers(root)
291
+ check_lookups(steps)
292
+ check_step_interfaces(steps) unless @specs.nil?
293
+ end
294
+
295
+ # SimpleParameterLookup returns a row only if an input selects it: an input named as a column of the header of the table.
296
+ # Without one, no row matches and the step ends with Failed (No configuration match), unless it returns the array of rows
297
+ # (SimpleParameterLookup#execute). The header of a table in a file (param_file_name) is not known.
298
+ def check_lookups(steps)
299
+ steps.each do |step|
300
+ next unless text_of(step, 'Step_type').eql?('SimpleParameterLookup')
301
+ values = @template_attributes["SimpleParameterLookup__#{text_of(step, 'Action_id')}"]
302
+ next if values.nil? || blank?(values['parameters_associations']) || !blank?(values['param_file_name'])
303
+ next if TRUE_VALUES.include?(values['return_array_of_rows'].to_s.downcase)
304
+ header = values['parameters_associations'].lines.first.to_s.strip.split(LOOKUP_DELIMITER, -1).map(&:strip)
305
+ # a carried variable is also given to the step under its name
306
+ provided = step.get_elements('Inputs/Input').map { |input| input.attributes['name'] } + step.get_elements('Outputs/Carry_thru').map { |carry| carry.attributes['name'] }
307
+ next if provided.include?(LOOKUP_INDEX_INPUT) || provided.intersect?(header)
308
+ add(
309
+ ERROR, 'step.lookup_without_key', node_path(step),
310
+ "no input is mapped to a column of the table (#{header.join(', ')}): no row matches and the step ends with Failed, unless return_array_of_rows is set"
311
+ )
312
+ end
313
+ end
314
+
315
+ # Line styles of links in the designer: it writes `new <router>()` in the page to draw each link (WhiteboardsHelper#connect_xmlsteps)
316
+ ROUTERS = %w[Manhattan Bezier Null Fan].map { |name| "draw2d.#{name}ConnectionRouter" }.freeze
317
+
318
+ # Links: prerequisites of nodes and links of parameters (Parameter in Start or Step); absent router: Manhattan
319
+ def check_routers(root)
320
+ root.each_element('//Prerequisite[@router] | //Start/Parameters/Parameter[@router] | //Step/Parameters/Parameter[@router]') do |link|
321
+ router = link.attributes['router']
322
+ next if ROUTERS.include?(router)
323
+ add(
324
+ ERROR, 'graph.router', node_path(link.parent.parent),
325
+ "router '#{router}' of the link from '#{link.attributes['id']}' is not one of #{ROUTERS.join(', ')}: the designer cannot draw the workflow"
326
+ )
327
+ end
328
+ end
329
+
330
+ def node_path(node)
331
+ node.name.eql?('Step') ? "graph/Step[#{text_of(node, 'Step_name')}]" : "graph/#{node.name}"
332
+ end
333
+
334
+ def check_node(node, ids)
335
+ path = node_path(node)
336
+ factor = text_of(node, 'Synch_factor')
337
+ if factor.nil? || !factor.match?(/\A-?\d+\z/)
338
+ add(ERROR, 'graph.synch_factor', path, 'Synch_factor is missing or not an integer')
339
+ elsif factor.to_i.negative?
340
+ add(ERROR, 'graph.synch_factor_negative', path, 'negative Synch_factor is never met')
341
+ elsif factor.to_i.positive? && %w[x y].any? { |name| node.elements['Synch_factor'].attributes[name].to_s.empty? }
342
+ # the designer writes the position of the join symbol, and draws the workflow with it (WhiteboardsHelper#draw_operator)
343
+ add(WARNING, 'graph.synch_factor_position', path, 'Synch_factor is not 0 and has no attributes x and y (position of the join symbol): the designer shows an empty canvas')
344
+ end
345
+ node.get_elements('Prerequisites/Prerequisite').each do |pre|
346
+ add(ERROR, 'graph.prerequisite_unknown', path, "prerequisite refers to unknown node '#{pre.attributes['id']}'") unless ids.include?(pre.attributes['id'])
347
+ end
348
+ end
349
+
350
+ def check_steps(steps, templates)
351
+ names = []
352
+ steps.each do |step|
353
+ name = text_of(step, 'Step_name')
354
+ type = text_of(step, 'Step_type')
355
+ action_id = text_of(step, 'Action_id')
356
+ path = node_path(step)
357
+ if UNNAMED.include?(name.to_s)
358
+ add(ERROR, 'graph.step_unnamed', "graph/Step[#{step.attributes['id']}]", 'step is not named')
359
+ elsif names.include?(name)
360
+ add(ERROR, 'graph.step_name_not_unique', path, 'step name is not unique')
361
+ end
362
+ names.push(name)
363
+ if UNNAMED.include?(type.to_s)
364
+ add(ERROR, 'graph.step_type_missing', path, 'Step_type is not defined')
365
+ elsif action_id.to_s.empty?
366
+ add(ERROR, 'graph.action_id_missing', path, 'Action_id is missing')
367
+ elsif !templates.include?("#{type}__#{action_id}")
368
+ add(ERROR, 'graph.template_missing', path, "no action template #{type}__#{action_id} in element 3")
369
+ end
370
+ STEP_CHILDREN.each do |child|
371
+ add(ERROR, 'graph.step_element_missing', path, "#{child} is missing (may be empty)") if step.elements[child].nil?
372
+ end
373
+ add(ERROR, 'graph.step_orphan', path, 'no prerequisite, the step never runs (orphaned)') if step.get_elements('Prerequisites/Prerequisite').empty?
374
+ check_output_references(step, steps, path)
375
+ end
376
+ end
377
+
378
+ def check_output_references(step, steps, path)
379
+ step.get_elements('Inputs/Input/Map_to').each do |map|
380
+ next unless map.attributes['type'].eql?('output')
381
+ source = steps.find { |other| text_of(other, 'Step_name').eql?(map.attributes['step_name']) }
382
+ next unless source.nil? || !text_of(source, 'Step_type').eql?(map.attributes['step_type']) || !text_of(source, 'Action_id').eql?(map.attributes['action_id'])
383
+ add(ERROR, 'graph.output_unknown', path, "input #{map.parent.attributes['name']} refers to unknown output of step '#{map.attributes['step_name']}' (step_name, step_type, action_id must match)")
384
+ end
385
+ end
386
+
387
+ def check_branches(closing, steps)
388
+ success = ->(pre) { SUCCESS_STATUSES.include?(pre.attributes['status'].to_s.downcase) }
389
+ consumers = (closing + steps).flat_map { |node| node.get_elements('Prerequisites/Prerequisite').select(&success).map { |pre| pre.attributes['id'] } }
390
+ steps.each do |step|
391
+ next if consumers.include?(step.attributes['id'])
392
+ add(ERROR, 'graph.step_no_output', node_path(step), 'no node follows it with status Complete (step does not have output)')
393
+ end
394
+ return unless closing.all? { |node| node.get_elements('Prerequisites/Prerequisite').empty? }
395
+ add(ERROR, 'graph.no_termination', 'graph', 'workflow does not terminate: End, Fail and Error have no prerequisite')
396
+ end
397
+
398
+ # Starvation: a node that waits for exclusive conditions never starts, and the work order never ends.
399
+ # Conditions of a node: pairs [step id, status] that must all be true for the node to start.
400
+ # With Synch_factor 0 (AND), those of all its prerequisites; otherwise (OR), those common to all its prerequisites.
401
+ # A prerequisite on a step adds its status to the conditions of that step.
402
+ # A step ends with one status: two statuses of the same step in the conditions of a node are exclusive,
403
+ # for example the Complete and Failed exits of a step, or an if and an else branch joined without OR.
404
+ # Not checked: steps after an ArrayFanout (one execution per element, with different statuses).
405
+ def check_starvation(start_id, closing, steps)
406
+ nodes = (closing + steps).to_h { |node| [node.attributes['id'], node] }
407
+ fanned = steps.select do |step|
408
+ step.get_elements('Prerequisites/Prerequisite').any? { |pre| text_of(nodes[pre.attributes['id']] || step, 'Step_type').eql?('ArrayFanout') }
409
+ end.map { |step| step.attributes['id'] }
410
+ memo = {}
411
+ conditions = lambda do |id, visiting|
412
+ return [] if id.eql?(start_id) || !nodes.key?(id) || visiting.include?(id)
413
+ return memo[id] if memo.key?(id)
414
+ node = nodes[id]
415
+ branches = node.get_elements('Prerequisites/Prerequisite').map do |pre|
416
+ source = pre.attributes['id']
417
+ own = source.eql?(start_id) || fanned.include?(source) ? [] : [[source, pre.attributes['status'].to_s.downcase]]
418
+ (own + conditions.call(source, visiting + [id])).uniq
419
+ end
420
+ memo[id] = if branches.empty? then []
421
+ elsif text_of(node, 'Synch_factor').to_i.zero? then branches.reduce(:|)
422
+ else branches.reduce(:&)
423
+ end
424
+ end
425
+ exclusive = ->(list) { list.group_by(&:first).select { |_, pairs| pairs.size > 1 }.transform_values { |pairs| pairs.map(&:last) } }
426
+ nodes.each do |id, node|
427
+ conflicts = exclusive.call(conditions.call(id, []))
428
+ # report where the conflict starts, not on the nodes that follow
429
+ inherited = node.get_elements('Prerequisites/Prerequisite').flat_map { |pre| exclusive.call(conditions.call(pre.attributes['id'], [])).keys }
430
+ conflicts.except(*inherited).each do |step_id, statuses|
431
+ step_name = text_of(nodes[step_id], 'Step_name')
432
+ add(
433
+ ERROR, 'graph.starvation', node_path(node),
434
+ "never starts: it waits for step '#{step_name}' to end with #{statuses.map(&:capitalize).join(' and ')}, which are exclusive; " \
435
+ 'join exclusive branches with an OR (Synch_factor 1), otherwise the work order never ends'
436
+ )
437
+ end
438
+ end
439
+ end
440
+
441
+ # Loop: a link back to a node it comes from. A step runs once in a work order: it does not run again.
442
+ # @return [Boolean] true if a loop was found
443
+ def check_loops(closing, steps)
444
+ found = false
445
+ nodes = (closing + steps).to_h { |node| [node.attributes['id'], node] }
446
+ sources = nodes.transform_values { |node| node.get_elements('Prerequisites/Prerequisite').map { |pre| pre.attributes['id'] }.uniq & nodes.keys }
447
+ state = {}
448
+ visit = lambda do |id|
449
+ state[id] = :visiting
450
+ sources[id].each do |source|
451
+ if state[source].eql?(:visiting)
452
+ found = true
453
+ add(
454
+ ERROR, 'graph.loop', node_path(nodes[source]),
455
+ "link back to '#{text_of(nodes[id], 'Step_name') || nodes[id].name}' (loop): a step runs only once in a work order; " \
456
+ 'to repeat, use a sub-workflow with retries, or an ArrayFanout'
457
+ )
458
+ elsif state[source].nil?
459
+ visit.call(source)
460
+ end
461
+ end
462
+ state[id] = :done
463
+ end
464
+ nodes.each_key { |id| visit.call(id) if state[id].nil? }
465
+ found
466
+ end
467
+
468
+ def check_parameters(root, steps)
469
+ linked = (root.get_elements('Start/Parameters/Parameter') + steps.flat_map { |step| step.get_elements('Parameters/Parameter') }).map { |link| link.attributes['id'] }
470
+ defined = {}
471
+ root.get_elements('Parameters/Parameter').each do |parameter|
472
+ name = text_of(parameter, 'Name')
473
+ type = text_of(parameter, 'Value_type')
474
+ path = "graph/Parameter[#{name || parameter.attributes['id']}]"
475
+ add(ERROR, 'graph.parameter_name', path, 'Name is missing') if name.to_s.empty?
476
+ add(ERROR, 'graph.parameter_type', path, "Value_type must be one of #{VALUE_TYPES.join(', ')}") unless VALUE_TYPES.include?(type)
477
+ mandatory_at_runtime = text_of(parameter, 'Runtime_override').eql?('true') && !text_of(parameter, 'Runtime_optional').eql?('true')
478
+ add(WARNING, 'graph.parameter_value', path, 'Value is missing') if !mandatory_at_runtime && parameter.elements['Value'].nil?
479
+ defined[name] = parameter.attributes['id']
480
+ end
481
+ steps.each do |step|
482
+ step.get_elements('Inputs/Input/Map_to').each do |map|
483
+ next unless PARAMETER_MAPPINGS.include?(map.attributes['type'])
484
+ id = defined[map.attributes['name']]
485
+ if id.nil?
486
+ add(ERROR, 'graph.parameter_undefined', node_path(step), "input refers to undefined parameter '#{map.attributes['name']}'")
487
+ elsif !linked.include?(id)
488
+ add(ERROR, 'graph.parameter_not_linked', node_path(step), "parameter '#{map.attributes['name']}' is not linked to Start or to a step: inputs mapped to it are not created")
489
+ end
490
+ end
491
+ end
492
+ end
493
+
494
+ # --- Checks with the specifications of plugins ---
495
+
496
+ def blank?(value)
497
+ value.nil? || value.to_s.strip.empty?
498
+ end
499
+
500
+ # @param values [Hash{String=>String}] Attributes of the template
501
+ def check_template_attributes(path, type, values)
502
+ spec = @specs[type]
503
+ if spec.nil?
504
+ similar = @specs.keys.find { |name| name.casecmp?(type) }
505
+ return add(ERROR, 'template.plugin_case', path, "plugin #{type} does not exist, class names are case sensitive: #{similar}") if similar
506
+ return add(WARNING, 'template.plugin_unknown', path, "plugin #{type} is not in the specifications: its template and the inputs of its steps are not checked")
507
+ end
508
+ columns = spec['attributes'] || {}
509
+ values.each do |name, value|
510
+ attribute = columns[name]
511
+ if attribute.nil?
512
+ next if TEMPLATE_COLUMNS.include?(name)
513
+ next add(ERROR, 'template.attribute_unknown', path, "#{type} has no attribute #{name}: the import fails") if @plain
514
+ next add(WARNING, 'template.attribute_unknown', path, "#{type} has no attribute #{name}: the import may fail")
515
+ end
516
+ # a placeholder is replaced at run time
517
+ next if blank?(value) || value.include?('<%')
518
+ check_value(path, name, attribute, value)
519
+ end
520
+ end
521
+
522
+ # Values are cast by the database layer (ActiveRecord): a text that is not false is true, a text that is not a number is 0
523
+ def check_value(path, name, attribute, value)
524
+ case attribute['type']
525
+ when 'boolean'
526
+ return if (TRUE_VALUES + FALSE_VALUES).include?(value.downcase)
527
+ return add(WARNING, 'template.attribute_value', path, "#{name}: #{value} is read as true: write 1 or 0")
528
+ when *INTEGER_TYPES
529
+ return add(WARNING, 'template.attribute_value', path, "#{name}: #{value} is not an integer, it is read as #{value.to_i}") unless value.match?(/\A-?\d+\z/)
530
+ end
531
+ allowed = attribute['values']
532
+ return if allowed.nil? || allowed.map(&:to_s).include?(value)
533
+ add(ERROR, 'template.attribute_value', path, "#{name}: #{value} is not an allowed value: #{allowed.join(', ')}")
534
+ end
535
+
536
+ # Value of an attribute as the plugin sees it: typed for booleans and integers
537
+ def typed(value, type)
538
+ return value unless value.is_a?(String)
539
+ case type
540
+ when 'boolean' then !FALSE_VALUES.include?(value.downcase)
541
+ when *INTEGER_TYPES then value.to_i
542
+ else value
543
+ end
544
+ end
545
+
546
+ # Values of the attributes of a template as the plugin sees them: nil when blank
547
+ # An attribute not in the template takes its database default; in plain format, so does a blank one (and false is blank)
548
+ # @return [Hash{String=>Object}]
549
+ def effective_values(spec, values)
550
+ (spec['attributes'] || {}).to_h do |name, attribute|
551
+ value = values[name]
552
+ value = nil if blank?(value) || (@plain && value.eql?('false'))
553
+ value = attribute['default'] if value.nil? && (@plain || !values.key?(name))
554
+ [name, typed(value, attribute['type'])]
555
+ end
556
+ end
557
+
558
+ # @param conditions [Hash] `if_blank`, `if_set`, `when` and `conditional` of an input or output of the specifications
559
+ # @return [Boolean, nil] true if the conditions are met, nil if unknown
560
+ def met(conditions, values)
561
+ return if conditions['conditional']
562
+ Array(conditions['if_blank']).all? { |name| values[name].nil? } &&
563
+ Array(conditions['if_set']).all? { |name| !values[name].nil? } &&
564
+ (conditions['when'] || {}).all? do |name, condition|
565
+ value = values[name]
566
+ if condition.key?('in')
567
+ condition['in'].any? { |item| item.nil? ? value.nil? : item.to_s.eql?(value.to_s) }
568
+ else
569
+ !value.nil? && condition['not_in'].none? { |item| item.to_s.eql?(value.to_s) }
570
+ end
571
+ end
572
+ end
573
+
574
+ # @return [Array<String>] Names declared by a type map `{'name'=>'type'}` (Ruby or JSON syntax) or a list `['name']`
575
+ def declared_names(value)
576
+ text = value.strip
577
+ return text.scan(/['"]([^'"]+)['"]\s*(?:=>|:)/).flatten if text.start_with?('{')
578
+ return text.scan(/['"]([^'"]+)['"]/).flatten if text.start_with?('[')
579
+ []
580
+ end
581
+
582
+ # Names given by the value of a template attribute, for a name in angle brackets (`<attribute>`) in the specification
583
+ # @return [Array<String>, nil] The names, or `nil` if they cannot be found in the value
584
+ def names_in_value(value)
585
+ text = value.to_s.strip
586
+ return [] if text.empty?
587
+ return [text] if text.match?(SINGLE_NAME)
588
+ header = text.lines.first.strip
589
+ return header.split(TABLE_SEPARATOR).map(&:strip) if header.include?(TABLE_SEPARATOR)
590
+ nil
591
+ end
592
+
593
+ # Inputs and outputs of a step, for the attributes of its template
594
+ # @return [Hash] `:inputs` and `:outputs`: name => {`exists`, `required`} (true, false or nil if unknown),
595
+ # and `:complete`: kind => true if all inputs (or outputs) are known, so that an other one does not exist
596
+ def step_interface(spec, attributes)
597
+ values = effective_values(spec, attributes)
598
+ interface = {inputs: {}, outputs: {}, complete: {}}
599
+ %i[inputs outputs].each do |kind|
600
+ # false when names come from an attribute value whose format is not known
601
+ names_known = true
602
+ (spec[kind.to_s] || {}).each do |name, info|
603
+ exists = met(info, values)
604
+ next if exists.eql?(false)
605
+ names = [name]
606
+ # a name in angle brackets comes from the value of that attribute
607
+ if name =~ /\A<(\w+)>\z/
608
+ names = names_in_value(values[Regexp.last_match(1)])
609
+ names_known = false if names.nil?
610
+ end
611
+ required = info['required'].is_a?(Hash) ? met(info['required'], values) : info['required']
612
+ Array(names).each do |each_name|
613
+ interface[kind][each_name] = {'exists' => exists, 'required' => required.eql?('depends') ? nil : required}
614
+ end
615
+ end
616
+ declared = (spec['attributes'] || {}).values.any? { |attribute| attribute['declares'].to_s.end_with?(kind.to_s) }
617
+ partly = Array(spec['notes']).any? { |note| note.match?(PARTLY_DEFINED[kind]) }
618
+ interface[:complete][kind] = names_known && (declared || !partly) && blank?(attributes[SPEC_CODE[kind]])
619
+ end
620
+ (spec['attributes'] || {}).each do |name, attribute|
621
+ value = attributes[name]
622
+ next if blank?(value)
623
+ case attribute['declares']
624
+ when 'required_inputs', 'optional_inputs'
625
+ required = attribute['declares'].eql?('required_inputs')
626
+ declared_names(value).each { |input| interface[:inputs][input] ||= {'exists' => true, 'required' => required} }
627
+ when 'outputs'
628
+ declared_names(value).each { |output| interface[:outputs][output] ||= {'exists' => true} }
629
+ end
630
+ next unless attribute['placeholders']
631
+ value.scan(PLACEHOLDER).flatten.each { |input| interface[:inputs][input] ||= {'exists' => true, 'required' => true} }
632
+ # inputs of other placeholders (functions of variables) are not known
633
+ interface[:complete][:inputs] = false if value.scan('<%').size > value.scan(PLACEHOLDER).size
634
+ end
635
+ interface
636
+ end
637
+
638
+ def check_step_interfaces(steps)
639
+ interfaces = {}
640
+ interface_of = lambda do |step|
641
+ type = text_of(step, 'Step_type')
642
+ key = "#{type}__#{text_of(step, 'Action_id')}"
643
+ next if @specs[type].nil? || !@template_attributes.key?(key)
644
+ interfaces[key] ||= step_interface(@specs[type], @template_attributes[key])
645
+ end
646
+ steps.each do |step|
647
+ path = node_path(step)
648
+ type = text_of(step, 'Step_type')
649
+ interface = interface_of.call(step)
650
+ unless interface.nil?
651
+ inputs = step.get_elements('Inputs/Input').map { |input| input.attributes['name'] }
652
+ # a carried variable is also given to the step under its name
653
+ provided = inputs + step.get_elements('Outputs/Carry_thru').map { |carry| carry.attributes['name'] }
654
+ unknown = interface[:complete][:inputs] ? inputs - interface[:inputs].keys : []
655
+ unknown.each { |name| add(WARNING, 'step.input_unknown', path, "#{type} has no input #{name} with this template: Orchestrator ignores it") }
656
+ interface[:inputs].each do |name, input|
657
+ next unless input['exists'].eql?(true) && input['required'].eql?(true) && !provided.include?(name)
658
+ add(ERROR, 'step.input_missing', path, "required input #{name} is not mapped")
659
+ end
660
+ if interface[:complete][:outputs]
661
+ (step.get_elements('Outputs/Output').map { |output| output.attributes['name'] } - interface[:outputs].keys).each do |name|
662
+ add(WARNING, 'step.output_unknown', path, "#{type} has no output #{name} with this template: its settings are ignored")
663
+ end
664
+ end
665
+ end
666
+ step.get_elements('Inputs/Input/Map_to').each do |map|
667
+ next unless map.attributes['type'].eql?('output')
668
+ source = steps.find { |other| STEP_IDENTITY.all? { |name| text_of(other, name).eql?(map.attributes[name.downcase]) } }
669
+ source_interface = source && interface_of.call(source)
670
+ next if source_interface.nil? || !source_interface[:complete][:outputs] || source_interface[:outputs].key?(map.attributes['name'])
671
+ add(
672
+ WARNING, 'step.output_not_produced', path,
673
+ "input #{map.parent.attributes['name']} maps output #{map.attributes['name']} of step '#{map.attributes['step_name']}', which #{map.attributes['step_type']} does not produce with its template"
674
+ )
675
+ end
676
+ end
677
+ end
678
+ end
679
+ end