metanorma-plugin-lutaml 0.6.0 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e9b3ce23d274ff457c34d3c5a9e2bf73d12d8cd4de898a061f6a3522bb45439
-  data.tar.gz: 79a063b985b79fa7b835bbc73acd0ada7761c496f058bcfc8d455882cb6d95ea
+  metadata.gz: 19d2399f31818f5f72d0f6237199042a1b729457039c033e5a45d29645b56eda
+  data.tar.gz: 0a3a779c6e6b556c7597fc2f9fc708565fb3e0841969eccc855334fcf9e98a88
 SHA512:
-  metadata.gz: 20896ef23b4b4a2258e3803a9d07bca26778f495aa57dfdd5bcd878ac6db1c09d139a066b7e6dde086e25b1c4a77b74266ae7f51f251613c783a4be7d38771da
-  data.tar.gz: b10b0ce496d8ac08e4cd83d85065c94292f4846dcaf27544fdc122fba228c9303395076d42e2fd5eecc12090f16faeb564b0300c7bc40ea37f3648dae433ec25
+  metadata.gz: a03c8868605097a556281eae23b1f2a9b40f3991e7f7304c0f70ef2d3f8e8477fbd69389c2c49482b0774b729dbb02a8bbb39f3949b7453a5542e4a957c4a70e
+  data.tar.gz: 1001f84dc00d0430552d89f63525af328fa1d2fec6143ce7fbf5c49764f83f5d27b0673d4c57756514498236eb91857cadaf981400097e2d1556f8a554d962c1

data/README.adoc

@@ -28,9 +28,9 @@ LutaML supports accessing EXPRESS models via the
 https://github.com/lutaml/expressir[Expressir] parser.
 
 
-=== Usage of the `lutaml` macro
+=== Usage of the `lutaml` command
 
-Given an `example.exp` EXPRESS file with the content:
+Given an `example.exp` EXPRESS file with content:
 
 [source,exp]
 ----
@@ -57,7 +57,7 @@ SCHEMA test_schema 'test';
 END_SCHEMA;
 ----
 
-And the `lutaml` macro block:
+And the `lutaml` block:
 
 [source,adoc]
 -----
@@ -137,6 +137,7 @@ Example of usage:
 = Document title
 Author
 :lutaml-express-index: index_name; /path/to/express_files; cache=/path/to/cache_file.yaml
+
 [lutaml,index_name,context]
 ----
 {% for schema in context.schemas %}
@@ -145,6 +146,64 @@ Author
 ----
 -----
 
+=== Using `config.yaml`
+
+This functionality allows `[lutaml_express]` blocks to load a full set of
+EXPRESS schemas in one index, and then provide a select ("filter") option
+per-block via a separate YAML file.
+
+[source,adoc]
+----
+:lutaml-express-index: schemas_1; schemas_all.yaml;
+
+[lutaml_express,schemas_1,repo,leveloffset=+1,config_yaml=select.yaml]
+---
+{% assign selected = repo.schemas | where: "selected" %}
+{% render "templates/resources/schema" for selected as schema %}
+---
+----
+
+Where `schemas_all.yml` provides all schemas:
+
+[source,yaml]
+----
+---
+schemas:
+  action_schema:
+    path: "../../schemas/resources/action_schema/action_schema.exp"
+  application_context_schema:
+    path: "../../schemas/resources/application_context_schema/application_context_schema.exp"
+  approval_schema:
+    path: "../../schemas/resources/approval_schema/approval_schema.exp"
+...
+----
+
+And `select.yaml` only selects 2 schemas:
+
+[source,yaml]
+----
+---
+schemas:
+  - action_schema
+  - application_context_schema
+----
+
+The resulting block adds the `select` attribute to every schema of the the
+"context" object, which allows you to filter those out for complex operations
+via Liquid:
+
+[source,liquid]
+----
+[lutaml_express,schemas_1,repo,leveloffset=+1,config_yaml=select.yaml]
+---
+{% assign selected = repo.schemas | where: "selected" %}
+... do things with `selected` ...
+----
+
+NOTE: This functionality is used in the ISO 10303 SRL to load the full schema
+set at once but only render the selected schemas in individual documents.
+
+
 == Usage with UML
 
 === Rendering a LutaML view: `lutaml_diagram`

data/lib/metanorma/plugin/lutaml/liquid/multiply_local_file_system.rb

@@ -12,13 +12,18 @@ module Metanorma
 
           def read_template_file(template_path)
             full_path = full_path(template_path)
-            raise FileSystemError, "No such template '#{template_path}'" unless File.exist?(full_path)
+
+            unless File.exist?(full_path)
+              raise FileSystemError, "No such template '#{template_path}'"
+            end
 
             File.read(full_path)
           end
 
           def full_path(template_path)
-            raise ::Liquid::FileSystemError, "Illegal template name '#{template_path}'" unless %r{\A[^./][a-zA-Z0-9_/]+\z}.match?(template_path)
+            unless %r{\A[^./][a-zA-Z0-9_/]+\z}.match?(template_path)
+              raise ::Liquid::FileSystemError, "Illegal template name '#{template_path}'"
+            end
 
             result_path = if template_path.include?('/')
               roots
@@ -40,6 +45,10 @@ module Metanorma
                 .find { |path| File.file?(path) }
             end
 
+            if result_path.nil?
+              raise ::Liquid::FileSystemError, "No documents in template path '#{File.expand_path(template_path)}'"
+            end
+
             unless roots.any? { |root| File.expand_path(result_path).start_with?(File.expand_path(root)) }
               raise ::Liquid::FileSystemError, "Illegal template path '#{File.expand_path(result_path)}'"
             end

data/lib/metanorma/plugin/lutaml/lutaml_preprocessor.rb

@@ -16,164 +16,256 @@ module Metanorma
         REMARKS_ATTRIBUTE = "remarks"
 
         def process(document, reader)
-          r = Asciidoctor::PreprocessorNoIfdefsReader.new document, reader.lines
+          r = Asciidoctor::PreprocessorNoIfdefsReader.new(document, reader.lines)
           input_lines = r.readlines.to_enum
-          has_lutaml = !input_lines.select { |x| lutaml?(x) }.empty?
+
+          has_lutaml = input_lines.any? { |line| lutaml?(line) }
+
           express_indexes = Utils.parse_document_express_indexes(
-            document, input_lines
+            document,
+            input_lines
           )
-          result_content = processed_lines(document, input_lines,
-                                           express_indexes)
-          has_lutaml and log(document, result_content)
+
+          result_content = process_input_lines(
+            document: document,
+            input_lines: input_lines,
+            express_indexes: express_indexes,
+          )
+
+          log(document, result_content) if has_lutaml
+
           Asciidoctor::PreprocessorNoIfdefsReader.new(document, result_content)
         end
 
         protected
 
-        def log(document, result)
-          File.open("#{document.attr('docfile')}.lutaml.log.txt",
-                    "w:UTF-8") do |f|
-            f.write(result.join("\n"))
+        def log(doc, text)
+          File.open("#{doc.attr('docfile')}.lutaml.log.txt", "w:UTF-8") do |f|
+            f.write(text.join("\n"))
           end
         end
 
         def lutaml?(line)
-          line.match(/^\[(?:\blutaml\b|\blutaml_express\b),([^,]+)?,?([^,]+)?,?([^,]+)?\]/)
+          line.match(/^\[(?:\blutaml\b|\blutaml_express\b),(?<index_names>[^,]+)?,?(?<context_name>[^,]+)?(?<options>,.*)?\]/)
         end
 
-        def content_from_files(document, file_paths)
-          file_list = file_paths.map do |file_path|
-            File.new(Utils.relative_file_path(document, file_path),
-                     encoding: "UTF-8")
-          end
-          ::Lutaml::Parser.parse(file_list)
+        def load_lutaml_file(document, file_path)
+          ::Lutaml::Parser.parse(
+            File.new(
+              Utils.relative_file_path(document, file_path),
+              encoding: "UTF-8"
+            )
+          )
         end
 
         private
 
-        def processed_lines(document, input_lines, express_indexes)
+        def process_input_lines(
+            document:,
+            input_lines:,
+            express_indexes:
+          )
+
           result = []
           loop do
-            result
-              .push(*process_text_blocks(
+            result.push(
+              *process_text_blocks(
                 document,
                 input_lines,
                 express_indexes,
-              ))
+              )
+            )
           end
           result
         end
 
         def process_text_blocks(document, input_lines, express_indexes)
           line = input_lines.next
-          block_match = lutaml?(line)
-          return [line] if block_match.nil?
+          block_header_match = lutaml?(line)
+
+          return [line] if block_header_match.nil?
+
+          index_names = block_header_match[:index_names].split(";").map(&:strip)
+          context_name = block_header_match[:context_name].strip
+
+          options = block_header_match[:options] &&
+            parse_options(block_header_match[:options].to_s.strip) || {}
 
           end_mark = input_lines.next
-          parse_template(document,
-                         collect_internal_block_lines(document,
-                                                      input_lines,
-                                                      end_mark),
-                         block_match,
-                         express_indexes)
+
+          render_template(
+            document: document,
+            lines: extract_block_lines(input_lines, end_mark),
+            index_names: index_names,
+            context_name: context_name,
+            options: options,
+            indexes: express_indexes
+          )
         end
 
-        def collect_internal_block_lines(_document, input_lines, end_mark)
-          current_block = []
+        def extract_block_lines(input_lines, end_mark)
+          block = []
           while (block_line = input_lines.next) != end_mark
-            current_block.push(block_line)
+            block.push(block_line)
           end
-          current_block
+          block
         end
 
-        def contexts_items(block_match, document, express_indexes)
-          contexts_names = block_match[1].split(";").map(&:strip)
-          file_paths = []
-          result = contexts_names.each_with_object([]) do |path, res|
-            if express_indexes[path]
-              res.push(express_indexes[path])
+        def gather_context_items(index_names:, document:, indexes:)
+          index_names.map do |path|
+            # TODO: Rephrase of the below TODO message.
+            # ::Lutaml::Parser.parse(file_list) can return an Array or just one.
+            # TODO: decide how to handle expressir multiply file parse as one
+            # object and lutaml
+
+            # Does this condition ever happen? That is only if the
+            # `lutaml-express-index` condition is not set
+            unless indexes[path]
+
+              full_path = Utils.relative_file_path(document, path)
+              unless File.file?(full_path)
+                raise StandardError.new(
+                  "Unable to load EXPRESS index for `#{path}`, " \
+                  "please define it at `:lutaml-express-index:` or specify " \
+                  "the full path."
+                )
+              end
+              wrapper = load_lutaml_file(document, path)
+              indexes[path] = {
+                wrapper: wrapper,
+                serialized_hash: wrapper.to_liquid
+              }
             else
-              file_paths.push(path)
+              indexes[path][:serialized_hash] ||= indexes[path][:wrapper].to_liquid
             end
+
+            indexes[path]
           end
-          if !file_paths.empty?
-            from_files = content_from_files(document, file_paths)
-            # TODO: decide how to handle expressir multiply file parse as one object and lutaml
-            if from_files.is_a?(Array)
-              result.push(*from_files.map(&:to_liquid))
-            else
-              from_files = from_files.to_liquid
-              from_files["schemas"] = from_files["schemas"].map do |n|
-                n.merge("relative_path_prefix" => Utils.relative_file_path(document, File.dirname(n["file"])))
-              end
-              result.push(from_files)
+        end
+
+        def parse_yaml_config_file(document, file_path)
+          return nil if file_path.nil?
+
+          relative_file_path = Utils.relative_file_path(document, file_path)
+          YAML.safe_load(File.read(relative_file_path, encoding: "UTF-8"))
+        end
+
+        def decorate_schema_object(schema:, document:, indexes:, index_names:, selected:, options:)
+          # Mark a schema as "selected" with `.selected`
+          schema["selected"] = true if selected
+
+          # Provide pretty-formatted code under `.formatted`
+          index_found_key, index_found_value = indexes.detect do |k,v|
+            found = v[:wrapper].original_document.schemas.detect do |s|
+              s.id == schema["id"]
             end
           end
-          result
+
+          schema["formatted"] = index_found_value[:wrapper].original_document.schemas.detect do |s|
+            s.id == schema["id"]
+          end.to_s(no_remarks: true)
+
+          # Decorate the remaining things
+          decorate_context_items(
+            schema,
+            options.merge(
+              "relative_path_prefix" =>
+                Utils.relative_file_path(document, File.dirname(schema["file"]))
+            )
+          ) || {}
         end
 
-        def parse_template(document, current_block, block_match,
-express_indexes)
-          options = parse_options(block_match[3])
-          contexts_items(block_match, document, express_indexes)
-            .map do |items|
-              if items["schemas"]
-                items["schemas"] = items["schemas"].map do |j|
-                  opts = options.merge("relative_path_prefix" => j["relative_path_prefix"])
-                  decorate_context_items(j, opts)
-                end
+        def render_template(document:, lines:, context_name:, index_names:, options:, indexes:)
+
+          config_yaml_path = options.delete("config_yaml")
+          config_yaml = config_yaml_path ?
+            parse_yaml_config_file(document, config_yaml_path) :
+            {}
+
+          selected_schemas = config_yaml["schemas"]
+
+          gather_context_items(
+            index_names: index_names,
+            document: document,
+            indexes: indexes
+          ).map do |items|
+
+            serialized_hash = items[:serialized_hash]
+
+            if serialized_hash["schemas"]
+              serialized_hash["schemas"].map! do |schema|
+
+                decorate_schema_object(
+                  schema: schema,
+                  document: document,
+                  index_names: index_names,
+                  indexes: indexes,
+                  selected: selected_schemas && selected_schemas.include?(schema["id"]),
+                  options: options
+                )
               end
-              parse_context_block(document: document,
-                                  context_lines: current_block,
-                                  context_items: items,
-                                  context_name: block_match[2].strip)
-            end.flatten
+            end
+
+            render_block(
+              document: document,
+              block_lines: lines,
+              context_items: serialized_hash,
+              context_name: context_name
+            )
+          end.flatten
         rescue StandardError => e
-          document.logger.warn("Failed to parse lutaml block: #{e.message}")
+          raise e
+          document.logger.warn("Failed to parse LutaML block: #{e.message}")
           []
         end
 
         def parse_options(options_string)
           options_string
             .to_s
-            .scan(/(.+?)=(\s?[^\s]+)/)
+            .scan(/,\s*([^=]+?)=(\s*[^,]+)/)
             .map { |elem| elem.map(&:strip) }
             .to_h
         end
 
-        def decorate_context_items(context_items, options)
-          return context_items if !context_items.is_a?(Hash)
-
-          context_items
-            .map do |(key, val)|
-              if val.is_a?(Hash)
-                [key, decorate_context_items(val, options)]
-              elsif key == REMARKS_ATTRIBUTE
-                [key,
-                 val&.map do |remark|
-                   Metanorma::Plugin::Lutaml::ExpressRemarksDecorator
-                     .call(remark, options)
-                 end]
-              elsif val.is_a?(Array)
-                [key, val.map { |n| decorate_context_items(n, options) }]
-              else
-                [key, val]
+        def decorate_context_item(key, val, options)
+          if key == REMARKS_ATTRIBUTE
+            return [
+              key,
+              val&.map do |remark|
+                Metanorma::Plugin::Lutaml::ExpressRemarksDecorator
+                  .call(remark, options)
               end
-            end
-            .to_h
+            ]
+          end
+
+          case val
+          when Hash
+            [key, decorate_context_items(val, options)]
+          when Array
+            [key, val.map { |n| decorate_context_items(n, options) }]
+          else
+            [key, val]
+          end
+        end
+
+        def decorate_context_items(item, options)
+          return item unless item.is_a?(Hash)
+
+          item.map do |(key, val)|
+            decorate_context_item(key, val, options)
+          end.to_h
         end
 
-        def parse_context_block(context_lines:,
-                                context_items:,
-                                context_name:,
-                                document:)
+        def render_block(block_lines:, context_items:, context_name:, document:)
           render_result, errors = Utils.render_liquid_string(
-            template_string: context_lines.join("\n"),
+            template_string: block_lines.join("\n"),
             context_items: context_items,
             context_name: context_name,
             document: document,
           )
+
           Utils.notify_render_errors(document, errors)
+
           render_result.split("\n")
         end
       end

data/lib/metanorma/plugin/lutaml/lutaml_uml_datamodel_description_preprocessor.rb

@@ -201,7 +201,8 @@ options)
         end
 
         def package_entities(options)
-          options["packages"] or return {}
+          return {} unless options["packages"]
+
           options["packages"]
             .find_all { |entity| entity.is_a?(Hash) && entity.values.first["render_entities"] }
             .map do |entity|

data/lib/metanorma/plugin/lutaml/utils.rb

@@ -27,16 +27,22 @@ module Metanorma
         def render_liquid_string(template_string:, context_items:,
                                  context_name:, document:, include_path: nil)
           liquid_template = ::Liquid::Template.parse(template_string)
+
           # Allow includes for the template
-          include_paths = [Utils.relative_file_path(document, ""),
-                           include_path].compact
+          include_paths = [
+            Utils.relative_file_path(document, ""),
+            include_path
+          ].compact
+
           liquid_template.registers[:file_system] =
             ::Metanorma::Plugin::Lutaml::Liquid::LocalFileSystem
               .new(include_paths, ["_%s.liquid", "_%s.adoc"])
+
           rendered_string = liquid_template
             .render(context_name => context_items,
                     strict_variables: true,
                     error_mode: :warn)
+
           [rendered_string, liquid_template.errors]
         end
 
@@ -48,37 +54,52 @@ module Metanorma
           end
         end
 
-        def process_express_index(path, cache_path, document,
-force_read = false)
-          if cache_path
-            cache_full_path = Utils.relative_file_path(document,
-                                                       cache_path)
-          end
+        def load_express_repositories(path:, cache_path:, document:, force_read: false)
+
+          cache_full_path = cache_path &&
+            Utils.relative_file_path(document, cache_path)
+
+          # If there is cache and "force read" not set.
           if !force_read && cache_full_path && File.file?(cache_full_path)
-            return express_from_cache(cache_full_path)
+            return load_express_repo_from_cache(cache_full_path)
           end
 
+          # If there is no cache or "force read" is set.
           full_path = Utils.relative_file_path(document, path)
-          wrapper = express_from_path(document, full_path)
+          lutaml_wrapper = load_express_repo_from_path(document, full_path)
+
           if cache_full_path && !File.file?(cache_full_path)
-            express_write_cache(cache_full_path, wrapper.original_document,
-                                document)
+            save_express_repo_to_cache(
+              cache_full_path,
+              lutaml_wrapper.original_document,
+              document
+            )
           end
-          wrapper
+
+          lutaml_wrapper
+
         rescue Expressir::Error
           FileUtils.rm_rf(cache_full_path)
-          process_express_index(path, cache_path, document, true)
+
+          load_express_repositories(
+            path: path,
+            cache_path: cache_path,
+            document: document,
+            force_read: true
+          )
+
         rescue StandardError => e
           document.logger.warn("Failed to load #{full_path}: #{e.message}")
-          nil
+          raise e
+          # nil
         end
 
-        def express_from_cache(path)
+        def load_express_repo_from_cache(path)
           ::Lutaml::Parser
             .parse(File.new(path), ::Lutaml::Parser::EXPRESS_CACHE_PARSE_TYPE)
         end
 
-        def express_write_cache(path, repository, document)
+        def save_express_repo_to_cache(path, repository, document)
           root_path = Pathname.new(relative_file_path(document, ""))
           Expressir::Express::Cache
             .to_file(path,
@@ -86,47 +107,33 @@ force_read = false)
                      root_path: root_path)
         end
 
-        def express_from_path(document, path)
-          if File.directory?(path)
-            return express_from_folder(path)
-          end
+        def load_express_repo_from_path(document, path)
+          return load_express_from_folder(path) if File.directory?(path)
 
-          express_from_index(document, path)
+          load_express_from_index(document, path)
         end
 
-        def express_from_folder(folder)
+        def load_express_from_folder(folder)
           files = Dir["#{folder}/*.exp"].map do |nested_path|
             File.new(nested_path, encoding: "UTF-8")
           end
           ::Lutaml::Parser.parse(files)
         end
 
-        def express_decorate_wrapper(wrapper, document)
-          serialized = wrapper.to_liquid
-          serialized["schemas"].map! do |j|
-            j.merge(
-              "relative_path_prefix" => Utils
-                .relative_file_path(document, File.dirname(j["file"])),
-              "formatted" => wrapper.original_document
-                .schemas.detect {|s| s.id == j["id"] }.to_s(no_remarks: true)
-            )
-          end
-          serialized
-        end
-
-        def express_from_index(document, path)
+        def load_express_from_index(document, path)
           yaml_content = YAML.safe_load(File.read(path))
           root_path = yaml_content["path"]
-          schemas_paths = yaml_content
-            .fetch("schemas")
+          schemas_paths = yaml_content["schemas"]
             .map do |(schema_name, schema_values)|
             schema_values["path"] || File.join(root_path.to_s,
                                                "#{schema_name}.exp")
           end
+
           files_to_load = schemas_paths.map do |path|
             File.new(Utils.relative_file_path(document, path),
                      encoding: "UTF-8")
           end
+
           ::Lutaml::Parser.parse(files_to_load)
         end
 
@@ -134,20 +141,35 @@ force_read = false)
           express_indexes = {}
           loop do
             line = input_lines.next
+
+            # Finished parsing document attributes
             break if line.empty?
 
             match = line.match(LUTAML_EXP_IDX_TAG)
-            if match
-              name = match[:index_name]
-              path = match[:index_path]
-              cache = match[:cache_path]
-              repositories = process_express_index(path.strip, cache, document)
-              if repositories
-                express_indexes[name.strip] =
-                  express_decorate_wrapper(repositories, document)
-              end
+            next unless match
+
+            name = match[:index_name]&.strip
+            path = match[:index_path]&.strip
+            cache = match[:cache_path]&.strip
+
+            unless name && path
+              raise StandardError.new("No name and path set in `:lutaml-express-index:` attribute.")
+            end
+
+            lutaml_expressir_wrapper = load_express_repositories(
+              path: path,
+              cache_path: cache,
+              document: document
+            )
+
+            if lutaml_expressir_wrapper
+              express_indexes[name] = {
+                wrapper: lutaml_expressir_wrapper,
+                serialized_hash: nil
+              }
             end
           end
+
           express_indexes
         rescue StopIteration
           express_indexes

data/lib/metanorma/plugin/lutaml/version.rb

@@ -1,7 +1,7 @@
 module Metanorma
   module Plugin
     module Lutaml
-      VERSION = "0.6.0".freeze
+      VERSION = "0.6.3".freeze
     end
   end
 end

data/metanorma-plugin-lutaml.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "liquid"
   spec.add_dependency "lutaml"
   spec.add_dependency "relaton-cli"
-  spec.add_dependency "reverse_adoc"
+  spec.add_dependency "reverse_adoc", "~> 0.3.7"
 
   spec.add_development_dependency "debug"
   spec.add_development_dependency "equivalent-xml"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-plugin-lutaml
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.3
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-23 00:00:00.000000000 Z
+date: 2024-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -84,16 +84,16 @@ dependencies:
   name: reverse_adoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.7
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement