metanorma-plugin-datastruct 0.3.9 → 0.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b1a1aff3342ca19d9bad692c321719cc381a1d74d98b2827c23bb1da7f6d2de
-  data.tar.gz: 3c5f60aba996ac9e1f8924439e60d453c847da10189025917334fbcb200d1d85
+  metadata.gz: 35c1de600ac66e34295b75d0bfd4fee0328ea75901be90148928d347844e474b
+  data.tar.gz: 4752f9aa448c03e63af8a95f244013ddf5fe84b65b36b29b4a5e72621e1ef44f
 SHA512:
-  metadata.gz: 4e69b0df873ef7d4bb9db7fa7d998c8eb5ee943d14d153e472368087ee55afbcd62db101f3791f530f5cc7f7619eff5514ef92a66a8ea8b2ea5a7ef58e94a66e
-  data.tar.gz: f3746f6e9d3e917dd2ac0e861eb837d8fc6d665699737e41244c427b54112863dd29e5134b23ea3f3b2d7b228f93840ab29935c5da08c471638e7a75e1268ef3
+  metadata.gz: 0302bce3b5cc508e540ce7b897710ddec93de2592939f90fec3578a65f59aa47a326c7e73ac5af3ceea489cade3b48c8deea02950eab2920d3f9995acddf67ec
+  data.tar.gz: f34daa0895c9f5d404e18ef8a094f05c9b14348c2fccb815a5f9eb952587ac3e69b70f13381f0d3427c6c8da38e3bff9c5e912fb87a453b74337467a4b49b621

data/.github/workflows/rake.yml

@@ -2,6 +2,9 @@
 # See https://github.com/metanorma/cimas
 name: rake
 
+permissions:
+  contents: read
+
 on:
   push:
     branches: [ master, main ]

data/.github/workflows/release.yml

@@ -2,6 +2,11 @@
 # See https://github.com/metanorma/cimas
 name: release
 
+permissions:
+  contents: write
+  packages: write
+  id-token: write
+
 on:
   workflow_dispatch:
     inputs:
@@ -22,4 +27,3 @@ jobs:
     secrets:
       rubygems-api-key: ${{ secrets.METANORMA_CI_RUBYGEMS_API_KEY }}
       pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
-

data/CLAUDE.md

@@ -0,0 +1,60 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+`metanorma-plugin-datastruct` is a Ruby gem that provides Asciidoctor preprocessors for the Metanorma document processing pipeline. It allows Metanorma AsciiDoc documents to load data from YAML/JSON files and interpolate that data into the document using Liquid templates.
+
+## Build & Test Commands
+
+```bash
+bundle install            # install dependencies
+bundle exec rake          # run the full test suite (default rake task = rspec)
+bundle exec rspec         # run all specs
+bundle exec rspec spec/metanorma/plugin/datastruct/macros_yaml2text_spec.rb   # run a single spec file
+bundle exec rspec spec/metanorma/plugin/datastruct/macros_yaml2text_spec.rb:75  # run a single test by line number
+```
+
+## Architecture
+
+### Core preprocessors
+
+Both are Asciidoctor `Preprocessor` extensions that intercept `[yaml2text,...]` and `[json2text,...]` block macros:
+
+- **`Yaml2TextPreprocessor`** (`lib/metanorma/plugin/datastruct/yaml2_text_preprocessor.rb`) — parses YAML data files via `YAML.safe_load`
+- **`Json2TextPreprocessor`** (`lib/metanorma/plugin/datastruct/json2_text_preprocessor.rb`) — parses JSON data files via `JSON.parse`
+
+Both inherit from **`BaseStructuredTextPreprocessor`** (`base_structured_text_preprocessor.rb`), which handles:
+- Scanning lines for `[yaml2text|json2text,<file>,<context>]` block headers
+- Collecting block content and handling nested blocks (via `with_yaml_nested_context` / `with_json_nested_context`)
+- Transforming AsciiDoc-style `{variable}` interpolation into Liquid `{{variable}}` syntax
+- Rendering the combined template through the Liquid engine
+
+### Liquid extensions
+
+Custom Liquid tags and filters registered at load time in `base_structured_text_preprocessor.rb`:
+
+- **`KeyIterator`** (`lib/liquid/custom_blocks/key_iterator.rb`) — `{keyiterator}` tag, iterates over Hash keys or arrays with an index variable
+- **`WithYamlNestedContext`** / **`WithJsonNestedContext`** — `{with_yaml_nested_context}` / `{with_json_nested_context}` tags, load an additional data file into the Liquid context mid-template (enables nested file references)
+- **`CustomFilters.values`** (`lib/liquid/custom_filters/values.rb`) — `values` filter, exposes `Hash#values` in Liquid templates
+
+### Template syntax transformation
+
+`BaseStructuredTextPreprocessor.transform_line_liquid` converts custom AsciiDoc-like syntax to Liquid before rendering:
+- `{context.*,item,EOF}` → `{% keyiterator context, item %}` ... `{EOF}` → `{% endkeyiterator %}`
+- Single-brace `{var}` → double-brace `{{var}}`
+- `{var.#}` → `index` (loop index)
+- `{var + N}` / `{var - N}` → Liquid `plus`/`minus` filters
+- `{var.values[X]}` → uses the custom `values` filter
+
+### Test setup
+
+Tests use `metanorma-standoc` to process full AsciiDoc documents through the Metanorma pipeline, then compare the resulting XML output. The spec helper registers datastruct's preprocessors **before** standoc's extension group to avoid lutaml intercepting the same block names. Shared examples in `spec/support/` run the same test matrix for both YAML and JSON.
+
+## Key Dependencies
+
+- `asciidoctor` (~> 2.0.0) — document processing framework
+- `liquid` (>= 4) — template engine
+- `relaton-cli`, `isodoc` — Metanorma ecosystem dependencies
+- Dev dependencies: `metanorma`, `metanorma-standoc` for integration testing

data/Gemfile

@@ -1,13 +1,20 @@
-Encoding.default_external = Encoding::UTF_8
-Encoding.default_internal = Encoding::UTF_8
-
 source "https://rubygems.org"
-git_source(:github) { |repo| "https://github.com/#{repo}" }
 
+# Specify your gem's dependencies in metanorma-plugin-datastruct.gemspec
 gemspec
 
-begin
-  eval_gemfile("Gemfile.devel")
-rescue StandardError
-  nil
-end
+eval_gemfile("Gemfile.devel") if File.exist?("Gemfile.devel")
+
+gem "canon"
+gem "metanorma", github: "metanorma/metanorma", branch: "main"
+gem "metanorma-standoc", github: "metanorma/metanorma-standoc", branch: "main"
+gem "rake", "~> 13"
+gem "rspec"
+gem "rubocop"
+gem "rubocop-performance"
+gem "rubocop-rake"
+gem "rubocop-rspec"
+gem "simplecov"
+gem "timecop"
+gem "vcr"
+gem "webmock"

data/Gemfile.devel

@@ -0,0 +1,2 @@
+gem "metanorma-plugin-lutaml", git: "https://github.com/metanorma/metanorma-plugin-lutaml", branch: "main"
+gem "ogc-gml", "~> 1.1.0"

data/lib/liquid/custom_blocks/key_iterator.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "liquid"
+
+module Liquid
+  module CustomBlocks
+    class KeyIterator < Block
+      def initialize(tag_name, markup, tokens)
+        super
+        @context_name, @var_name = markup.split(",").map(&:strip)
+      end
+
+      def render(context)
+        collection = context[@context_name]
+        items = enumerable_items(collection)
+        result = +""
+        items.each.with_index do |item, index|
+          context["index"] = index
+          context[@var_name] = item
+          result << super
+        end
+        result
+      end
+
+      private
+
+      def enumerable_items(collection)
+        case collection
+        when Hash then collection.keys
+        else collection
+        end
+      end
+    end
+  end
+end

data/lib/liquid/custom_blocks/nested_context_block.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "liquid"
+
+module Liquid
+  module CustomBlocks
+    class NestedContextBlock < Block
+      def initialize(tag_name, markup, tokens)
+        super
+        @context_file_variable, @context_name = markup.split(",").map(&:strip)
+      end
+
+      def render(context)
+        context_file = context[@context_file_variable].to_s.strip
+        context[@context_name] = load_content(context_file)
+        super
+      end
+
+      private
+
+      def load_content(_file_path)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/liquid/custom_blocks/with_json_nested_context.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "nested_context_block"
+
+module Liquid
+  module CustomBlocks
+    class WithJsonNestedContext < NestedContextBlock
+      private
+
+      def load_content(file_path)
+        JSON.parse(File.read(file_path, encoding: "utf-8"))
+      end
+    end
+  end
+end

data/lib/liquid/custom_blocks/with_yaml_nested_context.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "yaml"
+require_relative "nested_context_block"
+
+module Liquid
+  module CustomBlocks
+    class WithYamlNestedContext < NestedContextBlock
+      private
+
+      def load_content(file_path)
+        YAML.safe_load(
+          File.read(file_path, encoding: "utf-8"),
+          permitted_classes: [Date, Time],
+          aliases: true,
+        )
+      end
+    end
+  end
+end

data/lib/liquid/custom_filters/values.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Liquid
+  module CustomFilters
+    def values(list)
+      return list unless list.respond_to?(:values)
+      list.values
+    end
+  end
+end

data/lib/metanorma/plugin/datastruct/base_structured_text_preprocessor.rb

@@ -1,56 +1,60 @@
 # frozen_string_literal: true
 
+require "set"
 require "liquid"
 require "asciidoctor"
 require "asciidoctor/reader"
-require_relative "liquid/custom_blocks/key_iterator"
-require_relative "liquid/custom_filters/values"
-require_relative "liquid/custom_filters/replace_regex"
-require_relative "liquid/custom_filters/loadfile"
+require "liquid/custom_blocks/key_iterator"
+require "liquid/custom_blocks/with_yaml_nested_context"
+require "liquid/custom_blocks/with_json_nested_context"
+require "liquid/custom_filters/values"
+require_relative "path_resolver"
 require_relative "source_extractor"
 
-module Asciidoctor
-  class PreprocessorNoIfdefsReader < PreprocessorReader
-    def preprocess_conditional_directive(_keyword, _target, _delimiter, _text)
-      false # decline to resolve idefs
-    end
-  end
-end
-
 module Metanorma
   module Plugin
     module Datastruct
-      # Base class for processing structured data blocks(yaml, json)
       class BaseStructuredTextPreprocessor <
         Asciidoctor::Extensions::Preprocessor
+        include PathResolver
+
         BLOCK_START_REGEXP = /\{(.+?)\.\*,(.+),(.+)\}/.freeze
         BLOCK_END_REGEXP = /\A\{[A-Z]+\}\z/.freeze
-        LOAD_FILE_REGEXP = /{% assign (.*) = (.*) \| load_file %}/.freeze
+        NESTED_CONTEXT_SUFFIX = {
+          "yaml2text" => "yaml",
+          "json2text" => "json",
+          "data2text" => "yaml",
+        }.freeze
+
+        def initialize(config = {})
+          super
+          register_liquid_extensions
+        end
 
         def process(document, reader)
-          r = ::Asciidoctor::PreprocessorNoIfdefsReader
-            .new document, reader.lines
-          input_lines = r.readlines
-          Metanorma::Plugin::Datastruct::SourceExtractor.extract(
-            document,
-            input_lines,
-          )
-          Asciidoctor::PreprocessorNoIfdefsReader
-            .new(document, processed_lines(document, input_lines.to_enum))
+          input_lines = reader.readlines.to_enum
+          Asciidoctor::Reader.new(processed_lines(document, input_lines))
         end
 
         protected
 
         def content_from_file(_document, _file_path)
-          raise ArgumentError, "Implement `content_from_file` in your class"
-        end
-
-        def content_from_anchor(_document, _file_path)
-          raise ArgumentError, "Implement `content_from_anchor` in your class"
+          raise NotImplementedError,
+                "Implement `content_from_file` in your subclass"
         end
 
         private
 
+        def register_liquid_extensions
+          env = Liquid::Environment.default
+          env.register_tag("keyiterator", Liquid::CustomBlocks::KeyIterator)
+          env.register_tag("with_yaml_nested_context",
+                           Liquid::CustomBlocks::WithYamlNestedContext)
+          env.register_tag("with_json_nested_context",
+                           Liquid::CustomBlocks::WithJsonNestedContext)
+          env.register_filter(Liquid::CustomFilters)
+        end
+
         def processed_lines(document, input_lines)
           result = []
           loop do
@@ -59,19 +63,9 @@ module Metanorma
           result
         end
 
-        def relative_file_path(document, file_path)
-          docfile_directory = File.dirname(
-            document.attributes["docfile"] || ".",
-          )
-          document
-            .path_resolver
-            .system_path(file_path, docfile_directory)
-        end
-
         def process_text_blocks(document, input_lines)
           line = input_lines.next
-          block_match = line.match(/^\[#{config[:block_name]},(.+?),(.+?)\]/) ||
-            line.match(/^\[#{config[:block_name]},(.+?)\]/)
+          block_match = line.match(/^\[#{config[:block_name]},(.+?),(.+?)\]/)
           return [line] if block_match.nil?
 
           end_mark = input_lines.next
@@ -82,127 +76,95 @@ module Metanorma
                          block_match)
         end
 
-        def collect_internal_block_lines(document, input_lines, end_mark) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        def collect_internal_block_lines(document, input_lines, end_mark)
           current_block = []
+          nested_marks = Set.new
           while (block_line = input_lines.next) != end_mark
-            if block_line.match?(LOAD_FILE_REGEXP)
-              load_file_match = block_line.match(LOAD_FILE_REGEXP)
-
-              # Add parent folder as argument to loadfile filter
-              block_line = "{% assign #{load_file_match[1]} = "\
-                           "#{load_file_match[2]} | loadfile: " \
-                           "\"#{document.attributes['docdir']}\" %}"
+            if nested_match = block_line
+                .match(/^\[#{config[:block_name]},(.+?),(.+?)\]/)
+              current_block
+                .push(*nested_context_tag(document,
+                                          nested_match[1],
+                                          nested_match[2]).split("\n"))
+              next nested_marks.add(input_lines.next)
             end
 
+            if nested_marks.include?(block_line)
+              current_block.push("{% endwith_#{nested_context_suffix}_nested_context %}")
+              next nested_marks.delete(block_line)
+            end
             current_block.push(block_line)
           end
           current_block
         end
 
-        def data_file_type
-          @config[:block_name].split("2").first
+        def nested_context_suffix
+          NESTED_CONTEXT_SUFFIX[config[:block_name]]
         end
 
-        def parse_template(document, current_block, block_match) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-          transformed_liquid_lines = current_block.map do |x|
-            transform_line_liquid(x)
-          end
-
-          contexts = if block_match[1].include?("=")
-                       content_from_multiple_contexts(
-                         document, block_match
-                       )
-                     elsif block_match[1].start_with?("#")
-                       {
-                         block_match[2].strip =>
-                           content_from_anchor(document, block_match[1][1..-1]),
-                       }
-                     else
-                       {
-                         block_match[2].strip =>
-                           content_from_file(document, block_match[1]),
-                       }
-                     end
+        def nested_context_tag(document, file_path, context_name)
+          absolute_file_path = relative_file_path(document, file_path)
+          <<~TEMPLATE
+            {% capture nested_file_path %}
+            #{absolute_file_path}
+            {% endcapture %}
+            {% with_#{nested_context_suffix}_nested_context nested_file_path, #{context_name}  %}
+          TEMPLATE
+        end
 
+        def parse_template(document, current_block, block_match)
+          transformed_liquid_lines = current_block
+            .map(&method(:transform_line_liquid))
+          context_items = content_from_file(document, block_match[1])
           parse_context_block(document: document,
                               context_lines: transformed_liquid_lines,
-                              contexts: contexts)
+                              context_items: context_items,
+                              context_name: block_match[2].strip)
         rescue StandardError => e
-          ::Metanorma::Util.log("Failed to parse #{config[:block_name]} \
-              block: #{e.message}", :error)
+          document.logger
+            .warn("Failed to parse #{config[:block_name]} \
+              block: #{e.message}")
           []
         end
 
-        def content_from_multiple_contexts(document, block_match) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-          contexts = {}
-          (1..block_match.size - 1).each do |i|
-            context_and_paths = block_match[i].strip
-            context_and_paths.split(",").each do |context_and_path|
-              context_and_path.strip!
-              context_name, path = context_and_path.split("=")
-              context_items = content_from_file(document, path)
-              contexts[context_name] = context_items
-            end
-          end
-
-          contexts
-        end
-
-        def transform_line_liquid(line) # rubocop:disable Metrics/MethodLength
-          if line.match?(BLOCK_START_REGEXP)
-            line.gsub!(BLOCK_START_REGEXP, '{% keyiterator \1, \2 %}')
-          end
-
-          if line.strip.match?(BLOCK_END_REGEXP)
-            line.gsub!(BLOCK_END_REGEXP, "{% endkeyiterator %}")
-          end
+        def transform_line_liquid(line)
+          line = line.gsub(BLOCK_START_REGEXP, '{% keyiterator \1, \2 %}') if line.match?(BLOCK_START_REGEXP)
+          line = line.gsub(BLOCK_END_REGEXP, "{% endkeyiterator %}") if line.strip.match?(BLOCK_END_REGEXP)
           line
             .gsub(/(?<!{){(?!%)([^{}]+)(?<!%)}(?!})/, '{{\1}}')
-            .gsub(/[a-z\.]+\#/, "index")
+            .gsub(/[a-z.]+\#/, "index")
             .gsub(/{{(.+)\s+\+\s+(\d+)\s*?}}/, '{{ \1 | plus: \2 }}')
             .gsub(/{{(.+)\s+-\s+(\d+)\s*?}}/, '{{ \1 | minus: \2 }}')
             .gsub(/{{(.+)\.values(.*?)}}/,
                   '{% assign custom_value = \1 | values %}{{custom_value\2}}')
         end
 
-        def parse_context_block(context_lines:, contexts:, document:)
+        def parse_context_block(context_lines:,
+                                context_items:,
+                                context_name:,
+                                document:)
           render_result, errors = render_liquid_string(
             template_string: context_lines.join("\n"),
-            contexts: contexts,
+            context_items: context_items,
+            context_name: context_name,
             document: document,
           )
           notify_render_errors(document, errors)
           render_result.split("\n")
         end
 
-        def render_liquid_string(template_string:, contexts:, document:) # rubocop:disable Metrics/MethodLength
-          liquid_template = ::Liquid::Template
-            .parse(template_string, environment: create_liquid_environment)
-
-          # Allow includes for the template
+        def render_liquid_string(template_string:, context_items:,
+                                 context_name:, document:)
+          liquid_template = Liquid::Template.parse(template_string)
           liquid_template.registers[:file_system] =
             ::Liquid::LocalFileSystem.new(relative_file_path(document, ""))
-
-          rendered_string = liquid_template.render(
-            contexts,
-            strict_variables: false,
-            error_mode: :warn,
-          )
+          rendered_string = liquid_template
+            .render(context_name => context_items,
+                    strict_variables: true,
+                    error_mode: :warn)
           [rendered_string, liquid_template.errors]
         end
 
-        def create_liquid_environment
-          ::Liquid::Environment.new.tap do |liquid_env|
-            liquid_env.register_tag(
-              "keyiterator",
-              ::Metanorma::Plugin::Datastruct::Liquid::CustomBlocks::KeyIterator, # rubocop:disable Layout/LineLength
-            )
-            liquid_env.register_filter(
-              ::Metanorma::Plugin::Datastruct::Liquid::CustomFilters,
-            )
-          end
-        end
-
         def notify_render_errors(document, errors)
           errors.each do |error_obj|
             document

data/lib/metanorma/plugin/datastruct/content.rb

@@ -2,19 +2,21 @@
 
 require "json"
 require "yaml"
+require_relative "path_resolver"
 
 module Metanorma
   module Plugin
     module Datastruct
       module Content
+        include PathResolver
+
         protected
 
-        # https://ruby-doc.org/stdlib-2.5.1/libdoc/psych/rdoc/Psych.html#method-c-safe_load
-        def yaml_content_from_file(resolved_file_path) # rubocop:disable Metrics/MethodLength
+        def yaml_content_from_file(resolved_file_path)
           unless File.exist?(resolved_file_path)
             ::Metanorma::Util.log(
               "YAML file referenced in [yaml2text] block not found: " \
-              "#{resolved_file_path}", :error
+              "#{resolved_file_path}", :error,
             )
             return
           end
@@ -27,62 +29,30 @@ module Metanorma
           )
         end
 
-        def yaml_content_from_anchor(document, anchor)
-          YAML.safe_load(
-            document.attributes["source_blocks"][anchor],
-            permitted_classes: [Date, Time, Symbol],
-            permitted_symbols: [],
-            aliases: true,
-          )
-        end
-
         def json_content_from_file(resolved_file_path)
           JSON.parse(File.read(resolved_file_path, encoding: "UTF-8"))
         end
 
-        def json_content_from_anchor(document, anchor)
-          JSON.parse(document.attributes["source_blocks"][anchor])
-        end
-
-        def content_from_file(document, file_path)
+        def load_file_content(document, file_path)
           resolved_file_path = relative_file_path(document, file_path)
-          load_content_from_file(resolved_file_path)
+          load_content_by_extension(resolved_file_path)
         end
 
-        def load_content_from_file(resolved_file_path)
+        private
+
+        def load_content_by_extension(resolved_file_path)
           unless File.exist?(resolved_file_path)
             ::Metanorma::Util
               .log("Failed to load content from file: #{resolved_file_path}",
                    :error)
           end
 
-          if json_file?(resolved_file_path)
+          if resolved_file_path.end_with?(".json")
             json_content_from_file(resolved_file_path)
           else
             yaml_content_from_file(resolved_file_path)
           end
         end
-
-        def content_from_anchor(document, anchor)
-          source_block = document.attributes["source_blocks"][anchor]
-          if json_content?(source_block)
-            json_content_from_anchor(document, anchor)
-          else
-            yaml_content_from_anchor(document, anchor)
-          end
-        end
-
-        def json_or_yaml_filepath?(file_path)
-          file_path.end_with?(".json", ".yaml", ".yml")
-        end
-
-        def json_file?(file_path)
-          file_path.end_with?(".json")
-        end
-
-        def json_content?(content)
-          content.start_with?("{", "[")
-        end
       end
     end
   end

data/lib/metanorma/plugin/datastruct/data2_text_preprocessor.rb

@@ -8,37 +8,17 @@ module Metanorma
     module Datastruct
       class Data2TextPreprocessor < BaseStructuredTextPreprocessor
         include Content
-        # search document for block `data2text`
-        #   after that take template from block and read file into this template
-        #   example:
-        #     [data2text,my_yaml=foobar.yaml,my_json=foobar.json]
-        #     ----
-        #     === {foobar.name}
-        #     {foobar.desc}
-        #
-        #     {my_json.symbol}:: {my_json.symbol_def}
-        #     ----
-        #
-        #   with content of `foobar.yaml` file equal to:
-        #     - name: spaghetti
-        #       desc: wheat noodles of 9mm diameter
-        #
-        #   and content of `foobar.json` file equal to:
-        #     {
-        #       "symbol": "SPAG",
-        #       "symbol_def": "the situation is message like spaghetti",
-        #     }
-        #
-        #   will produce:
-        #     === spaghetti
-        #     wheat noodles of 9mm diameter
-        #
-        #     SPAG:: the situation is message like spaghetti
 
         def initialize(config = {})
           super
           @config[:block_name] = "data2text"
         end
+
+        protected
+
+        def content_from_file(document, file_path)
+          load_file_content(document, file_path)
+        end
       end
     end
   end

data/lib/metanorma/plugin/datastruct/json2_text_preprocessor.rb

@@ -8,35 +8,18 @@ module Metanorma
     module Datastruct
       class Json2TextPreprocessor < BaseStructuredTextPreprocessor
         include Content
-        # search document for block `json2text`
-        #   after that take template from block and read file into this template
-        #   example:
-        #     [json2text,foobar.json]
-        #     ----
-        #     === {item.name}
-        #     {item.desc}
-        #
-        #     {item.symbol}:: {item.symbol_def}
-        #     ----
-        #
-        #   with content of `foobar.json` file equal to:
-        #     {
-        #       "name": "spaghetti",
-        #       "desc": "wheat noodles of 9mm diameter".
-        #       "symbol": "SPAG",
-        #       "symbol_def": "the situation is message like spaghetti",
-        #     }
-        #
-        #   will produce:
-        #     === spaghetti
-        #     wheat noodles of 9mm diameter
-        #
-        #     SPAG:: the situation is message like spaghetti
 
         def initialize(config = {})
           super
           @config[:block_name] = "json2text"
         end
+
+        protected
+
+        def content_from_file(document, file_path)
+          resolved = relative_file_path(document, file_path)
+          json_content_from_file(resolved)
+        end
       end
     end
   end

data/lib/metanorma/plugin/datastruct/path_resolver.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Plugin
+    module Datastruct
+      module PathResolver
+        def relative_file_path(document, file_path)
+          docfile_directory = File.dirname(
+            document.attributes["docfile"] || ".",
+          )
+          document
+            .path_resolver
+            .system_path(file_path, docfile_directory)
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/plugin/datastruct/source_extractor.rb

@@ -1,20 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "path_resolver"
+
 module Metanorma
   module Plugin
     module Datastruct
       class SourceExtractor
-        # example:
-        #   - [[abc]]
-        ANCHOR_REGEX_1 = /^\[\[(?<id>[^\]]*)\]\]\s*$/.freeze
-
-        # examples:
-        #   - [#abc]
-        #   - [source#abc,ruby]
-        ANCHOR_REGEX_2 = /^\[[^#,]*#(?<id>[^,\]]*)[,\]]/.freeze
+        include PathResolver
 
-        # examples:
-        #   - [id=abc]
-        #   - [source,id="abc"]
-        ANCHOR_REGEX_3 = /^\[(?:.+,)?id=['"]?(?<id>[^,\]'"]*)['"]?[,\]]/.freeze
+        ANCHOR_PATTERNS = [
+          /^\[\[(?<id>[^\]]*)\]\]\s*$/,
+          /^\[[^#,]*#(?<id>[^,\]]*)[,\]]/,
+          /^\[(?:.+,)?id=['"]?(?<id>[^,\]'"]*)['"]?[,\]]/,
+        ].freeze
 
         def initialize(document, input_lines)
           @document = document
@@ -27,17 +25,17 @@ module Metanorma
           new(document, input_lines).extract
         end
 
-        def extract # rubocop:disable Metrics/AbcSize
+        def extract
           lines = @input_lines.to_enum
 
           loop do
             line = lines.next
 
             if /^embed::|^include::/.match?(line.strip)
-              file_lines = read(filename(@document, line)) or next
+              file_lines = read(filename(line)) or next
               SourceExtractor.extract(@document, file_lines)
             elsif m = match_anchor(line)
-              @document.attributes["source_blocks"][m[:id]] = read_section lines
+              @document.attributes["source_blocks"][m[:id]] = read_section(lines)
             end
           end
         end
@@ -45,42 +43,29 @@ module Metanorma
         private
 
         def match_anchor(line)
-          line.match(ANCHOR_REGEX_1) ||
-            line.match(ANCHOR_REGEX_2) ||
-            line.match(ANCHOR_REGEX_3)
-        end
-
-        def readlines_safe(file)
-          return [] if file.eof?
-
-          file.readlines
+          ANCHOR_PATTERNS.each do |pattern|
+            match = line.match(pattern)
+            return match if match
+          end
+          nil
         end
 
         def read(inc_path)
-          inc_path or return nil
-          ::File.open inc_path, "r" do |fd|
-            readlines_safe(fd).map(&:chomp)
+          return nil unless inc_path
+
+          File.open(inc_path, "r") do |fd|
+            fd.eof? ? [] : fd.readlines.map(&:chomp)
           end
         end
 
-        def filename(document, line)
+        def filename(line)
           m = /(^include::|^embed::)([^\[]+)\[/.match(line)
           return nil unless m
 
-          file_path = relative_file_path(document, m[2])
-
+          file_path = relative_file_path(@document, m[2])
           File.exist?(file_path) ? file_path : nil
         end
 
-        def relative_file_path(document, file_path)
-          docfile_directory = File.dirname(
-            document.attributes["docfile"] || ".",
-          )
-          document
-            .path_resolver
-            .system_path(file_path, docfile_directory)
-        end
-
         def read_section(lines)
           m = lines.next.match(/^--+/)
           return "" unless m

data/lib/metanorma/plugin/datastruct/version.rb

@@ -1,7 +1,7 @@
 module Metanorma
   module Plugin
     module Datastruct
-      VERSION = "0.3.9".freeze
+      VERSION = "0.3.11".freeze
     end
   end
 end

data/lib/metanorma/plugin/datastruct/yaml2_text_preprocessor.rb

@@ -8,33 +8,18 @@ module Metanorma
     module Datastruct
       class Yaml2TextPreprocessor < BaseStructuredTextPreprocessor
         include Content
-        # search document for block `yaml2text`
-        #   after that take template from block and read file into this template
-        #   example:
-        #     [yaml2text,foobar.yaml]
-        #     ----
-        #     === {item.name}
-        #     {item.desc}
-        #
-        #     {item.symbol}:: {item.symbol_def}
-        #     ----
-        #
-        #   with content of `foobar.yaml` file equal to:
-        #     - name: spaghetti
-        #       desc: wheat noodles of 9mm diameter
-        #       symbol: SPAG
-        #       symbol_def: the situation is message like spaghetti at a kid's
-        #
-        #   will produce:
-        #     === spaghetti
-        #     wheat noodles of 9mm diameter
-        #
-        #     SPAG:: the situation is message like spaghetti at a kid's meal
 
         def initialize(config = {})
           super
           @config[:block_name] = "yaml2text"
         end
+
+        protected
+
+        def content_from_file(document, file_path)
+          resolved = relative_file_path(document, file_path)
+          yaml_content_from_file(resolved)
+        end
       end
     end
   end

data/metanorma-plugin-datastruct.gemspec

@@ -15,8 +15,7 @@ Gem::Specification.new do |spec|
   spec.license       = "BSD-2-Clause"
 
   # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added
-  # into git.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject do |f|
       f.match(%r{^(test|spec|features)/})
@@ -26,24 +25,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = ">= 2.7.0" # rubocop:disable Gemspec/RequiredRubyVersion
-
   spec.add_dependency "asciidoctor", "~> 2.0.0"
   spec.add_dependency "isodoc"
+  spec.add_dependency "liquid", ">= 4"
   spec.add_dependency "relaton-cli"
-
-  spec.add_development_dependency "byebug"
-  spec.add_development_dependency "equivalent-xml"
-  spec.add_development_dependency "metanorma"
-  spec.add_development_dependency "metanorma-standoc"
-  spec.add_development_dependency "rake", "~> 13"
-  spec.add_development_dependency "rspec", "~> 3.6"
-  spec.add_development_dependency "rubocop", "~> 1.58"
-  spec.add_development_dependency "rubocop-performance", "~> 1.19"
-  spec.add_development_dependency "simplecov", "~> 0.15"
-  spec.add_development_dependency "timecop", "~> 0.9"
-  spec.add_development_dependency "vcr", "~> 6.1.0"
-  spec.add_development_dependency "webmock"
-  spec.add_development_dependency "xml-c14n"
-  spec.metadata["rubygems_mfa_required"] = "false"
+  spec.metadata["rubygems_mfa_required"] = "true"
 end

data/metanorma.asciidoc.log.txt

@@ -0,0 +1 @@
+Document title

data/test.asciidoc.log.txt

@@ -0,0 +1,20 @@
+= Document title
+Author
+:docfile: test.adoc
+:nodoc:
+:novalid:
+:no-isobib:
+:imagesdir: spec/assets
+
+=== Nicaragua
+
+Amateur stations:: O[F-J][:digit:][0-9A-Z]{3}[:upper:]{1}
+
+
+
+
+=== Niger
+
+Amateur stations:: O[F-J][:upper:]{5,10}
+
+Experimental:: {"regex"=>"O[F-J][:upper:]{5,10}"}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-plugin-datastruct
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.3.11
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-05-14 00:00:00.000000000 Z
+date: 2026-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -39,195 +39,27 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: relaton-cli
+  name: liquid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: equivalent-xml
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: metanorma
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: metanorma-standoc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.6'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.6'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.58'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.58'
-- !ruby/object:Gem::Dependency
-  name: rubocop-performance
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.19'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.19'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.15'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.15'
-- !ruby/object:Gem::Dependency
-  name: timecop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.9'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.9'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 6.1.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 6.1.0
-- !ruby/object:Gem::Dependency
-  name: webmock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '4'
 - !ruby/object:Gem::Dependency
-  name: xml-c14n
+  name: relaton-cli
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -247,31 +79,37 @@ files:
 - ".hound.yml"
 - ".rspec"
 - ".rubocop.yml"
+- CLAUDE.md
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.devel
 - LICENSE
 - README.adoc
 - Rakefile
 - bin/console
 - bin/setup
+- lib/liquid/custom_blocks/key_iterator.rb
+- lib/liquid/custom_blocks/nested_context_block.rb
+- lib/liquid/custom_blocks/with_json_nested_context.rb
+- lib/liquid/custom_blocks/with_yaml_nested_context.rb
+- lib/liquid/custom_filters/values.rb
 - lib/metanorma-plugin-datastruct.rb
 - lib/metanorma/plugin/datastruct/base_structured_text_preprocessor.rb
 - lib/metanorma/plugin/datastruct/content.rb
 - lib/metanorma/plugin/datastruct/data2_text_preprocessor.rb
 - lib/metanorma/plugin/datastruct/json2_text_preprocessor.rb
-- lib/metanorma/plugin/datastruct/liquid/custom_blocks/key_iterator.rb
-- lib/metanorma/plugin/datastruct/liquid/custom_filters/loadfile.rb
-- lib/metanorma/plugin/datastruct/liquid/custom_filters/replace_regex.rb
-- lib/metanorma/plugin/datastruct/liquid/custom_filters/values.rb
+- lib/metanorma/plugin/datastruct/path_resolver.rb
 - lib/metanorma/plugin/datastruct/source_extractor.rb
 - lib/metanorma/plugin/datastruct/version.rb
 - lib/metanorma/plugin/datastruct/yaml2_text_preprocessor.rb
 - metanorma-plugin-datastruct.gemspec
+- metanorma.asciidoc.log.txt
+- test.asciidoc.log.txt
 homepage: https://github.com/metanorma/metanorma-plugin-datastruct
 licenses:
 - BSD-2-Clause
 metadata:
-  rubygems_mfa_required: 'false'
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -280,7 +118,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/metanorma/plugin/datastruct/liquid/custom_blocks/key_iterator.rb

@@ -1,31 +0,0 @@
-module Metanorma
-  module Plugin
-    module Datastruct
-      module Liquid
-        module CustomBlocks
-          class KeyIterator < ::Liquid::Block
-            def initialize(tag_name, markup, tokens)
-              super
-              @context_name, @var_name = markup.split(",").map(&:strip)
-            end
-
-            def render(context) # rubocop:disable Metrics/MethodLength
-              res = ""
-              iterator = if context[@context_name].is_a?(Hash)
-                           context[@context_name].keys
-                         else
-                           context[@context_name]
-                         end
-              iterator.each.with_index do |key, index|
-                context["index"] = index
-                context[@var_name] = key
-                res += super
-              end
-              res
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/metanorma/plugin/datastruct/liquid/custom_filters/loadfile.rb

@@ -1,18 +0,0 @@
-require_relative "../../content"
-
-module Metanorma
-  module Plugin
-    module Datastruct
-      module Liquid
-        module CustomFilters
-          include ::Metanorma::Plugin::Datastruct::Content
-
-          def loadfile(path, parent_folder = ".")
-            resolved_file_path = File.expand_path(path, parent_folder)
-            load_content_from_file(resolved_file_path)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/metanorma/plugin/datastruct/liquid/custom_filters/replace_regex.rb

@@ -1,14 +0,0 @@
-module Metanorma
-  module Plugin
-    module Datastruct
-      module Liquid
-        module CustomFilters
-          def replace_regex(text, regex_search, replace_value)
-            regex = /#{regex_search}/
-            text.to_s.gsub(regex, replace_value)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/metanorma/plugin/datastruct/liquid/custom_filters/values.rb

@@ -1,13 +0,0 @@
-module Metanorma
-  module Plugin
-    module Datastruct
-      module Liquid
-        module CustomFilters
-          def values(list)
-            list.values
-          end
-        end
-      end
-    end
-  end
-end