metanorma-plugin-datastruct 0.3.10 → 0.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ddd2e0f19eb24c1c23dd1445ecdf17734b4b70263d1098e9446454c6e5bbddc6
-  data.tar.gz: 346f50de2a3a8a36a1dbc7ab89141226ec5ae5133fc0664bb106c7a01fd7356e
+  metadata.gz: 35c1de600ac66e34295b75d0bfd4fee0328ea75901be90148928d347844e474b
+  data.tar.gz: 4752f9aa448c03e63af8a95f244013ddf5fe84b65b36b29b4a5e72621e1ef44f
 SHA512:
-  metadata.gz: 67bab763402f9e0d651effa3fa6e76228331459dd67e4b2faec2b6bb411490aaf7a5751cbddcd4302a9d8962e9412e2de6c0431ff5e9f0222687c3dc88a9c140
-  data.tar.gz: 6a87d9f07929323519f1cc4ea9402df25918eb5ec464115be321abe695b4f579f10fac1c72822abccb8bd293656b8660acb47386bcbd76685b120966b652aa45
+  metadata.gz: 0302bce3b5cc508e540ce7b897710ddec93de2592939f90fec3578a65f59aa47a326c7e73ac5af3ceea489cade3b48c8deea02950eab2920d3f9995acddf67ec
+  data.tar.gz: f34daa0895c9f5d404e18ef8a094f05c9b14348c2fccb815a5f9eb952587ac3e69b70f13381f0d3427c6c8da38e3bff9c5e912fb87a453b74337467a4b49b621

data/CLAUDE.md

@@ -0,0 +1,60 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+`metanorma-plugin-datastruct` is a Ruby gem that provides Asciidoctor preprocessors for the Metanorma document processing pipeline. It allows Metanorma AsciiDoc documents to load data from YAML/JSON files and interpolate that data into the document using Liquid templates.
+
+## Build & Test Commands
+
+```bash
+bundle install            # install dependencies
+bundle exec rake          # run the full test suite (default rake task = rspec)
+bundle exec rspec         # run all specs
+bundle exec rspec spec/metanorma/plugin/datastruct/macros_yaml2text_spec.rb   # run a single spec file
+bundle exec rspec spec/metanorma/plugin/datastruct/macros_yaml2text_spec.rb:75  # run a single test by line number
+```
+
+## Architecture
+
+### Core preprocessors
+
+Both are Asciidoctor `Preprocessor` extensions that intercept `[yaml2text,...]` and `[json2text,...]` block macros:
+
+- **`Yaml2TextPreprocessor`** (`lib/metanorma/plugin/datastruct/yaml2_text_preprocessor.rb`) — parses YAML data files via `YAML.safe_load`
+- **`Json2TextPreprocessor`** (`lib/metanorma/plugin/datastruct/json2_text_preprocessor.rb`) — parses JSON data files via `JSON.parse`
+
+Both inherit from **`BaseStructuredTextPreprocessor`** (`base_structured_text_preprocessor.rb`), which handles:
+- Scanning lines for `[yaml2text|json2text,<file>,<context>]` block headers
+- Collecting block content and handling nested blocks (via `with_yaml_nested_context` / `with_json_nested_context`)
+- Transforming AsciiDoc-style `{variable}` interpolation into Liquid `{{variable}}` syntax
+- Rendering the combined template through the Liquid engine
+
+### Liquid extensions
+
+Custom Liquid tags and filters registered at load time in `base_structured_text_preprocessor.rb`:
+
+- **`KeyIterator`** (`lib/liquid/custom_blocks/key_iterator.rb`) — `{keyiterator}` tag, iterates over Hash keys or arrays with an index variable
+- **`WithYamlNestedContext`** / **`WithJsonNestedContext`** — `{with_yaml_nested_context}` / `{with_json_nested_context}` tags, load an additional data file into the Liquid context mid-template (enables nested file references)
+- **`CustomFilters.values`** (`lib/liquid/custom_filters/values.rb`) — `values` filter, exposes `Hash#values` in Liquid templates
+
+### Template syntax transformation
+
+`BaseStructuredTextPreprocessor.transform_line_liquid` converts custom AsciiDoc-like syntax to Liquid before rendering:
+- `{context.*,item,EOF}` → `{% keyiterator context, item %}` ... `{EOF}` → `{% endkeyiterator %}`
+- Single-brace `{var}` → double-brace `{{var}}`
+- `{var.#}` → `index` (loop index)
+- `{var + N}` / `{var - N}` → Liquid `plus`/`minus` filters
+- `{var.values[X]}` → uses the custom `values` filter
+
+### Test setup
+
+Tests use `metanorma-standoc` to process full AsciiDoc documents through the Metanorma pipeline, then compare the resulting XML output. The spec helper registers datastruct's preprocessors **before** standoc's extension group to avoid lutaml intercepting the same block names. Shared examples in `spec/support/` run the same test matrix for both YAML and JSON.
+
+## Key Dependencies
+
+- `asciidoctor` (~> 2.0.0) — document processing framework
+- `liquid` (>= 4) — template engine
+- `relaton-cli`, `isodoc` — Metanorma ecosystem dependencies
+- Dev dependencies: `metanorma`, `metanorma-standoc` for integration testing

data/lib/liquid/custom_filters/values.rb

@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 module Liquid
   module CustomFilters
     def values(list)
+      return list unless list.respond_to?(:values)
       list.values
     end
   end

data/lib/metanorma/plugin/datastruct/base_structured_text_preprocessor.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "set"
 require "liquid"
 require "asciidoctor"
 require "asciidoctor/reader"
@@ -19,6 +20,11 @@ module Metanorma
 
         BLOCK_START_REGEXP = /\{(.+?)\.\*,(.+),(.+)\}/.freeze
         BLOCK_END_REGEXP = /\A\{[A-Z]+\}\z/.freeze
+        NESTED_CONTEXT_SUFFIX = {
+          "yaml2text" => "yaml",
+          "json2text" => "json",
+          "data2text" => "yaml",
+        }.freeze
 
         def initialize(config = {})
           super
@@ -72,7 +78,7 @@ module Metanorma
 
         def collect_internal_block_lines(document, input_lines, end_mark)
           current_block = []
-          nested_marks = []
+          nested_marks = Set.new
           while (block_line = input_lines.next) != end_mark
             if nested_match = block_line
                 .match(/^\[#{config[:block_name]},(.+?),(.+?)\]/)
@@ -80,11 +86,11 @@ module Metanorma
                 .push(*nested_context_tag(document,
                                           nested_match[1],
                                           nested_match[2]).split("\n"))
-              next nested_marks.push(input_lines.next)
+              next nested_marks.add(input_lines.next)
             end
 
             if nested_marks.include?(block_line)
-              current_block.push("{% endwith_#{data_file_type}_nested_context %}")
+              current_block.push("{% endwith_#{nested_context_suffix}_nested_context %}")
               next nested_marks.delete(block_line)
             end
             current_block.push(block_line)
@@ -92,8 +98,8 @@ module Metanorma
           current_block
         end
 
-        def data_file_type
-          config[:block_name].split("2").first
+        def nested_context_suffix
+          NESTED_CONTEXT_SUFFIX[config[:block_name]]
         end
 
         def nested_context_tag(document, file_path, context_name)
@@ -102,7 +108,7 @@ module Metanorma
             {% capture nested_file_path %}
             #{absolute_file_path}
             {% endcapture %}
-            {% with_#{data_file_type}_nested_context nested_file_path, #{context_name}  %}
+            {% with_#{nested_context_suffix}_nested_context nested_file_path, #{context_name}  %}
           TEMPLATE
         end
 
@@ -122,13 +128,8 @@ module Metanorma
         end
 
         def transform_line_liquid(line)
-          if line.match?(BLOCK_START_REGEXP)
-            line.gsub!(BLOCK_START_REGEXP, '{% keyiterator \1, \2 %}')
-          end
-
-          if line.strip.match?(BLOCK_END_REGEXP)
-            line.gsub!(BLOCK_END_REGEXP, "{% endkeyiterator %}")
-          end
+          line = line.gsub(BLOCK_START_REGEXP, '{% keyiterator \1, \2 %}') if line.match?(BLOCK_START_REGEXP)
+          line = line.gsub(BLOCK_END_REGEXP, "{% endkeyiterator %}") if line.strip.match?(BLOCK_END_REGEXP)
           line
             .gsub(/(?<!{){(?!%)([^{}]+)(?<!%)}(?!})/, '{{\1}}')
             .gsub(/[a-z.]+\#/, "index")

data/lib/metanorma/plugin/datastruct/content.rb

@@ -33,28 +33,26 @@ module Metanorma
           JSON.parse(File.read(resolved_file_path, encoding: "UTF-8"))
         end
 
-        def content_from_file(document, file_path)
+        def load_file_content(document, file_path)
           resolved_file_path = relative_file_path(document, file_path)
-          load_content_from_file(resolved_file_path)
+          load_content_by_extension(resolved_file_path)
         end
 
-        def load_content_from_file(resolved_file_path)
+        private
+
+        def load_content_by_extension(resolved_file_path)
           unless File.exist?(resolved_file_path)
             ::Metanorma::Util
               .log("Failed to load content from file: #{resolved_file_path}",
                    :error)
           end
 
-          if json_file?(resolved_file_path)
+          if resolved_file_path.end_with?(".json")
             json_content_from_file(resolved_file_path)
           else
             yaml_content_from_file(resolved_file_path)
           end
         end
-
-        def json_file?(file_path)
-          file_path.end_with?(".json")
-        end
       end
     end
   end

data/lib/metanorma/plugin/datastruct/data2_text_preprocessor.rb

@@ -8,37 +8,17 @@ module Metanorma
     module Datastruct
       class Data2TextPreprocessor < BaseStructuredTextPreprocessor
         include Content
-        # search document for block `data2text`
-        #   after that take template from block and read file into this template
-        #   example:
-        #     [data2text,my_yaml=foobar.yaml,my_json=foobar.json]
-        #     ----
-        #     === {foobar.name}
-        #     {foobar.desc}
-        #
-        #     {my_json.symbol}:: {my_json.symbol_def}
-        #     ----
-        #
-        #   with content of `foobar.yaml` file equal to:
-        #     - name: spaghetti
-        #       desc: wheat noodles of 9mm diameter
-        #
-        #   and content of `foobar.json` file equal to:
-        #     {
-        #       "symbol": "SPAG",
-        #       "symbol_def": "the situation is message like spaghetti",
-        #     }
-        #
-        #   will produce:
-        #     === spaghetti
-        #     wheat noodles of 9mm diameter
-        #
-        #     SPAG:: the situation is message like spaghetti
 
         def initialize(config = {})
           super
           @config[:block_name] = "data2text"
         end
+
+        protected
+
+        def content_from_file(document, file_path)
+          load_file_content(document, file_path)
+        end
       end
     end
   end

data/lib/metanorma/plugin/datastruct/json2_text_preprocessor.rb

@@ -8,35 +8,18 @@ module Metanorma
     module Datastruct
       class Json2TextPreprocessor < BaseStructuredTextPreprocessor
         include Content
-        # search document for block `json2text`
-        #   after that take template from block and read file into this template
-        #   example:
-        #     [json2text,foobar.json]
-        #     ----
-        #     === {item.name}
-        #     {item.desc}
-        #
-        #     {item.symbol}:: {item.symbol_def}
-        #     ----
-        #
-        #   with content of `foobar.json` file equal to:
-        #     {
-        #       "name": "spaghetti",
-        #       "desc": "wheat noodles of 9mm diameter".
-        #       "symbol": "SPAG",
-        #       "symbol_def": "the situation is message like spaghetti",
-        #     }
-        #
-        #   will produce:
-        #     === spaghetti
-        #     wheat noodles of 9mm diameter
-        #
-        #     SPAG:: the situation is message like spaghetti
 
         def initialize(config = {})
           super
           @config[:block_name] = "json2text"
         end
+
+        protected
+
+        def content_from_file(document, file_path)
+          resolved = relative_file_path(document, file_path)
+          json_content_from_file(resolved)
+        end
       end
     end
   end

data/lib/metanorma/plugin/datastruct/version.rb

@@ -1,7 +1,7 @@
 module Metanorma
   module Plugin
     module Datastruct
-      VERSION = "0.3.10".freeze
+      VERSION = "0.3.11".freeze
     end
   end
 end

data/metanorma.asciidoc.log.txt

@@ -0,0 +1 @@
+Document title

data/test.asciidoc.log.txt

@@ -0,0 +1,20 @@
+= Document title
+Author
+:docfile: test.adoc
+:nodoc:
+:novalid:
+:no-isobib:
+:imagesdir: spec/assets
+
+=== Nicaragua
+
+Amateur stations:: O[F-J][:digit:][0-9A-Z]{3}[:upper:]{1}
+
+
+
+
+=== Niger
+
+Amateur stations:: O[F-J][:upper:]{5,10}
+
+Experimental:: {"regex"=>"O[F-J][:upper:]{5,10}"}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-plugin-datastruct
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.3.11
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-11 00:00:00.000000000 Z
+date: 2026-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -79,6 +79,7 @@ files:
 - ".hound.yml"
 - ".rspec"
 - ".rubocop.yml"
+- CLAUDE.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.devel
@@ -102,6 +103,8 @@ files:
 - lib/metanorma/plugin/datastruct/version.rb
 - lib/metanorma/plugin/datastruct/yaml2_text_preprocessor.rb
 - metanorma-plugin-datastruct.gemspec
+- metanorma.asciidoc.log.txt
+- test.asciidoc.log.txt
 homepage: https://github.com/metanorma/metanorma-plugin-datastruct
 licenses:
 - BSD-2-Clause