lutaml-model 0.8.95 → 0.8.96

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5933a3971f4f39e30541f504636641cd059e7a97751ba67a710203dbfdcc45a
-  data.tar.gz: c23b70227e80d99427b9d4125a09e33d9751112bc72a5e6abaca73d6a3897c60
+  metadata.gz: 7b4bd8b10dc92c3a53f615bb3e13c907d1cc4976eb547a5a2da41aa2837bef17
+  data.tar.gz: e1e5eacb6d80dc8108d65a5264d5e9d872d4259103672f6a88e0d912b437002f
 SHA512:
-  metadata.gz: 473b9970246c24172d6040a9c5062fad06d0fea413a5e04956ac98c0990324f2543668436c9259ad2a30b8647555d1e00102f6346afd3018814d68c2841ca8e5
-  data.tar.gz: f48b568bdc715a7649a262ca5757474f56ecc52d775121be99f18f067570d48d5401ff29cd7141ff768e9ea0a3e0aef5996d32129b8907c386cfd3c72e9f5370
+  metadata.gz: efa1a2368c6f2f0352dd928eb33c4c45c9580a431ca3847fe709fcb1026f29deaf2f55e7ec52eafaebdac537ef0c5d55588b22d37d6f6bb27d332165ad9a795a
+  data.tar.gz: 5380981def9666c4ec1fa0f4cb7f977dcba141a5278003a96486d814e7c812d2083339baa3d9d8079c3029d7aef0e84226be40eb188315bd5270c974baed1dda

data/docs/_guides/grammar-backed-formats.adoc

@@ -0,0 +1,69 @@
+= Grammar-backed string formats
+
+A serialization format whose input is a *grammar*, not a data structure:
+identifier strings, query languages, DSLs — text that is parsed, not
+decoded. lutaml-model supports registering these as first-class formats
+through the same `FormatRegistry` extension point as xml/json/yaml; the
+parse side is implemented by a grammar engine (the reference
+implementation is https://github.com/parsanol/parsanol-ruby[Parsanol]'s
+`Parsanol::PARG::Lutaml.register`, which consumes a compiled PARG
+artifact plus its bindings).
+
+== The contract
+
+A grammar-backed format needs three things:
+
+* a *parse* — text → attribute hash (the grammar engine's job)
+* *bindings* — capture names → model attributes (the artifact's
+  bindings section)
+* a *model class* — a regular `Serializable` with attributes; no
+  hand-written `from_h`/`to_h` anywhere
+
+== Reference implementation (Parsanol)
+
+[source,ruby]
+----
+require "parsanol/parg/lutaml"
+
+Parsanol::PARG::Lutaml.register(
+  IsoIdentifier,
+  format_name: :pubid_iso,
+  artifact: "artifacts/iso.json", # compiled grammar + bindings + tests
+  entry: "identifier",
+)
+
+IsoIdentifier.from_pubid_iso("ISO/IEC 12345-1:2020")
+# => #<IsoIdentifier number: "12345-1", year: 2020, ...>
+----
+
+What `register` does:
+
+. loads the checksummed artifact (parse failures raise the engine's
+  error type — register it via `error_types:` so users can rescue it);
+. registers the format through `FormatRegistry.register` with an
+  adapter whose `parse` is artifact-parse-then-bind;
+. defines `from_<format>` on the model class.
+
+The artifact is the parser of record: it carries the grammar, the
+bindings, and the conformance tests (the grammar's own test block is
+baked in and can be re-run by consumers), and its checksum pins the
+pinned version of the language.
+
+== Writing the grammar side
+
+Grammars are authored in PARG (`.parg` — ABNF-flavoured surface, PEG
+semantics, one rule per line) and compiled with the `parsanol` CLI:
+
+----
+parsanol compile grammars/iso.parg --tables tables
+----
+
+The compile-time lint rejects left recursion and shadowed
+alternatives; the embedded test block runs at compile time, so a
+grammar that stops accepting its documented inputs fails the build
+rather than the consumer.
+
+== See also
+
+* parsanol-ruby `docs/pg-language-guide.md` — the PARG language
+* `Parsanol::PARG::Lutaml.register` — the bridge (reference implementation)

data/docs/_guides/index.adoc

@@ -26,6 +26,7 @@ Task-oriented guides for accomplishing specific goals with Lutaml::Model.
 == Advanced mapping
 
 * link:../advanced-mapping[Advanced Attribute Mapping] - Same-name dispatch on attribute values (`when_attribute`), grouped declarations, custom methods, delegation
+* link:../grammar-backed-formats[Grammar-Backed String Formats] - Register a grammar engine as a serialization format (identifier strings, DSLs)
 
 == Performance
 
@@ -47,6 +48,7 @@ See link:../references/rdf-namespaces[RDF Namespaces] for namespace classes,
 * link:../value-transformations[Value Transformations] - Transform values during serialization
 * link:../missing-values-handling[Missing Values Handling] - Handle nil/empty/omitted
 * link:../advanced-mapping[Advanced Mapping] - Custom methods and delegation
+* link:../grammar-backed-formats[Grammar-Backed String Formats] - Grammar engines as serialization formats
 * link:../schema-generation[Schema Generation] - Generate schemas from models
 * link:../creating-xsd[Creating XSD Schemas] - Complete reference for XSD generation from LutaML models
 * link:../schema-import[Schema Import] - Import schemas to create models

data/lib/lutaml/model/serialize/attribute_definition.rb

@@ -8,6 +8,24 @@ module Lutaml
       # Extracted from serialize.rb to improve code organization.
       # Provides methods for defining and validating model attributes.
       module AttributeDefinition
+        # Whether the per-class accessors and the state-defaults seed are
+        # compiled from Ruby source with a String class_eval (the fast
+        # path). Opal has no runtime compiler unless opal-parser is
+        # bundled, so there every such method is defined from a block
+        # with define_method instead.
+        # Core dispatch for the block-form methods, bound per call so an
+        # attribute or method a model defines with one of these names
+        # (send, __send__, instance_variable_get, const_get, ...) is never invoked;
+        # the compiled form reads ivars and calls methods directly.
+        IVAR_GET = ::Kernel.instance_method(:instance_variable_get)
+        IVAR_SET = ::Kernel.instance_method(:instance_variable_set)
+        SEND = ::BasicObject.instance_method(:__send__)
+        CONST_GET = ::Module.instance_method(:const_get)
+
+        def self.source_compilation?
+          !Lutaml::Model.opal?
+        end
+
         # Define attribute methods on the model class
         #
         # @param attr [Attribute] The attribute to define methods for
@@ -44,6 +62,26 @@ module Lutaml
             return
           end
 
+          unless AttributeDefinition.source_compilation?
+            seeds = attrs.map do |name, attr|
+              sentinel = if attr.collection?
+                           Lutaml::Model::Serialize::LAZY_EMPTY_COLLECTION
+                         else
+                           Lutaml::Model::UninitializedClass.instance
+                         end
+              [:"@#{name}", sentinel]
+            end
+            # Raw readers for every attribute the register sees, as the
+            # compiled form below does (#933).
+            attrs.each_key { |name| define_raw_state_reader(name) }
+            define_method(method_name) do
+              # Returns the last seeded value, as the compiled form's final
+              # assignment does.
+              seeds.reduce(nil) { |_, (ivar, sentinel)| IVAR_SET.bind_call(self, ivar, sentinel) }
+            end
+            return
+          end
+
           lines = attrs.map do |name, attr|
             if attr.collection?
               "@#{name} = Lutaml::Model::Serialize::LAZY_EMPTY_COLLECTION"
@@ -82,6 +120,14 @@ module Lutaml
           RUBY
         end
 
+        # Block-form raw-state reader (no String eval), defined on self.
+        def define_raw_state_reader(name)
+          ivar = :"@#{name}"
+          define_method(:"__lutaml_raw_#{name}") do
+            IVAR_GET.bind_call(self, ivar)
+          end
+        end
+
         # Historical getter shape for punctuation-named attributes and
         # any name the compiled form cannot express.
         def define_reflective_attribute_methods(name, attr)
@@ -145,11 +191,16 @@ module Lutaml
           # the state-defaults compilation) so late declarations get it
           # too -- XMI's extension loader declares attributes after
           # first parse.
-          if name.to_s.match?(PLAIN_NAME)
-            # class_eval on self: a bare `model` receiver resolves
-            # through the host class's own namespace (omml's Base
-            # answers a Store model) and defined the reader on the
-            # wrong object (#933).
+          # Without a source compiler, compile_state_defaults! defines
+          # its raw readers from blocks; define one for every name here
+          # too (define_method takes any name, not only plain ones).
+          # Both forms define on self: a bare `model` receiver resolves
+          # through the host class's own namespace (omml's Base answers
+          # a Store model) and defined the reader on the wrong object
+          # (#933).
+          if !AttributeDefinition.source_compilation?
+            define_raw_state_reader(name)
+          elsif name.to_s.match?(PLAIN_NAME)
             class_eval(<<~RUBY, __FILE__, __LINE__ + 1) # rubocop:disable Style/DocumentDynamicEvalDefinition
               def __lutaml_raw_#{name}
                 @#{name}
@@ -263,7 +314,9 @@ module Lutaml
           # Array on EVERY read — the largest per-call allocation source
           # on instance-heavy parses (TODO.max-perf/06). The optional-arg
           # form allocates nothing, and the read is a direct @ivar.
-          if attr.collection?
+          if !AttributeDefinition.source_compilation?
+            define_block_attribute_getter(name, attr)
+          elsif attr.collection?
             # class_eval interpolates, e.g.:
             #   def items(arg = Lutaml::Model::Serialize::NO_ARG)
             #     if arg.equal?(Lutaml::Model::Serialize::NO_ARG)
@@ -319,7 +372,9 @@ module Lutaml
                            end
           define_method(attr_reader_method) { attr } unless reader_defined
 
-          if attr.collection?
+          if !AttributeDefinition.source_compilation?
+            define_block_attribute_setters(name, attr)
+          elsif attr.collection?
             # class_eval interpolates, e.g.:
             #   def items=(value)
             #     value_set_for(:items)
@@ -379,6 +434,77 @@ module Lutaml
           end
         end
 
+        # Block form of the compiled getter in define_regular_attribute_methods,
+        # for runtimes without a source compiler (see source_compilation?).
+        def define_block_attribute_getter(name, attr)
+          name = name.to_sym # the compiled form passes :name literals
+          ivar = :"@#{name}"
+          if attr.collection?
+            # The compiled getter resolves Array in the model class's scope.
+            scope = self
+            define_method(name) do |arg = Lutaml::Model::Serialize::NO_ARG|
+              if arg.equal?(Lutaml::Model::Serialize::NO_ARG)
+                materialize_lazy_collection(name)
+              else
+                current = IVAR_GET.bind_call(self, ivar) || []
+                new_value = current.is_a?(CONST_GET.bind_call(scope, :Array)) ? current + [arg] : arg
+                IVAR_SET.bind_call(self, ivar, new_value)
+                record_mutation(name, arg)
+                arg
+              end
+            end
+          else
+            define_method(name) do |arg = Lutaml::Model::Serialize::NO_ARG|
+              if arg.equal?(Lutaml::Model::Serialize::NO_ARG)
+                IVAR_GET.bind_call(self, ivar)
+              else
+                public_send(:"#{name}=", arg)
+                arg
+              end
+            end
+          end
+        end
+
+        # Block form of the compiled setters in define_regular_attribute_methods.
+        # The Attribute is read through the same hidden accessor the
+        # compiled form calls, not captured: a setter redefined over an
+        # enum shorthand keeps casting with the accessor's Attribute.
+        def define_block_attribute_setters(name, attr)
+          name = name.to_sym # the compiled form passes :name literals
+          ivar = :"@#{name}"
+          handle = :"__attribute_definition_#{name}"
+          if attr.collection?
+            assign = lambda do |model, value|
+              current = IVAR_GET.bind_call(model, ivar)
+              unless current.equal?(Lutaml::Model::Serialize::LAZY_EMPTY_COLLECTION) &&
+                  (value.nil? || Lutaml::Model::Utils.uninitialized?(value))
+                IVAR_SET.bind_call(model, ivar, value)
+              end
+              SEND.bind_call(model, :record_mutation_collection, name, value)
+            end
+            define_method(:"#{name}=") do |value|
+              value_set_for(name)
+              assign.call(self, SEND.bind_call(self, handle).cast_value(value, lutaml_register))
+            end
+            define_method(:"__assign_parsed_#{name}=") do |value|
+              value_set_for(name)
+              assign.call(self, value)
+            end
+          else
+            define_method(:"#{name}=") do |value|
+              value_set_for(name)
+              value = SEND.bind_call(self, handle).cast_value(value, lutaml_register)
+              IVAR_SET.bind_call(self, ivar, value)
+              record_mutation(name, value)
+            end
+            define_method(:"__assign_parsed_#{name}=") do |value|
+              value_set_for(name)
+              IVAR_SET.bind_call(self, ivar, value)
+              record_mutation(name, value)
+            end
+          end
+        end
+
         # Define an attribute for the model
         #
         # @param name [Symbol] The attribute name

data/lib/lutaml/model/serialize/format_conversion.rb

@@ -103,9 +103,14 @@ module Lutaml
 
           register = Lutaml::Model::Config.default_register
           plan = Lutaml::Xml::Leptris::PlanCompiler.compile(self, register)
-          return nil unless plan && !plan[:namespaced] &&
+          return nil unless plan &&
             Lutaml::Xml::Leptris::PlanSerializer.serializable?(plan)
 
+          # Namespaced plans ride the plan serializer too: the DOM
+          # emission spells prefixed names and hoists declarations
+          # (the #847 xmlns drop is fixed in PlanSerializer); any
+          # unspellable shape answers nil there and falls back
+          # interpretively.
           Lutaml::Xml::Leptris::PlanSerializer.call(instance, plan)
         end
 

data/lib/lutaml/model/serialize/model_import.rb

@@ -268,6 +268,11 @@ module Lutaml
         # Uses TracePoint to detect when a class definition is complete,
         # allowing deferred import resolution.
         def setup_trace_point
+          # Opal has no TracePoint. Symbol imports still resolve lazily
+          # on first use (ensure_imports!), so there the class is marked
+          # finalized at once instead of at the end of its body.
+          return @finalized = true if Lutaml::Model.opal?
+
           @trace ||= TracePoint.new(:end) do |_tp|
             if include?(Lutaml::Model::Serialize)
               @finalized = true

data/lib/lutaml/model/version.rb

@@ -2,6 +2,6 @@
 
 module Lutaml
   module Model
-    VERSION = "0.8.95"
+    VERSION = "0.8.96"
   end
 end

data/lib/lutaml/model.rb

@@ -294,3 +294,10 @@ require "#{__dir__}/xml/format"
 # Builder must be prepended AFTER XML so its initialize runs first
 # (Builder -> XML InstanceMethods -> Serialize)
 Lutaml::Model::Serialize.prepend(Lutaml::Model::Serialize::Builder)
+
+# Opal does not propagate a module prepended into an already-included
+# module to classes that included it earlier (see lib/lutaml/xml/format.rb).
+# Serializable included Serialize at boot, so Builder goes onto it directly.
+if Lutaml::Model.opal?
+  Lutaml::Model::Serializable.prepend(Lutaml::Model::Serialize::Builder)
+end

data/lib/lutaml/xml/format.rb

@@ -243,9 +243,11 @@ Lutaml::Model::Serialize.prepend(
 #
 # Anonymous model classes (Class.new { include Serialize }) extend
 # Serialize::ClassMethods rather than inheriting from Serializable, so the
-# ModelImportExt override (root?) must land on ClassMethods too. The other
-# two (FormatConversion, InstanceMethods) are already prepended unconditionally
-# above, so Opal's no-double-prepend rule means we don't repeat them here.
+# ModelImportExt override (root?) must land on ClassMethods too.
+#
+# On Opal (and TruffleRuby, below) the ClassMethods.prepend(FormatConversion)
+# above does not reach Serializable's singleton chain, which extended
+# ClassMethods before it ran, so it is prepended onto that singleton directly.
 #
 # TruffleRuby (34.0.1) has the same propagation gap, on both the include
 # side and the extend/singleton side — a module prepended into an
@@ -273,13 +275,9 @@ if Lutaml::Model.opal? || RUBY_ENGINE == "truffleruby"
     Lutaml::Xml::Serialization::ModelImportExt,
   )
 
-  # TruffleRuby only: Opal's no-double-prepend rule makes the unconditional
-  # ClassMethods.prepend(FormatConversion) above sufficient there.
-  if RUBY_ENGINE == "truffleruby"
-    Lutaml::Model::Serializable.singleton_class.prepend(
-      Lutaml::Xml::Serialization::FormatConversion,
-    )
-  end
+  Lutaml::Model::Serializable.singleton_class.prepend(
+    Lutaml::Xml::Serialization::FormatConversion,
+  )
 end
 
 # Register XML-specific attribute override warning names

data/lib/lutaml/xml/leptris/plan_compiler.rb

@@ -292,14 +292,28 @@ module Lutaml
             needs_nodes = true unless ns_attr_names.empty?
 
             # A model combining exact-URI attribute rows with nested
-            # child rows stays interpretive: the engine's walk drops
-            # nested-children attribute capture under such plans
-            # (FontTable's w:name under an mc:Ignorable ns row —
-            # observed 1.9.311.3; filed upstream). Leaf models keep the
-            # lift — that is the measured win (uniword's deep property
-            # models are leaves).
+            # child rows needs the walk's nested attribute capture
+            # (leptris#1563, fixed 1.9.313.0): older engines drop the
+            # children's attribute capture under such plans, so they
+            # stay interpretive. Leaf models are unaffected.
             return opt_out!(:attr_type_ns_nested) if plan_attrs.any? { |a| a[:ns] } &&
-              rows.any?
+              rows.any? && !Leptris.nested_attr_capture_compatible?
+
+            # The catch-all row: kind :wildcard with an explicit :any ns
+            # form (pad0) — the two-pass walk routes every child no
+            # named row claimed here, tagged for hydrator routing.
+            # Document order recovery and bridging happen in the
+            # hydrator against the source node.
+            if (any_rule = mapping.any_element_rule)
+              any_attr = model_class.attributes(register)[any_rule.to]
+              return opt_out!(:attr_nil) if any_attr.nil?
+
+              needs_nodes = true
+              wildcard_tag = tag += 1
+              compiled << [any_rule, any_attr, :wildcard_any, nil, nil]
+              rows << { name: "__lutaml_any__", kind: :wildcard, ns: :any,
+                        type_tag: wildcard_tag }
+            end
 
             { descriptor: descriptor, tree: tree, rows: compiled,
               attr_rows: attr_rows, mapping: mapping,
@@ -307,7 +321,8 @@ module Lutaml
               ordered: mapping.ordered? || mapping.mixed_content?,
               needs_nodes: needs_nodes,
               collection_defaults: collection_defaults,
-              ns_attr_names: ns_attr_names.empty? ? nil : ns_attr_names }
+              ns_attr_names: ns_attr_names.empty? ? nil : ns_attr_names,
+              wildcard_tag: (wildcard_tag if defined?(wildcard_tag)) }
           end
 
           # Attribute rows: identity is (URI, local) (lutaml-model#744).
@@ -341,7 +356,8 @@ module Lutaml
 
             claim[:ns] += 1
             plan_attrs << { name: rule.name.to_s,
-                            ns: { exact: type_ns.uri.to_s } }
+                            ns: { exact: type_ns.uri.to_s,
+                                  prefix: type_ns.prefix_default&.to_s } }
             ns_attr_names << rule.name.to_s
             nil
           end
@@ -349,12 +365,11 @@ module Lutaml
           def compilable_mapping?(mapping)
             # Mapping's uniform interface: root_mappings defaults to
             # false on the base (KeyValue overrides with its own).
-            # A map_any_element catch-all is not plan-shaped yet — the
-            # compiled plan has no row for it, so plan-path models
-            # would silently drop every catch-all child (0.8.86's
-            # smoke catch). The interpretive path owns them until the
-            # plan learns the catch-all row.
-            return false if mapping.any_element_rule
+            # The map_any_element catch-all rides the engine's wildcard
+            # row (leptris#1552, named rows take precedence); engines
+            # without the kind keep the interpretive refusal.
+            return false if mapping.any_element_rule &&
+              !Leptris.wildcard_rows_compatible?
 
             mapping.root_element && !mapping.root_mappings?
           end
@@ -415,9 +430,9 @@ module Lutaml
           # prefix. Blank-namespace rules (xmlns="") match :none.
           def child_ns(rule, _model_ns)
             uri = rule.namespace
-            return { exact: uri.to_s } if uri && !uri.to_s.empty?
+            return :none if uri.nil? || uri.to_s.empty?
 
-            :none
+            { exact: uri.to_s, prefix: rule.prefix&.to_s }
           end
 
           def content_rows(rows)
@@ -431,7 +446,10 @@ module Lutaml
             # Mapping's uniform interface: namespace_class defaults
             # to nil on the base (Xml overrides).
             ns_class = mapping.namespace_class
-            ns_class&.uri ? { exact: ns_class.uri.to_s } : nil
+            return nil unless ns_class&.uri
+
+            { exact: ns_class.uri.to_s,
+              prefix: ns_class.prefix_default&.to_s }
           end
 
           def scalar_type?(attr, register)

data/lib/lutaml/xml/leptris/plan_hydrator.rb

@@ -201,7 +201,7 @@ module Lutaml
                   v = rule.transform_value(attr, v, :from, :xml) if rule.transform.is_a?(Class)
                   assign(kwargs, delegates, delegate, rule, attr, v)
                 end
-              when :raw, :custom_method, :polymorphic, :content_deferred
+              when :wildcard_any
                 # interpreted post-instance (interpret_deferred)
               when :content
                 runs = content_runs(value)
@@ -352,6 +352,39 @@ module Lutaml
 
                 instance.public_send(:"#{attr.name}=",
                                      attr.collection? ? results : results.first)
+              when :wildcard_any
+                # map_any_element: every child element no named row
+                # claims, in document order, bridged as wrappers — the
+                # interpretive pass's contract. The wildcard row's
+                # type_tag marks its captures in the walk.
+                next unless node
+
+                claimed = plan[:rows].filter_map do |wr, _wa, _wk, spelling, _wd|
+                  next if wr.equal?(rule) || wr.name.nil?
+
+                  spelling ? spelling.to_s : wr.name.to_s
+                end.uniq
+                elements = []
+                node.children.each do |child|
+                  next unless child.is_a?(::Leptris::XML::Element)
+                  next if claimed.include?(child.name)
+
+                  elements << bridge_element(child)
+                end
+                next if elements.empty?
+
+                # The interpretive pass casts each collected element
+                # through the attribute: String keeps the wrapper,
+                # Serializable types build model instances.
+                results = elements.map do |element|
+                  attr.cast(element, :xml, register,
+                            lutaml_parent: instance,
+                            lutaml_root: instance.lutaml_root || instance)
+                end
+                instance.public_send(
+                  :"#{attr.name}=",
+                  attr.collection? ? results : results.first,
+                )
               when :polymorphic
                 results = deferred_elements(value, rule, buckets).map do |element|
                   attr.cast(element, :xml, register,