@metaobjectsdev/render 0.9.0 → 0.11.0-rc.1

package/src/extract/types.ts

@@ -85,6 +85,13 @@ export interface FieldSpec {
   readonly defaultValue: string | null;
   /** FR-011: resolved enum normalization mode (from `@normalize`; default `"strip"`). */
   readonly normalize: NormalizeMode;
+  /**
+   * `@xmlText`: this field receives its element's TEXT CONTENT (analogous to JAXB `@XmlValue` /
+   * Jackson `@JacksonXmlText` / .NET `[XmlText]`). The extract engine reads it from the
+   * `#text` sentinel the lenient XML reader carries when an element has both attributes and a
+   * text body, instead of a same-named child. Absent/false for normal fields and for JSON.
+   */
+  readonly textContent?: boolean;
 }
 
 /**
@@ -115,6 +122,14 @@ export function scalar(
   };
 }
 
+/**
+ * A field that receives its element's TEXT CONTENT — the `@xmlText` marker (see
+ * {@link FieldSpec.textContent}). A scalar with the `textContent` flag set; coerced to `kind`.
+ */
+export function textContentField(name: string, kind: FieldKind, required: boolean): FieldSpec {
+  return { ...scalar(name, kind, required), textContent: true };
+}
+
 export function enumField(
   name: string,
   required: boolean,
@@ -231,16 +246,23 @@ export type OnField = (fieldPath: string, rawValue: string, spec: FieldSpec) =>
 /**
  * Bounded runtime override surface. aliases/normalizers are MERGED with the
  * schema's, runtime winning on key conflict. onField is the single hook.
+ *
+ * `rootless` (XML only): when `true`, the input has NO enclosing root element — the payload's
+ * fields ARE the top-level elements (a flat sequence like `<a>..</a><b>..</b>`). The engine
+ * parses those top-level elements directly instead of locating a `<rootName>` span, so the caller
+ * need not synthesize a wrapper. No effect for JSON. Default `false` (a single root element is
+ * expected, as before). Mirrors Java ExtractOptions.rootless.
  */
 export interface ExtractOptions {
   readonly tolerance: Tolerance;
   readonly aliases: Readonly<Record<string, string>>;
   readonly normalizers: Readonly<Record<string, (raw: string) => unknown | null>>;
   readonly onField: OnField | null;
+  readonly rootless: boolean;
 }
 
 export function defaults(): ExtractOptions {
-  return { tolerance: Tolerance.NORMAL, aliases: {}, normalizers: {}, onField: null };
+  return { tolerance: Tolerance.NORMAL, aliases: {}, normalizers: {}, onField: null, rootless: false };
 }
 
 /** Normalize a partial / undefined options bag into a complete ExtractOptions. */
@@ -251,6 +273,7 @@ export function normalizeOptions(opts?: Partial<ExtractOptions> | null): Extract
     aliases: opts.aliases == null ? {} : { ...opts.aliases },
     normalizers: opts.normalizers == null ? {} : { ...opts.normalizers },
     onField: opts.onField ?? null,
+    rootless: opts.rootless ?? false,
   };
 }
 

package/src/extract/xml-forgiving-reader.ts

@@ -1,5 +1,21 @@
 // Stage-4 tolerant XML reader for the bounded corpus malformation set. Never throws.
-// Mirrors Java XmlForgivingReader. Must not index-out-of-range on a leading close tag.
+// Mirrors Java XmlForgivingReader: maps an element's child elements, text, AND attributes
+// into the field map, and handles self-closing tags (<x a="1"/>). Must not index-out-of-range
+// on a leading close tag.
+//
+// Representation:
+//   - text-only element, no attributes        → its trimmed text (string) — unchanged
+//   - self-closing / attributes-only element   → a record of attribute name→value ("" when none)
+//   - element with child elements (± attrs)     → a record merging attributes + child entries
+//                                                  (a child element wins a name collision)
+//   - element with text AND attributes          → a record of the attributes plus the body text
+//                                                  under TEXT_KEY (a scalar consumer unwraps it)
+//   - repeated sibling tags                     → an array (unchanged)
+
+/** Reserved key holding an element's own text content when the element is represented as a
+ *  record (because it also carries attributes). '#' is not a legal XML name char, so it never
+ *  collides with a real attribute or child-element name. */
+export const TEXT_KEY = "#text";
 
 function quote(s: string): string {
   return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
@@ -12,7 +28,11 @@ function matchFrom(source: string, flags: string, text: string, from: number): R
   return g.exec(text);
 }
 
-const OPEN_TAG_SRC = "<([A-Za-z_][A-Za-z0-9_]*)(\\s[^>]*)?>";
+// tag name + everything up to the closing '>' (attributes and/or a trailing '/' for a
+// self-closing tag). Non-greedy so the first '>' closes the open tag.
+const OPEN_TAG_SRC = "<([A-Za-z_][A-Za-z0-9_]*)([^>]*?)>";
+// one attribute: name = "double" | 'single' | bareword.
+const ATTR_SRC = "([A-Za-z_:][A-Za-z0-9_:.\\-]*)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s/>]+))";
 
 export function readXml(span: string | null | undefined, caseInsensitive: boolean): Record<string, unknown> {
   const out: Record<string, unknown> = {};
@@ -25,6 +45,21 @@ export function readXml(span: string | null | undefined, caseInsensitive: boolea
   return out;
 }
 
+/**
+ * Rootless read: parse the WHOLE text's top-level elements directly, with no enclosing root
+ * element to strip (a flat sequence like `<a>..</a><b>..</b>`). Used for `ExtractOptions.rootless`
+ * responses. Leading/trailing non-element text is ignored. Never throws. Mirrors Java readRootless.
+ */
+export function readXmlRootless(
+  text: string | null | undefined,
+  caseInsensitive: boolean,
+): Record<string, unknown> {
+  const out: Record<string, unknown> = {};
+  if (text == null || text.trim().length === 0) return out;
+  parseChildren(text, caseInsensitive, out);
+  return out;
+}
+
 function parseChildren(inner: string, ci: boolean, out: Record<string, unknown>): void {
   const flags = ci ? "i" : "";
   let pos = 0;
@@ -33,8 +68,19 @@ function parseChildren(inner: string, ci: boolean, out: Record<string, unknown>)
     if (m == null) break;
     const tag = m[1] ?? "";
     const key = ci ? tag.toLowerCase() : tag;
-    const contentStart = m.index + m[0].length;
 
+    let rawAttrs = (m[2] ?? "").trim();
+    const selfClosing = rawAttrs.endsWith("/");
+    if (selfClosing) rawAttrs = rawAttrs.slice(0, -1).trim();
+    const attrs = parseAttrs(rawAttrs, ci);
+
+    if (selfClosing) {
+      accumulate(out, key, Object.keys(attrs).length === 0 ? "" : attrs);
+      pos = m.index + m[0].length;
+      continue;
+    }
+
+    const contentStart = m.index + m[0].length;
     const closeRe = `</${quote(tag)}\\s*>`;
     const close = matchFrom(closeRe, flags, inner, contentStart);
 
@@ -44,11 +90,25 @@ function parseChildren(inner: string, ci: boolean, out: Record<string, unknown>)
       contentEnd = close.index;
       next = close.index + close[0].length;
     } else {
-      // unclosed tag: extract text up to the next sibling open tag
+      // unclosed tag: extract content up to the next sibling open tag.
       const sib = matchFrom(OPEN_TAG_SRC, flags, inner, contentStart);
       if (sib != null) {
-        contentEnd = sib.index;
-        next = contentEnd;
+        // When the unclosed element's content begins IMMEDIATELY with a child open tag
+        // (no leading text), that child was almost certainly meant to be NESTED, not a
+        // sibling — a common LLM malformation is dropping the parent's close tag while
+        // still emitting a real child element (e.g. <check ...><payoff>text). Absorb the
+        // remainder of this span as the unclosed element's content so the child nests
+        // under it. When there IS leading text before the first child tag (e.g. <t>hi<c>..),
+        // keep the sibling split — the leading text is the unclosed element's body and the
+        // following tag is its sibling. Mirrors Java XmlForgivingReader.
+        const noLeadingText = inner.substring(contentStart, sib.index).trim().length === 0;
+        if (noLeadingText) {
+          contentEnd = inner.length;
+          next = inner.length;
+        } else {
+          contentEnd = sib.index;
+          next = contentEnd;
+        }
       } else {
         contentEnd = inner.length;
         next = inner.length;
@@ -56,16 +116,43 @@ function parseChildren(inner: string, ci: boolean, out: Record<string, unknown>)
     }
 
     const content = inner.substring(contentStart, contentEnd);
-    const value: unknown = content.includes("<") ? nestedOrText(content, ci) : content.trim();
-    accumulate(out, key, value);
+    accumulate(out, key, combine(attrs, content, ci));
     pos = next;
   }
 }
 
-function nestedOrText(content: string, ci: boolean): unknown {
-  const nested: Record<string, unknown> = {};
-  parseChildren(content, ci, nested);
-  return Object.keys(nested).length === 0 ? content.trim() : nested;
+/** Combine an element's attributes with its body (nested children or plain text). */
+function combine(attrs: Record<string, unknown>, content: string, ci: boolean): unknown {
+  if (content.includes("<")) {
+    const nested: Record<string, unknown> = {};
+    parseChildren(content, ci, nested);
+    if (Object.keys(nested).length > 0) {
+      // attributes first; a child element wins a name collision
+      return { ...attrs, ...nested };
+    }
+  }
+  return textValue(attrs, content);
+}
+
+function textValue(attrs: Record<string, unknown>, content: string): unknown {
+  const text = content.trim();
+  if (Object.keys(attrs).length === 0) return text;
+  return { ...attrs, [TEXT_KEY]: text };
+}
+
+function parseAttrs(rawAttrs: string, ci: boolean): Record<string, unknown> {
+  const attrs: Record<string, unknown> = {};
+  if (rawAttrs.length === 0) return attrs;
+  const re = new RegExp(ATTR_SRC, "g");
+  let a: RegExpExecArray | null;
+  while ((a = re.exec(rawAttrs)) != null) {
+    const rawName = a[1];
+    if (rawName === undefined) continue; // group 1 is mandatory in a match; guards strict TS
+    const name = ci ? rawName.toLowerCase() : rawName;
+    const val = a[2] ?? a[3] ?? a[4] ?? "";
+    if (!Object.prototype.hasOwnProperty.call(attrs, name)) attrs[name] = val;
+  }
+  return attrs;
 }
 
 function accumulate(out: Record<string, unknown>, key: string, value: unknown): void {

package/src/index.ts

@@ -3,11 +3,14 @@ export { type Provider, InMemoryProvider } from "./provider.js";
 export { ESCAPERS, type RenderFormat } from "./escapers.js";
 export {
   verify,
+  resolveTemplateVariable,
+  parseTemplate,
   ERR_VAR_NOT_ON_PAYLOAD,
   ERR_PARTIAL_UNRESOLVED,
   ERR_REQUIRED_SLOT_UNUSED,
   ERR_OUTPUT_TAG_MISSING,
   type PayloadField,
+  type ResolveStack,
   type VerifyError,
   type VerifyOptions,
 } from "./verify.js";
@@ -21,6 +24,7 @@ export {
   Tolerance,
   ExtractionReport,
   scalar,
+  textContentField,
   enumField,
   enumArray,
   range,

package/src/verify.ts

@@ -54,20 +54,34 @@ const MAX_DEPTH = 32;
 
 // A Mustache parse token: [type, value, start, end, subTokens?, ...].
 type Token = readonly unknown[];
-// The context stack — innermost context last, mirroring Mustache lookup order.
-type Stack = readonly PayloadField[][];
+/**
+ * The context stack — innermost context last, mirroring Mustache lookup order.
+ * Generic over the field node so consumers (e.g. the docs annotator) can resolve
+ * an ENRICHED field tree (carrying owner/type metadata) through the EXACT same
+ * walk verify uses, guaranteeing the two surfaces agree.
+ */
+export type ResolveStack<F extends PayloadField = PayloadField> = readonly F[][];
 
-function find(fields: PayloadField[], name: string): PayloadField | undefined {
+function find<F extends PayloadField>(fields: F[], name: string): F | undefined {
   return fields.find((f) => f.name === name);
 }
 
-// Resolve a (possibly dotted) variable path the way Mustache does: the FIRST
-// segment is looked up through the context stack (innermost → outermost); each
-// remaining segment is a direct descent into the resolved field's `fields`.
-// Returns the resolved field, or undefined if any segment is missing.
-function resolve(stack: Stack, path: string): PayloadField | undefined {
+/**
+ * Resolve a (possibly dotted) variable path the way Mustache does: the FIRST
+ * segment is looked up through the context stack (innermost → outermost); each
+ * remaining segment is a direct descent into the resolved field's `fields`.
+ * Returns the resolved field, or undefined if any segment is missing.
+ *
+ * EXPORTED so the docs annotator can share this ONE resolution (annotator ⇆
+ * verify must agree). Generic over the node type: an enriched tree resolves the
+ * same way, since only `name`/`fields` drive the walk.
+ */
+export function resolveTemplateVariable<F extends PayloadField>(
+  stack: ResolveStack<F>,
+  path: string,
+): F | undefined {
   const segs = path.split(".");
-  let current: PayloadField | undefined;
+  let current: F | undefined;
   for (let i = stack.length - 1; i >= 0; i--) {
     const hit = find(stack[i]!, segs[0]!);
     if (hit) {
@@ -76,15 +90,27 @@ function resolve(stack: Stack, path: string): PayloadField | undefined {
     }
   }
   for (let i = 1; current && i < segs.length; i++) {
-    current = current.fields ? find(current.fields, segs[i]!) : undefined;
+    current = current.fields ? (find(current.fields, segs[i]!) as F | undefined) : undefined;
   }
   return current;
 }
 
+// Internal alias preserving the original call sites unchanged.
+const resolve = resolveTemplateVariable;
+
 function parse(text: string): Token[] {
   return Mustache.parse(text) as unknown as Token[];
 }
 
+/**
+ * Parse a template into Mustache tokens (`[type, value, start, end, subTokens?]`),
+ * the SAME parse verify walks. Exported so the docs annotator tokenizes through
+ * one parser (no divergent re-tokenization). Returns a readonly token list.
+ */
+export function parseTemplate(text: string): readonly (readonly unknown[])[] {
+  return parse(text);
+}
+
 // An opening tag is `<tag` immediately followed by `>` or XML whitespace, so
 // attributes are allowed (`<answer foo="1">`) but a longer name is not over-matched
 // (`<answers>` does not satisfy `answer`).
@@ -124,7 +150,7 @@ export function verify(
   // (no second resolution pass).
   const staticTexts: string[] = [templateText];
 
-  function walk(tokens: Token[], stack: Stack, seen: readonly string[]): void {
+  function walk(tokens: Token[], stack: ResolveStack, seen: readonly string[]): void {
     const atRoot = stack.length === 1 && stack[0] === root;
     for (const tok of tokens) {
       const type = tok[0] as string;