@metaobjectsdev/render 0.8.1-rc.1 → 0.9.0-rc.1

package/src/extract/extract.ts

@@ -0,0 +1,187 @@
+// Public entry point. Runs the staged pipeline; NEVER throws. Mirrors Java Extract.
+
+import {
+  Format,
+  FieldKind,
+  FieldExtraction,
+  Tolerance,
+  normalizeOptions,
+} from "./types.js";
+import type { FieldSpec, ExtractOptions, ExtractionOutcome, ExtractSchema } from "./types.js";
+import { ExtractionReport } from "./types.js";
+import { strip } from "./strip.js";
+import { locateJson, locateXml } from "./locate.js";
+import { readJson, TRUNCATED } from "./json-forgiving-reader.js";
+import { readXml } from "./xml-forgiving-reader.js";
+import { coerceValue, scalarCoerce, MALFORMED } from "./coerce.js";
+
+/** The forgiving entry point: extract dirty `text` against `schema`. Never throws. */
+export function extract(
+  text: string | null | undefined,
+  schema: ExtractSchema,
+  opts?: Partial<ExtractOptions> | null,
+): ExtractionOutcome {
+  const o = normalizeOptions(opts);
+  const report = new ExtractionReport();
+  const data: Record<string, unknown> = {};
+
+  const stripped = strip(text);
+  const ci = o.tolerance !== Tolerance.STRICT;
+
+  const span =
+    schema.format === Format.JSON ? locateJson(stripped) : locateXml(stripped, schema.rootName, ci);
+
+  let raw: Record<string, unknown>;
+  if (span == null) {
+    raw = {};
+  } else if (schema.format === Format.JSON) {
+    raw = readJson(span);
+  } else {
+    raw = readXml(span, ci);
+  }
+
+  if (isEmptyRecord(raw) && (stripped.length === 0 || span == null)) {
+    report.markEmpty();
+  }
+
+  extractFields(schema.fields, raw, "", data, report, o, ci);
+  return { data, report };
+}
+
+function extractFields(
+  fields: readonly FieldSpec[],
+  raw: Record<string, unknown>,
+  prefix: string,
+  data: Record<string, unknown>,
+  report: ExtractionReport,
+  o: ExtractOptions,
+  ci: boolean,
+): void {
+  for (const f of fields) {
+    const path = prefix.length === 0 ? f.name : `${prefix}.${f.name}`;
+    const present = lookup(raw, f.name, ci);
+    if (present === undefined) {
+      // FR-011 / Phase B: an absent field with a declared @default fills the value → DEFAULTED
+      // (which satisfies a @required field). Generalized to all field kinds: an enum default is
+      // its member string as-is; a non-enum default is coerced to the field's kind via the pure
+      // scalar coerce (so @default "0" on field.int yields integer 0). A non-coercible non-enum
+      // default is treated as no default.
+      if (f.defaultValue != null) {
+        const coerced =
+          f.kind === FieldKind.ENUM ? f.defaultValue : scalarCoerce(f.defaultValue, f);
+        if (coerced !== MALFORMED) {
+          data[f.name] = coerced;
+          report.addCoercion({ fieldPath: path, from: "", to: f.defaultValue, kind: "default" });
+          report.set(path, FieldExtraction.DEFAULTED);
+          continue;
+        }
+      }
+      report.set(path, f.required ? FieldExtraction.LOST_REQUIRED : FieldExtraction.LOST_OPTIONAL);
+      continue;
+    }
+    if (present === TRUNCATED) {
+      // present-but-garbled (empty/cut-off value)
+      report.set(path, FieldExtraction.MALFORMED);
+      continue;
+    }
+    if (f.array) {
+      // An array field: a single non-list value is treated as a one-element array
+      // (e.g. a single repeated-XML tag). Each element is coerced/recursed independently.
+      const elements: unknown[] = Array.isArray(present) ? present : [present];
+      const out: unknown[] = [];
+      let anyMalformed = false;
+      // Phase B (array-of-enum): an enum element flows through the SAME enum coercion pipeline a
+      // scalar enum uses (extractValue → coerceValue → coerceEnum), and is CLASSIFIED per element
+      // by indexed path (tags[0], tags[1], …) exactly as a scalar enum: EXTRACTED / DEFAULTED (via
+      // @coerceDefault) / MALFORMED. Non-enum scalar arrays keep their existing behavior (raw
+      // element list, no per-element states).
+      const enumElements = f.kind === FieldKind.ENUM;
+      for (let idx = 0; idx < elements.length; idx++) {
+        const elemPath = `${path}[${idx}]`;
+        const v = extractValue(f, elements[idx], elemPath, report, o, ci);
+        if (v === MALFORMED) {
+          anyMalformed = true;
+          if (enumElements) report.set(elemPath, FieldExtraction.MALFORMED);
+        } else {
+          out.push(v);
+          if (enumElements) report.set(elemPath, classifyCoerced(elemPath, report));
+        }
+      }
+      // Cross-port contract: a MALFORMED array still places its successfully-coerced
+      // elements into data (partial extraction), UNLIKE a MALFORMED scalar which is absent.
+      data[f.name] = out;
+      report.set(path, anyMalformed ? FieldExtraction.MALFORMED : FieldExtraction.EXTRACTED);
+      continue;
+    }
+    if (Array.isArray(present)) {
+      // a list where a singular value was expected
+      report.set(path, FieldExtraction.MALFORMED);
+      continue;
+    }
+    const v = extractValue(f, present, path, report, o, ci);
+    if (v === MALFORMED) {
+      report.set(path, FieldExtraction.MALFORMED);
+    } else {
+      data[f.name] = v;
+      // FR-011: a value reached via @coerceDefault (or @default) is DEFAULTED, not EXTRACTED.
+      report.set(path, classifyCoerced(path, report));
+    }
+  }
+}
+
+/**
+ * FR-011: classify a successfully-coerced field. DEFAULTED when its terminal (last-logged)
+ * coercion for this path is a default-class fallback; EXTRACTED otherwise. Nested objects
+ * (which log no coercion of their own) classify as EXTRACTED.
+ */
+function classifyCoerced(path: string, report: ExtractionReport): FieldExtraction {
+  let terminalKind: string | null = null;
+  for (const c of report.coercions()) if (c.fieldPath === path) terminalKind = c.kind;
+  return terminalKind === "coerceDefault" || terminalKind === "default"
+    ? FieldExtraction.DEFAULTED
+    : FieldExtraction.EXTRACTED;
+}
+
+/** Coerce one (non-array) element: nested-object recursion or scalar coercion. Returns MALFORMED on failure. */
+function extractValue(
+  f: FieldSpec,
+  present: unknown,
+  path: string,
+  report: ExtractionReport,
+  o: ExtractOptions,
+  ci: boolean,
+): unknown | typeof MALFORMED {
+  if (f.kind === FieldKind.OBJECT) {
+    if (f.nested != null && isPlainObject(present)) {
+      const nestedData: Record<string, unknown> = {};
+      extractFields(f.nested.fields, present as Record<string, unknown>, path, nestedData, report, o, ci);
+      return nestedData;
+    }
+    return MALFORMED; // object expected but scalar/non-map present
+  }
+  const rawStr = typeof present === "string" ? present : stringifyScalar(present);
+  return coerceValue(rawStr, f, o, path, report);
+}
+
+/** Case-folding lookup honoring tolerance. Returns `undefined` for absent (mirrors Java null). */
+function lookup(raw: Record<string, unknown>, name: string, ci: boolean): unknown {
+  if (Object.prototype.hasOwnProperty.call(raw, name)) return raw[name];
+  if (ci) {
+    const lower = name.toLowerCase();
+    for (const k of Object.keys(raw)) if (k.toLowerCase() === lower) return raw[k];
+  }
+  return undefined;
+}
+
+function isPlainObject(o: unknown): boolean {
+  return typeof o === "object" && o !== null && !Array.isArray(o);
+}
+
+function isEmptyRecord(o: Record<string, unknown>): boolean {
+  return Object.keys(o).length === 0;
+}
+
+/** Mirror Java String.valueOf for non-string forgiving-reader scalars. */
+function stringifyScalar(v: unknown): string {
+  return String(v);
+}

package/src/{recover → extract}/json-forgiving-reader.ts

@@ -2,7 +2,7 @@
 // Mirrors Java JsonForgivingReader. The no-hang + TRUNCATED contracts are load-bearing.
 
 /** Sentinel: a key appeared in the text but its value was empty/cut-off (present-but-garbled). */
-export const TRUNCATED: unique symbol = Symbol("recover.json.TRUNCATED");
+export const TRUNCATED: unique symbol = Symbol("extract.json.TRUNCATED");
 
 /** A character is JSON-insignificant whitespace. Mirrors Java Character.isWhitespace closely enough for the corpus. */
 function isWhitespace(c: string): boolean {

package/src/extract/normalize.ts

@@ -0,0 +1,23 @@
+// FR-011: enum-variant normalization for the Coerce stage.
+// ASCII-only by design: enum members are ASCII identifiers, so a pure [A-Za-z0-9]
+// transform is byte-identical across ports and sidesteps locale case-folding (Turkish-İ).
+// Mode comes from the @normalize attr (none|collapse|strip; default strip).
+
+export type NormalizeMode = "none" | "collapse" | "strip";
+
+/** ASCII-only enum normalization. Pure [A-Za-z0-9] transform → byte-identical cross-port. */
+export function normalizeEnum(s: string, mode: NormalizeMode): string {
+  if (mode === "none") return s;
+  const up = asciiUpper(s.trim());
+  if (mode === "collapse") return up.replace(/[\s_-]+/g, "_");
+  return up.replace(/[^A-Z0-9]/g, ""); // strip
+}
+
+function asciiUpper(s: string): string {
+  let out = "";
+  for (let i = 0; i < s.length; i++) {
+    const c = s.charCodeAt(i);
+    out += c >= 97 && c <= 122 ? String.fromCharCode(c - 32) : s[i];
+  }
+  return out;
+}

package/src/extract/types.ts

@@ -0,0 +1,346 @@
+import type { NormalizeMode } from "./normalize.js";
+
+// FR-010 extract engine — types & model (Tier-2 idiomatic TS port).
+//
+// Cross-port REFERENCE is the Java engine
+// (server/java/render/.../extract/). This file ports the Java records/enums to
+// idiomatic TS: enums become string-union `as const` objects (values match the
+// corpus / Java enum names exactly), records become readonly interfaces +
+// factory functions, and the mutable ExtractionReport is a class.
+
+/** Output format the dirty text claims to be. Corpus schema.json uses "JSON"/"XML". */
+export const Format = {
+  JSON: "JSON",
+  XML: "XML",
+} as const;
+export type Format = (typeof Format)[keyof typeof Format];
+
+/** The coercion target kinds the engine understands. OBJECT = nested ExtractSchema. */
+export const FieldKind = {
+  STRING: "STRING",
+  INT: "INT",
+  LONG: "LONG",
+  DOUBLE: "DOUBLE",
+  BOOLEAN: "BOOLEAN",
+  ENUM: "ENUM",
+  OBJECT: "OBJECT",
+} as const;
+export type FieldKind = (typeof FieldKind)[keyof typeof FieldKind];
+
+/**
+ * FROZEN cross-port per-field extraction classification. Do not reorder or add
+ * without an ADR. These string values are SERIALIZED in the conformance corpus.
+ */
+export const FieldExtraction = {
+  EXTRACTED: "EXTRACTED",
+  // A `@default`/`@coerceDefault`-backed value (absent-fill or present-but-uncoercible fallback).
+  DEFAULTED: "DEFAULTED",
+  LOST_OPTIONAL: "LOST_OPTIONAL",
+  LOST_REQUIRED: "LOST_REQUIRED",
+  MALFORMED: "MALFORMED",
+} as const;
+export type FieldExtraction = (typeof FieldExtraction)[keyof typeof FieldExtraction];
+
+/**
+ * STRICT: case-sensitive, minimal repair. NORMAL: case-insensitive keys/tags
+ * (default). LOOSE: maximal repair (currently identical to NORMAL — reserved).
+ */
+export const Tolerance = {
+  STRICT: "STRICT",
+  NORMAL: "NORMAL",
+  LOOSE: "LOOSE",
+} as const;
+export type Tolerance = (typeof Tolerance)[keyof typeof Tolerance];
+
+/** A recorded normalization/coercion. kind e.g. "normalize", "alias", "runtime-alias-override", "clamp", "coerceDefault", "default". */
+export interface Coercion {
+  readonly fieldPath: string;
+  readonly from: string;
+  readonly to: string;
+  readonly kind: string;
+}
+
+/**
+ * One field's extract descriptor. enumValues/enumAlias non-null only for ENUM;
+ * min/max non-null only for numeric range constraints; nested non-null only for OBJECT.
+ */
+export interface FieldSpec {
+  readonly name: string;
+  readonly kind: FieldKind;
+  readonly required: boolean;
+  readonly array: boolean;
+  readonly enumValues: readonly string[] | null;
+  readonly enumAlias: Readonly<Record<string, string>> | null;
+  readonly min: number | null;
+  readonly max: number | null;
+  readonly nested: ExtractSchema | null;
+  /** FR-011: present-but-uncoercible fallback member (from `@coerceDefault`). ENUM-only; null = none. */
+  readonly coerceDefault: string | null;
+  /**
+   * Absent-fill default (from `@default`). When the field is ABSENT, extract fills this value
+   * → DEFAULTED (which satisfies `@required`). Generalized to ALL field kinds (Phase B): for an
+   * enum it is the member string verbatim; for a non-enum it is coerced to `kind` via the pure
+   * scalar coerce (so `@default "0"` on `field.int` yields integer 0). null = no default.
+   */
+  readonly defaultValue: string | null;
+  /** FR-011: resolved enum normalization mode (from `@normalize`; default `"strip"`). */
+  readonly normalize: NormalizeMode;
+}
+
+/**
+ * A scalar field, optionally carrying an absent-fill `@default` (Phase B — generalized
+ * `@default`). When ABSENT from the model response, extract coerces `defaultValue` to `kind`
+ * and classifies the field DEFAULTED (which satisfies `@required`). `defaultValue == null` is
+ * the no-default case (back-compat with the original two-arg call).
+ */
+export function scalar(
+  name: string,
+  kind: FieldKind,
+  required: boolean,
+  defaultValue?: string | null,
+): FieldSpec {
+  return {
+    name,
+    kind,
+    required,
+    array: false,
+    enumValues: null,
+    enumAlias: null,
+    min: null,
+    max: null,
+    nested: null,
+    coerceDefault: null,
+    defaultValue: defaultValue ?? null,
+    normalize: "strip",
+  };
+}
+
+export function enumField(
+  name: string,
+  required: boolean,
+  values: readonly string[] | null,
+  aliases: Readonly<Record<string, string>> | null,
+  coerceDefault?: string | null,
+  normalize: NormalizeMode = "strip",
+  defaultValue?: string | null,
+): FieldSpec {
+  return {
+    name,
+    kind: FieldKind.ENUM,
+    required,
+    array: false,
+    enumValues: values == null ? null : [...values],
+    enumAlias: aliases == null ? {} : { ...aliases },
+    min: null,
+    max: null,
+    nested: null,
+    coerceDefault: coerceDefault ?? null,
+    defaultValue: defaultValue ?? null,
+    normalize,
+  };
+}
+
+/**
+ * Phase B (array-of-enum): an enum field that is a list (`array === true`). Each element flows
+ * through the SAME enum coercion pipeline a scalar enum uses (exact → normalize → `@enumAlias`
+ * → `@coerceDefault` → MALFORMED) and is classified independently by indexed path (`tags[0]`,
+ * `tags[1]`, …). Mirrors {@link enumField} but with `array = true`.
+ */
+export function enumArray(
+  name: string,
+  required: boolean,
+  values: readonly string[] | null,
+  aliases: Readonly<Record<string, string>> | null,
+  coerceDefault?: string | null,
+  normalize: NormalizeMode = "strip",
+  defaultValue?: string | null,
+): FieldSpec {
+  return {
+    name,
+    kind: FieldKind.ENUM,
+    required,
+    array: true,
+    enumValues: values == null ? null : [...values],
+    enumAlias: aliases == null ? {} : { ...aliases },
+    min: null,
+    max: null,
+    nested: null,
+    coerceDefault: coerceDefault ?? null,
+    defaultValue: defaultValue ?? null,
+    normalize,
+  };
+}
+
+export function range(
+  name: string,
+  kind: FieldKind,
+  required: boolean,
+  min: number | null,
+  max: number | null,
+): FieldSpec {
+  return {
+    name,
+    kind,
+    required,
+    array: false,
+    enumValues: null,
+    enumAlias: null,
+    min,
+    max,
+    nested: null,
+    coerceDefault: null,
+    defaultValue: null,
+    normalize: "strip",
+  };
+}
+
+export function object(name: string, required: boolean, array: boolean, nested: ExtractSchema | null): FieldSpec {
+  return {
+    name,
+    kind: FieldKind.OBJECT,
+    required,
+    array,
+    enumValues: null,
+    enumAlias: null,
+    min: null,
+    max: null,
+    nested,
+    coerceDefault: null,
+    defaultValue: null,
+    normalize: "strip",
+  };
+}
+
+/** Top-level extract descriptor. rootName = the XML root tag / logical JSON root name. */
+export interface ExtractSchema {
+  readonly format: Format;
+  readonly rootName: string;
+  readonly fields: readonly FieldSpec[];
+}
+
+export function extractSchema(format: Format, rootName: string, fields: readonly FieldSpec[] | null): ExtractSchema {
+  return { format, rootName, fields: fields == null ? [] : [...fields] };
+}
+
+/**
+ * ctx carries the field path and the FieldSpec; return null to fall through to
+ * default coercion. The single bespoke-coercion hook (the bounded "20%").
+ */
+export type OnField = (fieldPath: string, rawValue: string, spec: FieldSpec) => unknown | null;
+
+/**
+ * Bounded runtime override surface. aliases/normalizers are MERGED with the
+ * schema's, runtime winning on key conflict. onField is the single hook.
+ */
+export interface ExtractOptions {
+  readonly tolerance: Tolerance;
+  readonly aliases: Readonly<Record<string, string>>;
+  readonly normalizers: Readonly<Record<string, (raw: string) => unknown | null>>;
+  readonly onField: OnField | null;
+}
+
+export function defaults(): ExtractOptions {
+  return { tolerance: Tolerance.NORMAL, aliases: {}, normalizers: {}, onField: null };
+}
+
+/** Normalize a partial / undefined options bag into a complete ExtractOptions. */
+export function normalizeOptions(opts?: Partial<ExtractOptions> | null): ExtractOptions {
+  if (opts == null) return defaults();
+  return {
+    tolerance: opts.tolerance ?? Tolerance.NORMAL,
+    aliases: opts.aliases == null ? {} : { ...opts.aliases },
+    normalizers: opts.normalizers == null ? {} : { ...opts.normalizers },
+    onField: opts.onField ?? null,
+  };
+}
+
+/** Engine return. data is a forgiving record; Phase-2 codegen wraps it into a typed ExtractionResult<T>. */
+export interface ExtractionOutcome {
+  readonly data: Record<string, unknown>;
+  readonly report: ExtractionReport;
+}
+
+/** Typed result of a generated extract(...): best-effort value (null where lost/malformed) + report. */
+export interface ExtractionResult<T> {
+  readonly data: T | null;
+  readonly report: ExtractionReport;
+}
+
+/**
+ * Thrown by {@link orThrow} when a {@link ExtractionResult} lost a `@required` field. Mirrors
+ * Java's `ExtractException`. Carries the list of lost-required field paths.
+ */
+export class ExtractError extends Error {
+  readonly lostRequired: readonly string[];
+  constructor(lostRequired: readonly string[]) {
+    super(`extract: required field(s) lost: ${lostRequired.join(", ")}`);
+    this.name = "ExtractError";
+    this.lostRequired = [...lostRequired];
+  }
+}
+
+/**
+ * Opt-in strictness over a never-throwing {@link ExtractionResult}. Mirrors Java
+ * `ExtractionResult.orThrow()`. Throws a {@link ExtractError} iff the report has a lost
+ * `@required` field; otherwise returns `result.data`.
+ *
+ * <p>TS divergence from Java (documented): `ExtractionResult` is a plain interface (the generated
+ * output-parsers build it as an object literal), so `orThrow` is a free function rather than a
+ * method on the result. Semantics are identical.</p>
+ */
+export function orThrow<T>(result: ExtractionResult<T>): T | null {
+  if (result.report.hasLostRequired()) {
+    throw new ExtractError(result.report.lostRequired());
+  }
+  return result.data;
+}
+
+/** Mutable accumulator of per-field extraction classification, the degenerate-response flag, and coercion notes. */
+export class ExtractionReport {
+  // Insertion-ordered (Map preserves insertion order, mirroring Java LinkedHashMap).
+  private readonly _states = new Map<string, FieldExtraction>();
+  private readonly _coercions: Coercion[] = [];
+  private _empty = false;
+
+  set(fieldPath: string, state: FieldExtraction): void {
+    this._states.set(fieldPath, state);
+  }
+
+  addCoercion(c: Coercion): void {
+    this._coercions.push(c);
+  }
+
+  markEmpty(): void {
+    this._empty = true;
+  }
+
+  isEmpty(): boolean {
+    return this._empty;
+  }
+
+  states(): ReadonlyMap<string, FieldExtraction> {
+    return new Map(this._states);
+  }
+
+  coercions(): readonly Coercion[] {
+    return [...this._coercions];
+  }
+
+  lostRequired(): string[] {
+    return this.byState(FieldExtraction.LOST_REQUIRED);
+  }
+
+  malformed(): string[] {
+    return this.byState(FieldExtraction.MALFORMED);
+  }
+
+  hasLostRequired(): boolean {
+    return this.lostRequired().length > 0;
+  }
+
+  private byState(s: FieldExtraction): string[] {
+    const out: string[] = [];
+    for (const [k, v] of this._states) if (v === s) out.push(k);
+    return out;
+  }
+}

package/src/{recover → extract}/xml-forgiving-reader.ts

@@ -44,7 +44,7 @@ function parseChildren(inner: string, ci: boolean, out: Record<string, unknown>)
       contentEnd = close.index;
       next = close.index + close[0].length;
     } else {
-      // unclosed tag: recover text up to the next sibling open tag
+      // unclosed tag: extract text up to the next sibling open tag
       const sib = matchFrom(OPEN_TAG_SRC, flags, inner, contentStart);
       if (sib != null) {
         contentEnd = sib.index;

package/src/index.ts

@@ -12,28 +12,31 @@ export {
   type VerifyOptions,
 } from "./verify.js";
 
-// FR-010 tolerant recover engine (Tier-2 forgiving parser).
-export { recover } from "./recover/recover.js";
+// FR-010 tolerant extract engine (Tier-2 forgiving parser).
+export { extract } from "./extract/extract.js";
 export {
   Format,
   FieldKind,
-  FieldRecovery,
+  FieldExtraction,
   Tolerance,
-  RecoveryReport,
+  ExtractionReport,
   scalar,
   enumField,
+  enumArray,
   range,
   object,
-  recoverSchema,
+  extractSchema,
   defaults,
+  orThrow,
+  ExtractError,
   type FieldSpec,
-  type RecoverSchema,
-  type RecoverOptions,
-  type RecoverOutcome,
-  type RecoveryResult,
+  type ExtractSchema,
+  type ExtractOptions,
+  type ExtractionOutcome,
+  type ExtractionResult,
   type Coercion,
   type OnField,
-} from "./recover/types.js";
+} from "./extract/types.js";
 export {
   asString,
   asInt,
@@ -41,7 +44,7 @@ export {
   asDouble,
   asBool,
   asStringList,
-} from "./recover/recover-map.js";
+} from "./extract/extract-map.js";
 
 // FR-010 artifact 1 — output-format prompt renderer ("produce your answer like this").
 export { renderOutputFormat } from "./prompt/output-format-renderer.js";
@@ -53,3 +56,6 @@ export {
 } from "./prompt/prompt-overrides.js";
 export type { OutputFormatSpec } from "./prompt/output-format-spec.js";
 export type { PromptField } from "./prompt/prompt-field.js";
+
+// template.output render-helper result shape (shared per port).
+export type { EmailDocument } from "./email-document.js";