@plumix/core 0.5.0 → 0.7.0

package/dist/rpc/meta/core.js

@@ -1,6 +1,11 @@
 import { sql } from "drizzle-orm";
+import { accumulateEmbeddedTags } from "../../cache/embedded-tags.js";
 import { eq } from "../../db/index.js";
+import { isConditionHidden } from "../../plugin/fields/condition.js";
+import { anchorTemporalUtc } from "../../plugin/manifest.js";
 import { MetaReferenceError } from "./errors.js";
+import { META_FIELD_MESSAGES } from "./field-messages.js";
+import { extractStringId, healReferenceValue, isGroupField, isRepeaterField, referenceTargetOf, runFieldPipeline, } from "./field-pipeline.js";
 // Shared meta plumbing for every entity that stores a `meta` JSON
 // column (entries, terms, and — eventually — users). The storage
 // contract: writes merge into the bag via SQLite `json_set` /
@@ -32,70 +37,100 @@ export class MetaSanitizationError extends Error {
         return new MetaSanitizationError(ctx.key, "value_too_large");
     }
 }
+/**
+ * The whole-patch rejection produced by `sanitizeMetaInput` when any
+ * field's pipeline reports errors: every `{ path, message }` across
+ * every key of the request, so the admin form can address each
+ * offending input in one round-trip. Nothing is written when this
+ * throws.
+ */
+export class MetaValidationError extends Error {
+    static {
+        MetaValidationError.prototype.name = "MetaValidationError";
+    }
+    errors;
+    constructor(errors) {
+        super(`meta validation failed: ${errors.map((error) => error.path).join(", ")}`);
+        this.errors = errors;
+    }
+}
 /**
  * Validate an incoming meta map against a field-lookup fn produced by
  * the caller (entry vs term differ only in which registry they walk).
- * Unregistered keys and type-coercion failures throw so the caller can
- * surface them as 4xx before any write. `null` / `undefined` values
- * are deletion requests; everything else is coerced + sanitized.
+ * `null` / `undefined` values are deletion requests; everything else
+ * runs the per-field pipeline (coercion → `.sanitize()` → declarative
+ * constraints → `.validate()`). Pipeline rejections aggregate across
+ * the whole patch into one `MetaValidationError`; unregistered keys
+ * and oversized values keep the legacy fail-fast
+ * `MetaSanitizationError` surface.
  */
-export function sanitizeMetaInput(findField, input) {
+export async function sanitizeMetaInput(findField, input, mode = "strict") {
     if (input === undefined)
         return null;
     const upserts = new Map();
     const deletes = [];
+    const fieldErrors = [];
     for (const [key, rawValue] of Object.entries(input)) {
         const field = findField(key);
         if (!field) {
+            // A delete request (null/undefined) for a key the field system doesn't
+            // own is a harmless no-op — never fail the whole write over an untracked
+            // foreign key (e.g. one written by another plugin). Unknown *upserts*
+            // still reject so junk can't enter through this gate.
+            if (rawValue === null || rawValue === undefined)
+                continue;
             throw MetaSanitizationError.notRegistered({ key });
         }
-        if (rawValue === null || rawValue === undefined) {
+        if (isConditionHidden(field, input))
+            continue;
+        const result = await runFieldPipeline(field, rawValue, key, mode);
+        if (result.errors.length > 0) {
+            fieldErrors.push(...result.errors);
+            continue;
+        }
+        if (result.isDeletion === true) {
             deletes.push(key);
             continue;
         }
-        const coerced = coerceToType(field.type, key, rawValue);
-        const sanitized = field.sanitize
-            ? runSanitize(field.sanitize, key, coerced)
-            : coerced;
-        assertEncodedSize(key, sanitized);
-        upserts.set(key, sanitized);
+        assertEncodedSize(key, result.value);
+        upserts.set(key, result.value);
+    }
+    if (fieldErrors.length > 0) {
+        throw new MetaValidationError(fieldErrors);
     }
     return { upserts, deletes };
 }
 /**
- * Run a field's sanitize callback inside a try/catch — any thrown
- * `MetaSanitizationError` propagates as-is (so callbacks that already
- * use the precise error type can opt into specific reasons), but
- * generic errors (including plain `Error("invalid_value")` thrown
- * from builder-injected default sanitizers) are translated into a
- * uniform `invalid_value` failure. Callbacks therefore stay free to
- * throw vanilla errors without importing the package-internal error
- * class.
+ * Thin wrapper that translates thrown meta errors into the RPC
+ * handler's CONFLICT envelope. Pipeline rejections ship their
+ * `{ path, message }` list under `data.errors` (with `key` pointing
+ * at the first error's top-level field for legacy consumers); the
+ * fail-fast `MetaSanitizationError` reasons keep their existing
+ * `data.reason`/`data.key` shape.
  */
-function runSanitize(sanitize, key, value) {
-    try {
-        return sanitize(value);
-    }
-    catch (error) {
-        if (error instanceof MetaSanitizationError)
-            throw error;
-        // Buggy sanitize callbacks otherwise round to a generic
-        // `invalid_value` envelope, which is fine for the editor's UX
-        // but loses the underlying stack. Log it before translating so
-        // server logs preserve the diagnostic trail.
-        console.error(`[plumix] sanitize callback for meta key ${JSON.stringify(key)} threw:`, error);
-        throw MetaSanitizationError.invalidValue({ key });
-    }
-}
 /**
- * Thin wrapper that translates a thrown `MetaSanitizationError` into
- * the RPC handler's CONFLICT envelope.
+ * Map a whole-patch meta validation failure onto the RPC `CONFLICT`
+ * envelope — one place so the wire shape (`key` at the first error's
+ * top-level field, the full `errors` list) can't drift between the write
+ * path and the publish gate.
  */
-export function sanitizeMetaForRpc(findField, input, errors) {
+export function metaValidationConflict(error, errors) {
+    return errors.CONFLICT({
+        data: {
+            reason: "meta_invalid_value",
+            key: error.errors[0]?.path.split(".")[0],
+            errors: [...error.errors],
+        },
+    });
+}
+export async function sanitizeMetaForRpc(findField, input, errors, mode = "strict") {
     try {
-        return sanitizeMetaInput(findField, input);
+        return await sanitizeMetaInput(findField, input, mode);
     }
     catch (error) {
+        if (error instanceof MetaValidationError) {
+            throw metaValidationConflict(error, errors);
+        }
         if (error instanceof MetaSanitizationError) {
             throw errors.CONFLICT({
                 data: { reason: `meta_${error.reason}`, key: error.key },
@@ -104,6 +139,58 @@ export function sanitizeMetaForRpc(findField, input, errors) {
         throw error;
     }
 }
+/**
+ * Strict publish gate: validate a whole *stored* meta bag against the full
+ * field list before `entry.publish` promotes it onto the live row. Draft
+ * autosaves are lenient (business rules skipped so work-in-progress always
+ * saves), so publish is where required fields, bounds, formats, option
+ * membership, and row counts are finally enforced. It walks the field
+ * *definitions*, not just the bag, so a required field ABSENT from the bag
+ * is caught as well as one stored empty — `runFieldPipeline` sees
+ * `undefined` and rejects it in strict mode.
+ *
+ * Rejections aggregate across the bag into one `MetaValidationError` the
+ * RPC layer maps onto the admin inputs, blocking the publish. A passing bag
+ * comes back canonicalized (each registered value re-run through
+ * `.sanitize()`); a conditionally-hidden field can't be required, so it's
+ * skipped; keys the field system doesn't own (e.g. from an uninstalled
+ * plugin) pass through untouched, matching the read path. Field
+ * capabilities and reference existence are not re-checked here — a
+ * whole-bag gate would block a publisher over a co-author's field, and both
+ * were already enforced at write time.
+ */
+export async function validateAndPromoteMetaBag(fields, bag) {
+    const out = {};
+    const owned = new Set();
+    const fieldErrors = [];
+    for (const field of fields) {
+        owned.add(field.key);
+        // A hidden field is inactive, so it can't be required — but its stored
+        // value is kept untouched (not validated, not dropped), or the value a
+        // driver hides would be lost on publish and gone when the driver flips
+        // back.
+        if (isConditionHidden(field, bag)) {
+            if (field.key in bag)
+                out[field.key] = bag[field.key];
+            continue;
+        }
+        const result = await runFieldPipeline(field, bag[field.key], field.key);
+        if (result.errors.length > 0) {
+            fieldErrors.push(...result.errors);
+            continue;
+        }
+        if (result.isDeletion === true)
+            continue;
+        out[field.key] = result.value;
+    }
+    for (const [key, value] of Object.entries(bag)) {
+        if (!owned.has(key))
+            out[key] = value;
+    }
+    if (fieldErrors.length > 0)
+        throw new MetaValidationError(fieldErrors);
+    return out;
+}
 /**
  * Walk a sanitized patch, group every reference upsert by
  * `(kind, scope)`, and issue one `LookupAdapter.list({ ids })`
@@ -118,7 +205,7 @@ export function sanitizeMetaForRpc(findField, input, errors) {
  * TOCTOU note: validate runs in a separate query from the eventual
  * `applyMetaPatch`, and callers don't share a transaction. A
  * concurrent delete between validate and apply leaves an orphan id
- * in the meta bag; `filterMetaOrphans` masks it on read. Wrap the
+ * in the meta bag; `hydrateMetaBags` masks it on read. Wrap the
  * validate/apply pair in `ctx.db.transaction()` if a caller needs
  * serializable consistency.
  */
@@ -133,58 +220,41 @@ export async function validateMetaReferences(ctx, findField, patch) {
                 throw MetaSanitizationError.invalidValue({ key });
             }
             const ids = referenceIdsForValidation(key, value, target, fieldMax(field));
+            // Normalize eagerly — a validation failure below aborts the whole
+            // save, so a rewritten patch never persists on the failure path.
+            patch.upserts.set(key, target.multiple ? ids : ids[0]);
             const group = upsertGroup(groups, target, registered);
             for (const id of ids)
                 group.ids.add(id);
-            group.contributions.push({
-                errorKey: key,
-                ids,
-                applyNormalize: (rowsById) => {
-                    if (target.valueShape !== "object")
-                        return;
-                    if (target.multiple) {
-                        patch.upserts.set(key, ids.map((id) => buildCachedReference(id, rowsById)));
-                        return;
-                    }
-                    const [singleId] = ids;
-                    if (singleId !== undefined) {
-                        patch.upserts.set(key, buildCachedReference(singleId, rowsById));
-                    }
-                },
-            });
+            group.contributions.push({ errorKey: key, ids });
             continue;
         }
-        // Repeater fields don't carry a `referenceTarget` themselves but
-        // their rows can. Walk into rows so nested `entry` / `term` /
-        // `user` / `media` refs flow through the same `(kind, scope)`
-        // batch as top-level fields. Errors attribute to the top-level
-        // repeater key — the row index + subKey live in the developer
-        // log, not the wire response (per slice acceptance).
-        if (!isRepeaterField(field))
-            continue;
-        if (!Array.isArray(value))
-            continue;
-        collectRepeaterReferences(ctx, key, field, value, groups);
+        // Composite fields (repeater / group) don't carry a
+        // `referenceTarget` themselves but their rows / members can, at any
+        // nesting depth. Walk them so nested `entry` / `term` / `user` /
+        // `media` refs flow through the same `(kind, scope)` batch as
+        // top-level fields. Errors attribute to the top-level key — the
+        // dotted sub-path lives in the developer log, not the wire response.
+        if (isRepeaterField(field) || isGroupField(field)) {
+            collectCompositeReferences(ctx, key, key, field, value, groups);
+        }
     }
     for (const group of groups.values()) {
         if (group.ids.size === 0)
             continue;
-        const liveRows = await fetchLiveRows(ctx, group.registered, group.scope, group.ids, "validateMetaReferences");
-        const rowsById = new Map(liveRows.map((r) => [r.id, r]));
+        const liveIds = await fetchLiveIds(ctx, group.registered, group.scope, group.ids, "validateMetaReferences");
         for (const contribution of group.contributions) {
             for (const id of contribution.ids) {
-                if (!rowsById.has(id)) {
+                if (!liveIds.has(id)) {
                     if (contribution.diagnostic !== undefined) {
                         // Wire error keys on the top-level field; this log line
-                        // is the only place the row index + subKey surface, so
-                        // an engineer debugging a `meta_invalid_value` on a
-                        // repeater can locate the offending subField without
-                        // bisecting the saved bag. `JSON.stringify(id)` escapes
-                        // control characters so a user-supplied id with newlines
-                        // can't poison the log stream.
-                        console.error(`[plumix] meta repeater ${JSON.stringify(contribution.errorKey)} ` +
-                            `row ${String(contribution.diagnostic.rowIdx)} ` +
-                            `subField ${JSON.stringify(contribution.diagnostic.subKey)} ` +
+                        // is the only place the nested sub-path surfaces, so an
+                        // engineer debugging a `meta_invalid_value` on a composite
+                        // field can locate the offending cell without bisecting the
+                        // saved bag. `JSON.stringify` escapes control characters so
+                        // a user-supplied id with newlines can't poison the log.
+                        console.error(`[plumix] meta composite ${JSON.stringify(contribution.errorKey)} ` +
+                            `at ${JSON.stringify(contribution.diagnostic.path)} ` +
                             `references missing id ${JSON.stringify(id)}`);
                     }
                     throw MetaSanitizationError.invalidValue({
@@ -192,61 +262,60 @@ export async function validateMetaReferences(ctx, findField, patch) {
                     });
                 }
             }
-            contribution.applyNormalize(rowsById);
         }
     }
 }
-function isRepeaterField(field) {
-    if (!field)
-        return false;
-    return field.inputType === "repeater";
-}
-function collectRepeaterReferences(ctx, topKey, field, rows, groups) {
-    // Nested repeaters are rejected at registration by `repeater()`, so
-    // any subField that's itself a repeater wouldn't get here in
-    // practice. We don't recurse into subField repeaters either way —
-    // single-source-of-truth on the registration guard.
-    for (const [rowIdx, row] of rows.entries()) {
-        if (!row || typeof row !== "object" || Array.isArray(row))
-            continue;
-        const rowObj = row;
-        for (const subField of field.subFields) {
-            const target = referenceTargetOf(subField);
-            if (!target)
+// Walk a composite field's live value, collecting + normalizing every
+// nested reference in place. `path` is the dotted address of `field`
+// from the top-level key (developer-log only). Recurses through nested
+// repeaters and groups, so a reference at any depth flows through the
+// same `(kind, scope)` batch and is orphan-checked at write time.
+function collectCompositeReferences(ctx, topKey, path, field, value, groups) {
+    if (isRepeaterField(field)) {
+        if (!Array.isArray(value))
+            return;
+        for (const [rowIdx, row] of value.entries()) {
+            if (!isPlainObject(row))
                 continue;
-            const subValue = rowObj[subField.key];
+            collectMemberReferences(ctx, topKey, `${path}.${String(rowIdx)}`, field.subFields, row, groups);
+        }
+        return;
+    }
+    if (isGroupField(field)) {
+        if (!isPlainObject(value))
+            return;
+        collectMemberReferences(ctx, topKey, path, field.fields, value, groups);
+    }
+}
+function collectMemberReferences(ctx, topKey, path, members, container, groups) {
+    for (const member of members) {
+        const memberPath = `${path}.${member.key}`;
+        const subValue = container[member.key];
+        const target = referenceTargetOf(member);
+        if (target) {
             if (subValue === undefined || subValue === null)
                 continue;
             const registered = ctx.plugins.lookupAdapters.get(target.kind);
             if (!registered) {
                 throw MetaSanitizationError.invalidValue({ key: topKey });
             }
-            const ids = referenceIdsForValidation(topKey, subValue, target, fieldMax(subField));
+            const ids = referenceIdsForValidation(topKey, subValue, target, fieldMax(member));
+            // Normalize the cell in place — the top-level upsert value is the
+            // live storage shape, so mutating an object inside it is what the
+            // caller serializes.
+            container[member.key] = target.multiple ? ids : ids[0];
             const group = upsertGroup(groups, target, registered);
             for (const id of ids)
                 group.ids.add(id);
-            const subKey = subField.key;
             group.contributions.push({
                 errorKey: topKey,
                 ids,
-                diagnostic: { rowIdx, subKey },
-                applyNormalize: (rowsById) => {
-                    if (target.valueShape !== "object")
-                        return;
-                    // Rewrite the row's cached-object value in place — the
-                    // top-level upsert array is the live storage shape, so
-                    // mutating an object inside it is what the caller will
-                    // serialize.
-                    if (target.multiple) {
-                        rowObj[subKey] = ids.map((id) => buildCachedReference(id, rowsById));
-                        return;
-                    }
-                    const [singleId] = ids;
-                    if (singleId !== undefined) {
-                        rowObj[subKey] = buildCachedReference(singleId, rowsById);
-                    }
-                },
+                diagnostic: { path: memberPath },
             });
+            continue;
+        }
+        if (isRepeaterField(member) || isGroupField(member)) {
+            collectCompositeReferences(ctx, topKey, memberPath, member, subValue, groups);
         }
     }
 }
@@ -264,13 +333,6 @@ function upsertGroup(groups, target, registered) {
     }
     return group;
 }
-// Adapter cached fields override anything; user-supplied non-id keys
-// don't survive. Per-usage user fields (e.g. alt overrides) are a
-// post-v0.1 feature — when added, this is where they'd merge in.
-function buildCachedReference(id, rowsById) {
-    const row = rowsById.get(id);
-    return { id, ...(row?.cached ?? {}) };
-}
 // Defense-in-depth ceiling on the internal-aggregated id-batch size.
 // Above this, the live-id fetch throws rather than silently truncating.
 // The wire cap (100) and per-field cap (`HARD_MULTI_REFERENCE_LIMIT`,
@@ -305,20 +367,17 @@ function referenceGroupKey(target) {
 // 100, so we only hit the ceiling when many fields share `(kind, scope)`
 // and aggregate — at which point throwing beats silent truncation
 // (truncation would either reject valid writes or hide live targets).
-//
-// Returns `LookupResult[]` rather than just live ids so the validator
-// can read `cached` for the cached-object normalize step. Callers that
-// only care about presence build a `Set` from the row ids.
-async function fetchLiveRows(ctx, registered, scope, ids, callsite) {
+async function fetchLiveIds(ctx, registered, scope, ids, callsite) {
     if (ids.size > MAX_REFERENCE_GROUP_BATCH) {
         throw MetaReferenceError.batchSizeExceeded(callsite, ids.size, MAX_REFERENCE_GROUP_BATCH);
     }
     const idList = [...ids];
-    return registered.adapter.list(ctx, {
+    const rows = await registered.adapter.list(ctx, {
         ids: idList,
         scope,
         limit: idList.length,
     });
+    return new Set(rows.map((row) => row.id));
 }
 // Defensive upper bound on multi-reference array length, applied
 // per-field before grouping. Even with the batched `list({ ids })`
@@ -328,18 +387,11 @@ async function fetchLiveRows(ctx, registered, scope, ids, callsite) {
 // can declare a lower `max` and the validator picks the smaller.
 const HARD_MULTI_REFERENCE_LIMIT = 100;
 // Validates the wire shape of a reference value and returns the ids
-// to feed into the group's batched `list({ ids })` call. Four shapes
-// are accepted, dispatching on `target.multiple` + `target.valueShape`:
-//
-//  - multi=true,  valueShape="id"  (default): string[] of non-empty ids
-//  - multi=true,  valueShape="object":        `{ id }[]` — each item a
-//                                             cached-object reference
-//                                             (or bare strings for
-//                                             leniency on first write)
-//  - multi=false, valueShape="id"  (default): bare string id
-//  - multi=false, valueShape="object":        `{ id: string, ... }`
-//                                             (or a bare string for
-//                                             leniency on first write)
+// to feed into the group's batched `list({ ids })` call. Storage is
+// plain ids — a bare string (single) or string[] (multi) — but each
+// slot leniently accepts the retired cached-object shape
+// (`{ id, ... }`) so legacy values self-heal to the plain form on
+// the entity's next save.
 function referenceIdsForValidation(key, value, target, max) {
     if (target.multiple) {
         if (!Array.isArray(value)) {
@@ -351,56 +403,17 @@ function referenceIdsForValidation(key, value, target, max) {
         if (max !== undefined && value.length > max) {
             throw MetaSanitizationError.invalidValue({ key });
         }
-        if (target.valueShape === "object") {
-            // Lenient on input: each item can be `"42"` (first-write
-            // convenience) or `{ id: "42", ... }`. Always rewritten to the
-            // canonical `{ id, ...cached }` shape after the live-id check.
-            const ids = [];
-            for (const item of value) {
-                if (typeof item === "string" && item !== "") {
-                    ids.push(item);
-                    continue;
-                }
-                const id = extractStringId(item);
-                if (id !== null && id !== "") {
-                    ids.push(id);
-                    continue;
-                }
-                throw MetaSanitizationError.invalidValue({ key });
-            }
-            return ids;
-        }
-        for (const item of value) {
-            if (typeof item !== "string" || item === "") {
-                throw MetaSanitizationError.invalidValue({ key });
-            }
-        }
-        return value;
+        return value.map((item) => referenceItemId(key, item));
     }
-    if (target.valueShape === "object") {
-        // Lenient on input: accept either `"42"` (first-write convenience)
-        // or `{ id: "42", ... }`. Always rewritten to the canonical
-        // `{ id, ...cached }` shape after the live-id check.
-        if (typeof value === "string" && value !== "")
-            return [value];
-        const id = extractStringId(value);
-        if (id !== null && id !== "")
-            return [id];
-        throw MetaSanitizationError.invalidValue({ key });
-    }
-    if (typeof value !== "string") {
-        throw MetaSanitizationError.invalidValue({ key });
-    }
-    return [value];
-}
-// Returns the `id` string of a `{ id: string, ... }` object, or null
-// for any other shape (string, array, null, primitive, missing key).
-function extractStringId(value) {
-    if (typeof value !== "object" || value === null || Array.isArray(value)) {
-        return null;
-    }
-    const id = value.id;
-    return typeof id === "string" ? id : null;
+    return [referenceItemId(key, value)];
+}
+function referenceItemId(key, item) {
+    if (typeof item === "string" && item !== "")
+        return item;
+    const id = extractStringId(item);
+    if (id !== null && id !== "")
+        return id;
+    throw MetaSanitizationError.invalidValue({ key });
 }
 function fieldMax(field) {
     return field?.max;
@@ -418,201 +431,318 @@ export async function validateMetaReferencesForRpc(ctx, findField, patch, errors
     catch (error) {
         if (error instanceof MetaSanitizationError) {
             throw errors.CONFLICT({
-                data: { reason: `meta_${error.reason}`, key: error.key },
+                data: {
+                    reason: `meta_${error.reason}`,
+                    key: error.key,
+                    // Reference failures address the top-level field (row/subKey
+                    // detail stays in the server log) — shipping them under
+                    // `errors` too lets the admin form surface them inline.
+                    errors: [
+                        { path: error.key, message: META_FIELD_MESSAGES.invalidOption },
+                    ],
+                },
             });
         }
         throw error;
     }
 }
+// A reference field authored with `.returns("id")` reads the bare
+// stored id — the hydration walk skips it so no lookup query runs and
+// the id survives untouched. `"returns" in field` narrows to the field
+// variants that carry the flag (temporal's `"date"` never matches).
+function readsRawReferenceId(field) {
+    return field !== undefined && "returns" in field && field.returns === "id";
+}
 /**
  * Yield every reference-field occurrence in a decoded meta bag: top-level
- * reference fields, plus reference subFields inside each repeater row. One
- * structural walk so the orphan-strip pass doesn't reinline the
- * top-level/repeater traversal twice.
+ * reference fields, plus references nested inside repeater rows and
+ * groups at any depth. One structural walk so the hydration pass doesn't
+ * reinline the traversal twice.
  */
 function* referenceOccurrences(entries, findField) {
     for (const [key, value] of entries) {
-        const field = findField(key);
-        const target = referenceTargetOf(field);
-        if (target) {
-            yield { key, target, value };
-            continue;
-        }
-        if (!isRepeaterField(field))
-            continue;
+        yield* fieldOccurrences([key], findField(key), value);
+    }
+}
+// Recurse a single field's decoded value, yielding reference occurrences
+// with their full path. Nested field definitions come straight off the
+// composite field (`subFields` / `fields`) — `findField` only resolves
+// top-level keys.
+function* fieldOccurrences(path, field, value) {
+    const target = referenceTargetOf(field);
+    if (target) {
+        // `.returns("id")` opts out of the hydration join at any depth —
+        // leave the stored id(s) untouched (no resolve, no orphan-strip).
+        if (!readsRawReferenceId(field))
+            yield { path, target, value };
+        return;
+    }
+    if (isRepeaterField(field)) {
         if (!Array.isArray(value))
-            continue;
+            return;
         for (const [rowIdx, row] of value.entries()) {
-            if (!row || typeof row !== "object" || Array.isArray(row))
+            if (!isPlainObject(row))
                 continue;
-            const rowObj = row;
             for (const subField of field.subFields) {
-                const subTarget = referenceTargetOf(subField);
-                if (!subTarget)
-                    continue;
-                yield {
-                    key,
-                    target: subTarget,
-                    value: rowObj[subField.key],
-                    nested: { rowIdx, subKey: subField.key },
-                };
+                yield* fieldOccurrences([...path, rowIdx, subField.key], subField, row[subField.key]);
             }
         }
+        return;
+    }
+    if (isGroupField(field)) {
+        if (!isPlainObject(value))
+            return;
+        for (const member of field.fields) {
+            yield* fieldOccurrences([...path, member.key], member, value[member.key]);
+        }
     }
 }
-export async function filterMetaOrphans(ctx, findField, decoded) {
+/** Single-bag convenience over {@link hydrateMetaBags}. */
+export async function hydrateMetaReferences(ctx, findField, decoded) {
+    const [bag] = await hydrateMetaBags(ctx, [{ findField, decoded }]);
+    // hydrateMetaBags returns one bag per input by construction.
+    return bag ?? decoded;
+}
+export async function hydrateMetaBags(ctx, bags) {
     const candidates = [];
     const groups = new Map();
-    // Top-level refs and repeater-row subField refs get identical orphan-strip
-    // treatment, so the single `referenceOccurrences` walk feeds both. Nested
-    // candidates carry their `rowIdx`/`subKey`; Pass 3 rewrites the row slot in
-    // a per-key clone so the caller's `decoded` bag stays untouched.
-    for (const occ of referenceOccurrences(Object.entries(decoded), findField)) {
-        const registered = ctx.plugins.lookupAdapters.get(occ.target.kind);
-        if (!registered)
-            continue;
-        const ids = orphanCandidateIds(occ.target, occ.value);
-        if (ids === null)
-            continue; // non-array multi / non-string single — leave untouched
-        const groupKey = referenceGroupKey(occ.target);
-        candidates.push({
-            key: occ.key,
-            multiple: occ.target.multiple === true,
-            valueShape: occ.target.valueShape ?? "id",
-            groupKey,
-            ids,
-            nested: occ.nested,
-        });
-        if (ids.length === 0)
-            continue;
-        let group = groups.get(groupKey);
-        if (!group) {
-            group = { registered, scope: occ.target.scope, ids: new Set() };
-            groups.set(groupKey, group);
-        }
-        for (const id of ids)
-            group.ids.add(id);
-    }
-    // Pass 2: one `list({ ids })` per group, keyed for O(1) apply lookup.
-    const liveIdsByGroup = new Map();
-    for (const [groupKey, group] of groups) {
-        const rows = await fetchLiveRows(ctx, group.registered, group.scope, group.ids, "filterMetaOrphans");
-        liveIdsByGroup.set(groupKey, new Set(rows.map((row) => row.id)));
-    }
-    // Pass 3: apply the filter. Orphan handling differs by shape:
-    //  - multi + id: filter the string[] in place, dropping missing
-    //  - multi + object: filter the object[] preserving order
-    //  - single + id: null on missing
-    //  - single + object: null on missing; otherwise keep the stored
-    //    cached value as-is (no read-time refresh — write rewrites)
-    // Nested-in-repeater candidates rewrite the row's subField slot in
-    // a per-key cloned array so the input bag stays untouched.
-    const out = { ...decoded };
-    const empty = new Set();
-    for (const candidate of candidates) {
-        const { key, multiple, valueShape, groupKey, ids, nested } = candidate;
-        const liveIds = liveIdsByGroup.get(groupKey) ?? empty;
-        if (nested !== undefined) {
-            const rowObj = takeWritableRow(out, decoded, key, nested.rowIdx);
-            if (!rowObj)
+    // Top-level refs and nested (repeater / group) refs get identical
+    // treatment, so the single `referenceOccurrences` walk feeds both.
+    // Every candidate carries its full path; Pass 3 rewrites the slot in a
+    // per-path copy-on-write so callers' bags stay untouched.
+    const out = [];
+    for (const bag of bags) {
+        const outBag = { ...bag.decoded };
+        out.push(outBag);
+        for (const occ of referenceOccurrences(Object.entries(bag.decoded), bag.findField)) {
+            const registered = ctx.plugins.lookupAdapters.get(occ.target.kind);
+            if (!registered)
                 continue;
-            applyOrphanToSlot(rowObj, nested.subKey, multiple, valueShape, ids, liveIds);
-            continue;
-        }
-        if (multiple) {
-            if (valueShape === "object") {
-                const stored = decoded[key];
-                out[key] = stored.filter((item) => {
-                    const id = extractStringId(item);
-                    return id !== null && liveIds.has(id);
-                });
+            const ids = referenceCandidateIds(occ.target, occ.value);
+            if (ids === null)
+                continue; // non-array multi / non-string single — leave untouched
+            const groupKey = referenceGroupKey(occ.target);
+            candidates.push({
+                outBag,
+                decoded: bag.decoded,
+                path: occ.path,
+                multiple: occ.target.multiple === true,
+                groupKey,
+                ids,
+            });
+            if (ids.length === 0)
                 continue;
+            let group = groups.get(groupKey);
+            if (!group) {
+                group = { registered, scope: occ.target.scope, ids: new Set() };
+                groups.set(groupKey, group);
             }
-            out[key] = ids.filter((id) => liveIds.has(id));
-            continue;
+            for (const id of ids)
+                group.ids.add(id);
         }
-        const [singleId] = ids;
-        if (singleId === undefined)
-            continue; // single non-string already filtered upstream
-        if (!liveIds.has(singleId))
-            out[key] = null;
+    }
+    // Pass 2: one `hydrate({ ids })` (or `list({ ids })` fallback) per
+    // group, keyed for O(1) apply lookup. Groups are independent, so a
+    // bag mixing entry + user + media refs resolves them concurrently.
+    const resolutions = new Map(await Promise.all([...groups].map(async ([groupKey, group]) => [
+        groupKey,
+        await resolveGroup(ctx, group.registered.adapter, group.scope, group.ids),
+    ])));
+    // Pass 3: apply. Hydrated groups replace ids with their payloads;
+    // id-only groups keep ids. Multi refs stay dense and in stored
+    // order; single refs null on missing. Nested candidates copy-on-write
+    // each container down their path so callers' bags stay untouched.
+    const emptyResolution = { kind: "ids", liveIds: new Set() };
+    for (const candidate of candidates) {
+        const { outBag, decoded, path, multiple, groupKey, ids } = candidate;
+        const resolution = resolutions.get(groupKey) ?? emptyResolution;
+        const slot = takeWritableSlot(outBag, decoded, path);
+        if (!slot)
+            continue;
+        applyResolutionToSlot(slot.parent, slot.leafKey, multiple, ids, resolution);
     }
     return out;
 }
-function takeWritableRow(out, decoded, key, rowIdx) {
-    // Lazily clone the rows array (and the targeted row inside it) on
-    // first nested write so the caller's `decoded` bag stays untouched.
-    // Subsequent nested writes hit the same cloned array.
-    let arr = out[key];
-    if (arr === decoded[key]) {
-        if (!Array.isArray(arr))
-            return null;
-        const cloned = arr.map((row) => row && typeof row === "object" && !Array.isArray(row)
-            ? { ...row }
-            : row);
-        out[key] = cloned;
-        arr = cloned;
+/**
+ * Batched, tag-accounted hydration of a raw id set for one reference kind
+ * — the theme-facing counterpart to the meta pipeline's hydration (#1508).
+ * A theme holding an id-only reference field (a field declared
+ * `.returns("id")`, or one contributed by a third-party plugin) resolves
+ * it here instead of hand-rolled per-item fetches: ids resolve through the
+ * adapter's batched `hydrate` (chunked, one in-query per chunk) and every
+ * resolved entity folds its cache tag into the page through the same
+ * accumulator the meta pipeline uses, so the page is purged when an
+ * embedded entity changes.
+ *
+ * Returns the hydrated payloads dense and in the requested id order — ids
+ * that are gone or out of scope are dropped, mirroring multi-reference
+ * field hydration. An unregistered kind, an adapter without the `hydrate`
+ * contract, or an empty id set yields `[]`.
+ */
+export async function hydrateReferences(ctx, kind, ids, options = {}) {
+    const registered = ctx.plugins.lookupAdapters.get(kind);
+    if (!registered?.adapter.hydrate)
+        return [];
+    const unique = new Set(ids.filter((id) => id !== ""));
+    if (unique.size === 0)
+        return [];
+    const resolution = await resolveGroup(ctx, registered.adapter, options.scope, unique);
+    if (resolution.kind !== "hydrated")
+        return [];
+    return ids
+        .map((id) => resolution.byId.get(id))
+        .filter((p) => p !== undefined);
+}
+// Resolve one `(kind, scope)` group's aggregated ids. Chunked at
+// `HYDRATION_QUERY_ID_LIMIT` per in-query: a response-level group can
+// legitimately aggregate more ids than one query may carry (a
+// 100-entry archive × multi-reference fields), and a read-path throw
+// would kill the render — unlike the write-side `fetchLiveIds`, which
+// keeps throwing because a single patch exceeding the ceiling is a
+// caller bug. Ids are de-duped before chunking, so per-query batches
+// stay bounded and nothing is truncated.
+async function resolveGroup(ctx, adapter, scope, ids) {
+    const idList = [...ids];
+    if (adapter.hydrate) {
+        const byId = new Map();
+        for (const chunk of chunkIds(idList)) {
+            const payloads = await adapter.hydrate(ctx, { ids: chunk, scope });
+            for (const payload of payloads) {
+                byId.set(payload.id, payload);
+                // Fold this embedded entity's cache tag into the page's tags so
+                // a change to it purges the page that hydrated it (#1508). Runs
+                // on every read surface; only the public read-through reads the
+                // accumulator back, so admin/REST reads populate it harmlessly.
+                if (adapter.embeddedCacheTags) {
+                    accumulateEmbeddedTags(ctx, adapter.embeddedCacheTags(payload));
+                }
+            }
+        }
+        return { kind: "hydrated", byId };
+    }
+    const liveIds = new Set();
+    for (const chunk of chunkIds(idList)) {
+        const rows = await adapter.list(ctx, {
+            ids: chunk,
+            scope,
+            limit: chunk.length,
+        });
+        for (const row of rows)
+            liveIds.add(row.id);
+    }
+    return { kind: "ids", liveIds };
+}
+// Per-query id cap for the read path. 100 (not the 1000 aggregate
+// ceiling) because Cloudflare D1 caps bound parameters at 100 per
+// statement and `inArray` binds one per id — a bigger chunk works on
+// local SQLite and dies in production.
+const HYDRATION_QUERY_ID_LIMIT = 100;
+function* chunkIds(ids) {
+    for (let i = 0; i < ids.length; i += HYDRATION_QUERY_ID_LIMIT) {
+        yield ids.slice(i, i + HYDRATION_QUERY_ID_LIMIT);
     }
-    if (!Array.isArray(arr))
-        return null;
-    const row = arr[rowIdx];
-    if (!row || typeof row !== "object" || Array.isArray(row))
-        return null;
-    return row;
 }
-function applyOrphanToSlot(rowObj, subKey, multiple, valueShape, ids, liveIds) {
+// Write one candidate's resolved value into its slot — the leaf object
+// key of a container reached by walking the candidate's path.
+function applyResolutionToSlot(slot, key, multiple, ids, resolution) {
     if (multiple) {
-        if (valueShape === "object") {
-            const stored = rowObj[subKey];
-            rowObj[subKey] = stored.filter((item) => {
-                const id = extractStringId(item);
-                return id !== null && liveIds.has(id);
-            });
-            return;
-        }
-        rowObj[subKey] = ids.filter((id) => liveIds.has(id));
+        slot[key] =
+            resolution.kind === "hydrated"
+                ? ids
+                    .map((id) => resolution.byId.get(id))
+                    .filter((payload) => payload !== undefined)
+                : ids.filter((id) => resolution.liveIds.has(id));
         return;
     }
     const [singleId] = ids;
     if (singleId === undefined)
+        return; // single non-string already filtered upstream
+    if (resolution.kind === "hydrated") {
+        slot[key] = resolution.byId.get(singleId) ?? null;
+        return;
+    }
+    if (!resolution.liveIds.has(singleId))
+        slot[key] = null;
+}
+// Walk the candidate's path from the output bag to the parent container
+// of its leaf slot, copy-on-writing each container the first time it's
+// descended so the caller's `decoded` bag stays untouched. Containers
+// cloned by an earlier candidate (identity no longer matches `decoded`)
+// are reused, so sibling references in the same row/group land in one
+// clone. Returns the writable parent object and the leaf key, or null if
+// any segment's runtime shape doesn't match the declared structure
+// (hand-edited / migrated bags).
+function takeWritableSlot(outBag, decoded, path) {
+    let outContainer = outBag;
+    let decContainer = decoded;
+    for (let i = 0; i < path.length - 1; i++) {
+        const seg = path[i];
+        if (seg === undefined)
+            return null;
+        const outChild = readSegment(outContainer, seg);
+        const decChild = readSegment(decContainer, seg);
+        let writable = outChild;
+        // Still the shared decoded child → clone it into the output tree.
+        if (outChild === decChild) {
+            const clone = cloneContainer(outChild);
+            if (clone === null)
+                return null;
+            writeSegment(outContainer, seg, clone);
+            writable = clone;
+        }
+        outContainer = writable;
+        decContainer = decChild;
+    }
+    const leaf = path[path.length - 1];
+    if (typeof leaf !== "string")
+        return null;
+    if (!isPlainObject(outContainer))
+        return null;
+    return { parent: outContainer, leafKey: leaf };
+}
+function readSegment(container, seg) {
+    if (typeof seg === "number") {
+        return Array.isArray(container) ? container[seg] : undefined;
+    }
+    return isPlainObject(container) ? container[seg] : undefined;
+}
+function writeSegment(container, seg, value) {
+    if (typeof seg === "number") {
+        if (Array.isArray(container))
+            container[seg] = value;
         return;
-    if (!liveIds.has(singleId))
-        rowObj[subKey] = null;
+    }
+    if (isPlainObject(container))
+        container[seg] = value;
+}
+// Shallow clone an array or plain object; null for any other shape (the
+// path expected a container but the stored value isn't one).
+function cloneContainer(value) {
+    // `Array.isArray` widens to `any[]`; cast before spread so the clone
+    // stays `unknown[]` rather than leaking `any` into the walk.
+    if (Array.isArray(value))
+        return [...value];
+    if (isPlainObject(value))
+        return { ...value };
+    return null;
+}
+function isPlainObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 // Returns `null` to mean "no candidates from this key" (skip), or
 // the ids to feed into the group's batch query. Mirrors the storage-
 // shape guards in the apply step so we don't enqueue work that's
-// going to be skipped. Cached-object values (`valueShape === "object"`)
-// extract `id` from the object — the rest of the cached fields are
-// trusted on read (refresh happens on next write via the validator's
-// normalize step, not on every render).
-function orphanCandidateIds(target, value) {
+// going to be skipped. Callers pass values already decoded through
+// `decodeMetaBag`, so legacy object shapes have been healed to plain
+// ids by the time this runs.
+function referenceCandidateIds(target, value) {
     if (target.multiple) {
         if (!Array.isArray(value))
             return null;
-        if (target.valueShape === "object") {
-            const ids = [];
-            for (const item of value) {
-                const id = extractStringId(item);
-                if (id !== null)
-                    ids.push(id);
-            }
-            return ids;
-        }
         return value.filter((id) => typeof id === "string");
     }
-    if (target.valueShape === "object") {
-        const id = extractStringId(value);
-        return id !== null ? [id] : null;
-    }
     return typeof value === "string" ? [value] : null;
 }
-function referenceTargetOf(field) {
-    if (!field)
-        return undefined;
-    return field
-        .referenceTarget;
-}
 /**
  * Decode a raw meta bag (as returned by drizzle's JSON-mode column)
  * into the plugin-typed shape RPC consumers expect. Unregistered keys
@@ -626,10 +756,64 @@ export function decodeMetaBag(findField, raw) {
     const out = {};
     for (const [key, value] of Object.entries(raw)) {
         const field = findField(key);
-        out[key] = field ? coerceOnRead(field.type, value) : value;
+        out[key] = field ? decodeFieldValue(field, value) : value;
     }
     return out;
 }
+// Reference storage is plain ids, but bags written before the
+// write-time snapshot machinery was removed may hold `{ id, ... }`
+// objects. Reads yield the id; the next save persists the plain form
+// (`referenceItemId` accepts the legacy shape on write).
+function decodeFieldValue(field, value) {
+    const target = referenceTargetOf(field);
+    if (target)
+        return healReferenceValue(target, value);
+    if (isRepeaterField(field) && Array.isArray(value)) {
+        return value.map((row) => healRepeaterRow(field, row));
+    }
+    if (isTemporalField(field) && field.returns === "date") {
+        return projectTemporalDate(field.inputType, value);
+    }
+    return coerceOnRead(field.type, value);
+}
+function isTemporalField(field) {
+    return (field.inputType === "date" ||
+        field.inputType === "datetime" ||
+        field.inputType === "time");
+}
+// `.returns("date")` decode projection. All three variants anchor to
+// UTC — `date` at UTC midnight, `time` on 1970-01-01 UTC — so the
+// wall-clock components survive every server/browser timezone
+// combination (decode runs server-side, often UTC on Workers, while
+// the admin formats in the viewer's browser). Consumers read the
+// parts back with `getUTC*` or `timeZone: "UTC"` formatting; the
+// projection is the exact inverse of the write-side `Date` encoding
+// (`formatTemporalValue`). Unparseable stored values round to "no
+// value", matching the forgiving-read posture of `coerceOnRead`.
+function projectTemporalDate(inputType, value) {
+    if (typeof value !== "string" || value === "")
+        return undefined;
+    const parsed = new Date(anchorTemporalUtc(inputType, value));
+    return Number.isNaN(parsed.getTime()) ? undefined : parsed;
+}
+function healRepeaterRow(field, row) {
+    if (!row || typeof row !== "object" || Array.isArray(row))
+        return row;
+    const rowObj = row;
+    let healed = null;
+    for (const subField of field.subFields) {
+        const target = referenceTargetOf(subField);
+        if (!target)
+            continue;
+        const value = rowObj[subField.key];
+        const next = healReferenceValue(target, value);
+        if (next !== value) {
+            healed ??= { ...rowObj };
+            healed[subField.key] = next;
+        }
+    }
+    return healed ?? row;
+}
 export function isEmptyMetaPatch(patch) {
     return (patch === null || (patch.upserts.size === 0 && patch.deletes.length === 0));
 }
@@ -674,72 +858,6 @@ export async function loadMeta(ctx, table, idColumn, id, findField) {
     return decodeMetaBag(findField, row?.meta);
 }
 // --- internals below ---------------------------------------------------
-function coerceToType(type, key, value) {
-    switch (type) {
-        case "string":
-            return coerceString(key, value);
-        case "number":
-            return coerceNumber(key, value);
-        case "boolean":
-            return coerceBoolean(key, value);
-        case "json":
-            return coerceJson(key, value);
-    }
-}
-function coerceString(key, value) {
-    if (typeof value === "string")
-        return value;
-    if (typeof value === "number" && Number.isFinite(value))
-        return String(value);
-    if (typeof value === "boolean")
-        return String(value);
-    throw MetaSanitizationError.invalidValue({ key });
-}
-function coerceNumber(key, value) {
-    if (typeof value === "number" && Number.isFinite(value))
-        return value;
-    if (typeof value === "string") {
-        // Empty string comes from cleared form inputs; the admin dispatcher
-        // already sends `null` for those, but a direct RPC caller might send
-        // "" — reject here so we don't silently coerce to 0 (`Number("") === 0`).
-        if (value.trim() === "") {
-            throw MetaSanitizationError.invalidValue({ key });
-        }
-        const parsed = Number(value);
-        if (Number.isFinite(parsed))
-            return parsed;
-    }
-    if (typeof value === "boolean")
-        return value ? 1 : 0;
-    throw MetaSanitizationError.invalidValue({ key });
-}
-function coerceBoolean(key, value) {
-    if (typeof value === "boolean")
-        return value;
-    if (value === 1 || value === "1" || value === "true")
-        return true;
-    if (value === 0 || value === "0" || value === "false")
-        return false;
-    throw MetaSanitizationError.invalidValue({ key });
-}
-function coerceJson(key, value) {
-    // json keys take anything round-trippable through JSON.stringify —
-    // we reject values that throw (BigInt) or silently drop (functions,
-    // Symbols) so reads don't hand back `undefined` for something a
-    // plugin thought it stored. TS types JSON.stringify as always-
-    // string, but at runtime it returns `undefined` for unserializable
-    // inputs — hence the cast.
-    try {
-        const encoded = JSON.stringify(value);
-        if (encoded === undefined) {
-            throw MetaSanitizationError.invalidValue({ key });
-        }
-        return JSON.parse(encoded);
-    }
-    catch {
-        throw MetaSanitizationError.invalidValue({ key });
-    }
-}
 function assertEncodedSize(key, value) {
     const encoded = JSON.stringify(value);
     if (encoded === undefined)