@mohasinac/appkit 4.52.5 → 4.53.1

package/dist/_internal/server/features/search/index-build/build.js

@@ -0,0 +1,285 @@
+/*
+ * WHY: Turn the public collections into one small artifact the browser can
+ *      search locally, WITHOUT the read cost that makes the obvious version
+ *      impossible.
+ * WHAT: Full and incremental builds, plus the stats a caller must log.
+ *
+ * ## 🛑 THE READ BUDGET IS THE DESIGN. Read this before changing the cadence.
+ *
+ * Firestore bills **per document read regardless of `select()`** — projection
+ * cuts egress, not read count. So the naive version, a full rebuild every 15
+ * minutes, is `96 × 3,730 ≈ 358,000 reads/day` against a **50,000/day** free
+ * tier. Seven times over, for a feature nobody asked to be expensive. Hourly
+ * is still ~89,000.
+ *
+ * Hence two modes:
+ *
+ *   idle        manifest says not dirty               1 read       ~90/day
+ *   incremental `updatedAt > lastBuiltAt` per source  13 + changed ~200/day
+ *   full        everything; reconciles hard deletes   3,730        3,730/day
+ *
+ * ≈ 4,000 reads/day, about 8% of the tier.
+ *
+ * 🛑 THE INCREMENTAL CLAUSE NEEDS A COMPOSITE INDEX PER SOURCE, and an earlier
+ * version of this header claimed the opposite.
+ *
+ * A LONE range clause on `updatedAt` needs nothing — Firestore auto-creates
+ * single-field ascending indexes. But every source that also carries a
+ * `where` equality (`status == published`, `categoryType == brand`,
+ * `isActive == true`) combines equality + range, and that DOES require a
+ * composite. Only `category`, `part` and `model` — the three with no `where`
+ * — are index-free.
+ *
+ * Found by running the job against production, not by reading it: the FULL
+ * build has no range clause and passed cleanly, so nothing surfaced until the
+ * first INCREMENTAL pass returned `9 FAILED_PRECONDITION` for brand, store,
+ * blog and every other filtered source. Exactly the Root Cause #59 shape the
+ * old comment claimed immunity from.
+ *
+ * The indexes are declared in `appkit/firebase/base/firestore.indexes.json`.
+ * Adding a source with a `where` means adding its `(field, updatedAt)` pair
+ * there too, or that source silently stops updating incrementally.
+ *
+ * ## Unpublish vs hard delete — different mechanisms, say which is which
+ *
+ * **Unpublish is handled incrementally.** A product flipped `published ->
+ * draft` still has a fresh `updatedAt`, so the builder re-reads it, finds it
+ * fails the source's filter, and REMOVES it from the artifact within the
+ * cadence. That is the case that matters, because it is the one that happens.
+ *
+ * **A hard delete is invisible to an `updatedAt` query** and is reconciled by
+ * the daily full pass. Worst case is a dead link for up to 24 h, which is link
+ * rot rather than disclosure. Tombstone machinery would be real complexity for
+ * the rarer half of a split that can be described in two sentences.
+ *
+ * @tag domain:search
+ * @tag layer:server
+ * @tag pattern:none
+ * @tag access:server
+ * @tag consumers:searchIndexBuild
+ * @tag sideEffects:firestore-read
+ */
+import { normalizeError } from "../../../../../errors/normalize.js";
+import { serverLogger } from "../../../../../monitoring/server-logger.js";
+import { hidePublicTestData } from "../../tester/visibility.js";
+import { INDEX_TYPES, PUBLIC_INDEX_SOURCES, typeIndexFor, } from "./sources.js";
+import { INDEX_UPDATED_AT_FIELD, SEARCH_INDEX_SCHEMA_VERSION, } from "../../../../shared/search/index-record.js";
+/**
+ * The patch key, derived from the source's own `slugOf` so the REMOVE path and
+ * the ADD path cannot disagree — and so re-keying a previous artifact (which
+ * has only the slug) lands on the same string.
+ */
+function typeKeyFor(source, d) {
+    return source.typeKeyOf ? source.typeKeyOf(d) : source.typeKey;
+}
+function keyOf(source, id, d) {
+    const slug = source.slugOf(id, d);
+    return slug ? `${typeKeyFor(source, d)}:${slug}` : null;
+}
+async function readSource(db, source, since) {
+    let q = db.collection(source.collection);
+    for (const [field, value] of source.where ?? [])
+        q = q.where(field, "==", value);
+    /*
+     * 🛑 The incremental clause is applied INSTEAD OF nothing, not in addition
+     * to a page limit. An unbounded `updatedAt >` on a quiet collection returns
+     * a handful of rows; on a reseed it returns everything, which is correct —
+     * a reseed genuinely did change everything.
+     */
+    if (since)
+        q = q.where(INDEX_UPDATED_AT_FIELD, ">", since);
+    const snap = await q.select(...source.select, "isTestData", INDEX_UPDATED_AT_FIELD).get();
+    /*
+     * Firestore bills a minimum of one read for a query that matches nothing,
+     * so an idle incremental pass still costs one per source. Counting it is
+     * the difference between a budget and a guess.
+     */
+    return { docs: snap.docs, reads: Math.max(1, snap.docs.length) };
+}
+function toRecords(source, docs) {
+    const kept = [];
+    const removed = [];
+    let dropped = 0;
+    /*
+     * 🛑 `hidePublicTestData` is NOT optional here, and this call site differs
+     * from every other one. Elsewhere a sandbox row that slips through is shown
+     * to one viewer on one render. Here it is baked into a shared, immutable,
+     * CDN-cached body and published to everyone for the life of that version.
+     */
+    /*
+     * 🛑 An INTERSECTION, not a cast. `IndexDocData` is `Record<string,
+     * unknown>`, whose index signature makes `isTestData` an `unknown` and so
+     * fails `TestDataFlagged`. The lazy fix is `as any`, which would also
+     * silence the fact that `hidePublicTestData` is generic and preserves the
+     * row type — including `__id`, which the loop below needs. Naming the shape
+     * keeps both ends type-checked.
+     */
+    const rows = docs.map((d) => ({ ...d.data(), __id: d.id }));
+    const visible = hidePublicTestData(rows);
+    for (const row of visible) {
+        const id = row.__id;
+        const key = keyOf(source, id, row);
+        // No derivable slug means no href and no stable identity — such a row
+        // cannot be a search result at all, so it is dropped rather than keyed
+        // under something invented.
+        if (!key) {
+            dropped++;
+            continue;
+        }
+        const rec = source.toRecord(id, row);
+        if (!rec) {
+            dropped++;
+            // A document that fails `toRecord` on an INCREMENTAL pass is usually a
+            // row that just became non-public — so it must be removed, not ignored.
+            removed.push(key);
+            continue;
+        }
+        /*
+         * 🛑 Resolved per DOCUMENT, not per source. `products` holds all nine
+         * listing types and each has its own detail route, so a single type index
+         * for the collection would send every auction to /products/{slug} —
+         * Root Cause #48, rebuilt.
+         */
+        const typeKey = typeKeyFor(source, row);
+        const typeIdx = typeIndexFor(typeKey);
+        if (typeIdx < 0) {
+            dropped++;
+            continue;
+        }
+        const slug = key.slice(typeKey.length + 1);
+        const tuple = rec.facets === undefined
+            ? [typeIdx, rec.title, rec.detail, slug]
+            : [typeIdx, rec.title, rec.detail, slug, rec.facets];
+        kept.push([key, tuple]);
+    }
+    return { kept, removed, dropped };
+}
+function envelope(version, keyed, builtAt) {
+    return {
+        schema: SEARCH_INDEX_SCHEMA_VERSION,
+        version,
+        builtAt: builtAt.toISOString(),
+        types: INDEX_TYPES,
+        // Sorted by key so two builds over identical data produce identical bytes,
+        // which is what makes "did anything actually change" answerable.
+        r: Object.keys(keyed)
+            .sort()
+            .map((k) => keyed[k]),
+    };
+}
+/**
+ * Build from scratch. Reconciles hard deletes by construction, because the
+ * result contains exactly what the sources currently return.
+ */
+export async function buildFull(db, version, now = new Date()) {
+    const started = Date.now();
+    const keyed = {};
+    const perType = {};
+    let reads = 0;
+    let dropped = 0;
+    for (const source of PUBLIC_INDEX_SOURCES) {
+        try {
+            const { docs, reads: r } = await readSource(db, source, null);
+            reads += r;
+            const out = toRecords(source, docs);
+            for (const [k, rec] of out.kept) {
+                keyed[k] = rec;
+                const tk = INDEX_TYPES[rec[0]]?.key ?? source.typeKey;
+                perType[tk] = (perType[tk] ?? 0) + 1;
+            }
+            dropped += out.dropped;
+        }
+        catch (err) {
+            const e = normalizeError(err);
+            /*
+             * 🛑 A failing source does NOT silently shrink the artifact. A full
+             * build that quietly omits `products` would publish a search index
+             * missing a third of the site and report success — the exact shape of
+             * every silent-degradation bug this codebase keeps recording.
+             */
+            serverLogger.error("searchIndexBuild: source failed during FULL build — aborting", {
+                source: source.typeKey,
+                collection: source.collection,
+                error: e.message,
+            });
+            throw err;
+        }
+    }
+    return {
+        artifact: envelope(version, keyed, now),
+        stats: {
+            mode: "full",
+            reads,
+            perType,
+            dropped,
+            failed: 0,
+            durationMs: Date.now() - started,
+        },
+    };
+}
+/**
+ * Patch the previous artifact with everything that changed since it was built.
+ *
+ * 🛑 A source that fails here is logged and SKIPPED, not fatal — unlike the
+ * full build. The difference is what the failure costs: a skipped incremental
+ * source leaves that type's records at their previous (correct, slightly
+ * stale) values and the next pass picks them up, whereas a skipped full source
+ * would delete the type from the artifact entirely.
+ */
+export async function buildIncremental(db, previous, since, version, now = new Date()) {
+    const started = Date.now();
+    // Re-key the previous artifact. The tuple carries no id, so the key is
+    // rebuilt from type + slug — which is why `slug` must be stable per record.
+    const keyed = {};
+    for (const rec of previous.r) {
+        const key = keyFromRecord(rec);
+        if (key)
+            keyed[key] = rec;
+    }
+    const perType = {};
+    let reads = 0;
+    let dropped = 0;
+    let failed = 0;
+    for (const source of PUBLIC_INDEX_SOURCES) {
+        try {
+            const { docs, reads: r } = await readSource(db, source, since);
+            reads += r;
+            const out = toRecords(source, docs);
+            for (const k of out.removed)
+                delete keyed[k];
+            for (const [k, rec] of out.kept)
+                keyed[k] = rec;
+            perType[source.typeKey] = out.kept.length;
+            dropped += out.dropped;
+        }
+        catch (err) {
+            failed++;
+            const e = normalizeError(err);
+            serverLogger.warn("searchIndexBuild: source failed during INCREMENTAL build — skipped", {
+                source: source.typeKey,
+                collection: source.collection,
+                error: e.message,
+            });
+        }
+    }
+    return {
+        artifact: envelope(version, keyed, now),
+        stats: {
+            mode: "incremental",
+            reads,
+            perType,
+            dropped,
+            failed,
+            durationMs: Date.now() - started,
+        },
+    };
+}
+/**
+ * Re-key a record from a previous artifact. Mirrors `keyOf` — both sides key
+ * on `${typeKey}:${slug}`, which is the invariant that makes an incremental
+ * patch REPLACE a changed row rather than add a second copy of it.
+ */
+export function keyFromRecord(rec) {
+    const type = INDEX_TYPES[rec[0]];
+    return type ? `${type.key}:${rec[3]}` : null;
+}

package/dist/_internal/server/features/search/index-build/sources.d.ts

@@ -0,0 +1,53 @@
+import type { SearchIndexType } from "../../../../shared/search/index-record";
+import type { FirestoreValue } from "../../../../../schemas/types";
+export type IndexDocData = Record<string, FirestoreValue>;
+/** One record under construction. `null` drops the document from the index. */
+export interface BuiltRecord {
+    readonly title: string;
+    readonly detail: string;
+    readonly facets?: string;
+}
+export interface IndexSource {
+    /** Matches a `SearchIndexType.key`. */
+    readonly typeKey: string;
+    readonly collection: string;
+    /** Fields to project. A builder reads documents, not whole collections. */
+    readonly select: readonly string[];
+    /**
+     * Extra equality narrowing applied as a query clause. Only use this for a
+     * value every public row genuinely carries — otherwise filter in `toRecord`,
+     * because a Firestore equality on an absent field matches nothing and an
+     * inequality excludes every document lacking it.
+     */
+    readonly where?: ReadonlyArray<readonly [string, unknown]>;
+    /**
+     * The record's stable identity, and the artifact's patch key.
+     *
+     * 🛑 SEPARATE from `toRecord` on purpose. An incremental build must be able
+     * to key a document it is REMOVING — one that just became unpublished — and
+     * at that moment `toRecord` has returned null and cannot supply a slug. With
+     * the slug folded into `toRecord` the two paths keyed differently (doc id on
+     * the way out, slug on the way in), which agrees only where slug === id: so
+     * every changed product would have been ADDED as a duplicate rather than
+     * replaced, while categories and stores looked fine.
+     */
+    /**
+     * Resolve the type PER DOCUMENT, when one collection holds several.
+     * `products` is the only such collection today: all nine listing types live
+     * in it and each has its own detail route.
+     */
+    readonly typeKeyOf?: (d: IndexDocData) => string;
+    readonly slugOf: (id: string, d: IndexDocData) => string | null;
+    readonly toRecord: (id: string, d: IndexDocData) => BuiltRecord | null;
+}
+/**
+ * The type table shipped inside the artifact.
+ *
+ * 🛑 No `label`. The client maps `key` to a translated string; a heading baked
+ * in here could not be localised. Weights are a ranking nudge between types
+ * that scored equally, not a statement about importance.
+ */
+export declare const INDEX_TYPES: readonly SearchIndexType[];
+export declare const PUBLIC_INDEX_SOURCES: readonly IndexSource[];
+/** Index into `INDEX_TYPES` for a source's type key. */
+export declare function typeIndexFor(typeKey: string): number;