cursedbelt-server 1.0.0

package/dist/server/search/pgSearchProvider.d.ts

@@ -0,0 +1,91 @@
+/**
+ * `createPgSearchProvider` — the SAAS profile's ranked-retrieval tier (04 §3,
+ * audit gap L11 and frontier item 4(c)).
+ *
+ * Before this file, the saas profile had **no search tier at all**. `bootEngineApp`
+ * derived its `SearchProvider` from the local SQLite handles it does not open
+ * when a host injects storage, so under saas `search` was `undefined`, the
+ * `reindex` job fell through to the loud "needs a wired SearchProvider"
+ * placeholder, and `KnowledgeService.search` took its `LIKE` fallback for every
+ * tenant on the platform. 04 §1's gate text — "a leased `reindex` executing
+ * through the pg-dialect repo" — was unachievable, and the `genAppSaasSmoke`
+ * job leg asserted that the placeholder failed, as designed. This is the far
+ * side that makes both of those real.
+ *
+ * ## What is shared with the local provider, and what is not
+ *
+ * **The vector tier is shared, exactly.** Chunking, the little-endian embedding
+ * codec, the revision guard, the comparability predicates, the cosine rank and
+ * the reciprocal-rank fusion all come from `cwip`'s `searchVectorTier` — one
+ * implementation, driven here through a Postgres executor and there through a
+ * SQLite one. A second copy of ranking math is the worst kind of duplication:
+ * both copies keep returning plausible results after they diverge.
+ *
+ * **The keyword tier is NOT shared, and cannot be.** SQLite's is an FTS5
+ * external-content virtual table maintained by triggers; this one is a
+ * `tsvector` generated column with a GIN index. `domains/search.ts` already
+ * refuses to compile its chain for Postgres and says why. What IS held identical
+ * is the OBSERVABLE contract: the same 4× title weighting (`PG_RANK_WEIGHTS`
+ * against `bm25(search_fts, 4.0, 1.0)`), the same "higher score is better" sign,
+ * the same tenancy-as-a-predicate law, and the same behavior on a user typing
+ * punctuation into a search box. Two profiles that ranked one corpus differently
+ * would make "search ranks wrong in production" unreproducible on a laptop.
+ *
+ * ## Two things Postgres makes better, and one it makes worse
+ *
+ * **Better: a search box is not a query language, and here it is not a hazard
+ * either.** The SQLite provider catches an FTS5 syntax error and retries the
+ * whole input as a quoted phrase literal. `websearch_to_tsquery` cannot raise a
+ * syntax error on any input — quotes, `-`, `OR`, an unbalanced paren are all
+ * defined — so the retry has nothing to catch and does not exist. That is a real
+ * difference in the failure surface, not a stylistic one.
+ *
+ * **Better: the shadow cannot drift.** `tsv` is `GENERATED ALWAYS AS … STORED`,
+ * so no code path writes it — see `pgSearchSchema.ts`.
+ *
+ * **Worse: the connection accounting.** Every operation checks a connection out
+ * of the ONE pool, opens a transaction, scopes it to the tenant and returns it.
+ * The vector tier's chunk writes go through a per-statement scoped executor, so
+ * writing an eight-chunk document is eight checkouts rather than one. That is
+ * deliberate: the alternative is holding a pooled connection and an open
+ * transaction across eight embedding round-trips, which is how a search backlog
+ * takes an entire deployment's Postgres down. The chunk write is already
+ * dominated by inference latency; the checkout is noise beside it.
+ *
+ * ## The ceiling, again stated rather than hidden
+ *
+ * The vector rank is an exact cosine scan over a bounded candidate set, not an
+ * ANN index — no `pgvector`, no extension a customer's managed Postgres may
+ * refuse to install. `cwip`'s `searchVectorTier` header carries the numbers and
+ * the upgrade path; truncation is REPORTED through `onVectorTruncated` rather
+ * than swallowed, because a search tier that quietly degrades as a customer
+ * succeeds is worse than one that says so.
+ */
+import { type RebuildableSearchProvider } from 'cwip/search-provider';
+import type { ChunkOptions } from 'cwip/search';
+import { type SaasPool } from 'cwip/storage';
+export interface PgSearchProviderOptions {
+    /** The deployment's ONE pool. Injected — this module opens no socket (law #8). */
+    pool: SaasPool;
+    defaultLimit?: number;
+    maxLimit?: number;
+    /**
+     * The embedder backing the vector tier. Absent ⇒ the provider has no `embed`
+     * method at all and `search_chunks` stays honestly empty — the same capability
+     * signal the local provider gives, so a caller's `provider.embed?.(…)` test
+     * means the same thing on both profiles.
+     */
+    embed?(text: string, tenantId: string): Promise<Float32Array>;
+    /** Which model `embed` speaks for. Recorded per chunk; a query predicate. */
+    embedModel?: string;
+    chunkOptions?: ChunkOptions;
+    vectorCandidateLimit?: number;
+    onVectorTruncated?(detail: {
+        tenantId: string;
+        corpora: readonly string[];
+        scanned: number;
+    }): void;
+    /** Where a vector-tier failure goes. The keyword tier is the floor. */
+    onVectorError?(error: unknown, phase: 'index' | 'query'): void;
+}
+export declare function createPgSearchProvider(options: PgSearchProviderOptions): RebuildableSearchProvider;

package/dist/server/search/pgSearchProvider.js

@@ -0,0 +1,330 @@
+/**
+ * `createPgSearchProvider` — the SAAS profile's ranked-retrieval tier (04 §3,
+ * audit gap L11 and frontier item 4(c)).
+ *
+ * Before this file, the saas profile had **no search tier at all**. `bootEngineApp`
+ * derived its `SearchProvider` from the local SQLite handles it does not open
+ * when a host injects storage, so under saas `search` was `undefined`, the
+ * `reindex` job fell through to the loud "needs a wired SearchProvider"
+ * placeholder, and `KnowledgeService.search` took its `LIKE` fallback for every
+ * tenant on the platform. 04 §1's gate text — "a leased `reindex` executing
+ * through the pg-dialect repo" — was unachievable, and the `genAppSaasSmoke`
+ * job leg asserted that the placeholder failed, as designed. This is the far
+ * side that makes both of those real.
+ *
+ * ## What is shared with the local provider, and what is not
+ *
+ * **The vector tier is shared, exactly.** Chunking, the little-endian embedding
+ * codec, the revision guard, the comparability predicates, the cosine rank and
+ * the reciprocal-rank fusion all come from `cwip`'s `searchVectorTier` — one
+ * implementation, driven here through a Postgres executor and there through a
+ * SQLite one. A second copy of ranking math is the worst kind of duplication:
+ * both copies keep returning plausible results after they diverge.
+ *
+ * **The keyword tier is NOT shared, and cannot be.** SQLite's is an FTS5
+ * external-content virtual table maintained by triggers; this one is a
+ * `tsvector` generated column with a GIN index. `domains/search.ts` already
+ * refuses to compile its chain for Postgres and says why. What IS held identical
+ * is the OBSERVABLE contract: the same 4× title weighting (`PG_RANK_WEIGHTS`
+ * against `bm25(search_fts, 4.0, 1.0)`), the same "higher score is better" sign,
+ * the same tenancy-as-a-predicate law, and the same behavior on a user typing
+ * punctuation into a search box. Two profiles that ranked one corpus differently
+ * would make "search ranks wrong in production" unreproducible on a laptop.
+ *
+ * ## Two things Postgres makes better, and one it makes worse
+ *
+ * **Better: a search box is not a query language, and here it is not a hazard
+ * either.** The SQLite provider catches an FTS5 syntax error and retries the
+ * whole input as a quoted phrase literal. `websearch_to_tsquery` cannot raise a
+ * syntax error on any input — quotes, `-`, `OR`, an unbalanced paren are all
+ * defined — so the retry has nothing to catch and does not exist. That is a real
+ * difference in the failure surface, not a stylistic one.
+ *
+ * **Better: the shadow cannot drift.** `tsv` is `GENERATED ALWAYS AS … STORED`,
+ * so no code path writes it — see `pgSearchSchema.ts`.
+ *
+ * **Worse: the connection accounting.** Every operation checks a connection out
+ * of the ONE pool, opens a transaction, scopes it to the tenant and returns it.
+ * The vector tier's chunk writes go through a per-statement scoped executor, so
+ * writing an eight-chunk document is eight checkouts rather than one. That is
+ * deliberate: the alternative is holding a pooled connection and an open
+ * transaction across eight embedding round-trips, which is how a search backlog
+ * takes an entire deployment's Postgres down. The chunk write is already
+ * dominated by inference latency; the checkout is noise beside it.
+ *
+ * ## The ceiling, again stated rather than hidden
+ *
+ * The vector rank is an exact cosine scan over a bounded candidate set, not an
+ * ANN index — no `pgvector`, no extension a customer's managed Postgres may
+ * refuse to install. `cwip`'s `searchVectorTier` header carries the numbers and
+ * the upgrade path; truncation is REPORTED through `onVectorTruncated` rather
+ * than swallowed, because a search tier that quietly degrades as a customer
+ * succeeds is worse than one that says so.
+ */
+import { assertNonEmpty, clampLimit, clampPositive, countChunks, docKey, fuseSearchTiers, rankChunksByVector, UNKNOWN_EMBED_MODEL, writeDocChunks, } from 'cwip/search-provider';
+import { createPostgresControlExecutor, saasSetTenantSql, } from 'cwip/storage';
+import { AppError } from '../errors';
+import { PG_RANK_WEIGHTS } from './pgSearchSchema';
+const DEFAULT_LIMIT = 20;
+const MAX_LIMIT = 200;
+/** Mirrors the SQLite provider's fusion depth — see its `FUSION_DEPTH`. */
+const FUSION_DEPTH = 4;
+/** `ts_headline` window, in words. Comparable to the SQLite side's 14-token snippet. */
+const SNIPPET_WORDS = 18;
+export function createPgSearchProvider(options) {
+    const { pool } = options;
+    const defaultLimit = clampPositive(options.defaultLimit, DEFAULT_LIMIT);
+    const maxLimit = clampPositive(options.maxLimit, MAX_LIMIT);
+    const embedModel = options.embedModel ?? UNKNOWN_EMBED_MODEL;
+    /**
+     * Run `fn` on a connection scoped to `tenantId` for the duration of ONE
+     * transaction.
+     *
+     * The `BEGIN` is not decoration around a single statement: `set_config(…, true)`
+     * is transaction-local, and outside a transaction it would apply to the
+     * session and ride the connection back into the pool with another tenant's
+     * scope still set. That is the exact leak RLS exists to prevent, so the
+     * transaction is what makes the scoping safe rather than what makes it atomic.
+     */
+    const scoped = async (tenantId, fn) => {
+        assertNonEmpty(tenantId, 'tenantId');
+        return await pool.withConnection(async (connection) => {
+            await connection.execute('BEGIN');
+            try {
+                const setTenant = saasSetTenantSql(tenantId);
+                await connection.execute(setTenant.sql, setTenant.params);
+                const result = await fn(createPostgresControlExecutor(connection));
+                await connection.execute('COMMIT');
+                return result;
+            }
+            catch (error) {
+                // Best-effort: a connection already broken by the failure cannot roll
+                // back, and masking the original error with the rollback's would hide
+                // the thing worth reading.
+                await connection.execute('ROLLBACK').catch(() => undefined);
+                throw error;
+            }
+        });
+    };
+    /**
+     * An executor whose every statement is its own scoped transaction.
+     *
+     * This is what the vector tier runs on, and the shape is the point: the tier
+     * interleaves database writes with embedding round-trips, so a single
+     * long-lived transaction across it would pin a pooled connection for the
+     * duration of somebody else's inference queue. Each statement being separately
+     * atomic is also exactly what the tier's revision guard assumes — the guard is
+     * a `WHERE EXISTS` evaluated at write time, never a read-then-act.
+     */
+    const perStatementExecutor = (tenantId) => ({
+        dialect: 'postgres',
+        all: (statement) => scoped(tenantId, (exec) => exec.all(statement)),
+        get: (statement) => scoped(tenantId, (exec) => exec.get(statement)),
+        write: (statement) => scoped(tenantId, (exec) => exec.write(statement)),
+    });
+    const keywordQuery = (query, tenantId, corpora, limit) => {
+        const where = ['d.tenant_id = ?', "d.tsv @@ websearch_to_tsquery('simple', ?)"];
+        const params = [query, query, tenantId, query];
+        if (corpora.length > 0) {
+            where.push(`d.corpus IN (${corpora.map(() => '?').join(', ')})`);
+            params.push(...corpora);
+        }
+        params.push(limit);
+        return {
+            // `ts_headline` gets the same empty start/stop markers the SQLite
+            // `snippet()` call uses, so a client rendering a hit does not have to know
+            // which profile produced it.
+            sql: `SELECT d.source_id AS id,
+                   d.corpus    AS corpus,
+                   d.tenant_id AS tenant_id,
+                   d.title     AS title,
+                   ts_headline('simple', d.body, websearch_to_tsquery('simple', ?),
+                               'MaxWords=${SNIPPET_WORDS}, MinWords=1, ShortWord=1, HighlightAll=false, MaxFragments=1, StartSel="", StopSel=""') AS snippet,
+                   ts_rank('${PG_RANK_WEIGHTS}'::float4[], d.tsv, websearch_to_tsquery('simple', ?)) AS rank
+            FROM search_docs d
+            WHERE ${where.join(' AND ')}
+            ORDER BY rank DESC, d.id DESC
+            LIMIT ?`,
+            params,
+        };
+    };
+    return {
+        async now() {
+            // The INDEX's clock, in the index's own format — the same law the SQLite
+            // provider states. Comparing a JS `Date` against a value the database
+            // wrote is a format coupling nobody remembers is there.
+            const row = await pool.withConnection(async (connection) => createPostgresControlExecutor(connection).get({ sql: 'SELECT {{now}} AS now' }));
+            if (row === null)
+                throw new AppError('The search index clock returned no row.', {
+                    code: 'SEARCH_CLOCK_UNAVAILABLE',
+                    status: 500,
+                });
+            return row.now;
+        },
+        ...(options.embed ? { embed: options.embed } : {}),
+        async index(doc) {
+            assertNonEmpty(doc.corpus, 'corpus');
+            assertNonEmpty(doc.id, 'id');
+            assertNonEmpty(doc.tenantId, 'tenantId');
+            // The document lands first and alone — the keyword tier is the floor, and
+            // an embedding round-trip is not something to hold a write lock across.
+            const identity = await scoped(doc.tenantId, async (executor) => {
+                const row = await executor.get({
+                    sql: `INSERT INTO search_docs (corpus, source_id, tenant_id, user_id, title, body, rev, indexed_at)
+                VALUES (?, ?, ?, ?, ?, ?, 0, {{now}})
+                ON CONFLICT (corpus, source_id, tenant_id) DO UPDATE SET
+                  user_id    = excluded.user_id,
+                  title      = excluded.title,
+                  body       = excluded.body,
+                  rev        = search_docs.rev + 1,
+                  indexed_at = excluded.indexed_at
+                RETURNING id, rev`,
+                    params: [doc.corpus, doc.id, doc.tenantId, null, doc.title, doc.body],
+                });
+                if (row === null)
+                    throw new AppError('The search index upsert returned no row, so the document id is unknown.', {
+                        code: 'SEARCH_UPSERT_EMPTY',
+                        status: 500,
+                        context: { corpus: doc.corpus, id: doc.id },
+                    });
+                return row;
+            });
+            if (!options.embed)
+                return;
+            const embed = options.embed;
+            try {
+                await writeDocChunks(perStatementExecutor(doc.tenantId), {
+                    // A pg `bigint` arrives as a string; it is an opaque key here either way.
+                    docId: String(identity.id),
+                    rev: Number(identity.rev),
+                    text: doc.title ? `${doc.title}\n\n${doc.body}` : doc.body,
+                    tenantId: doc.tenantId,
+                    embed: (text, tenantId) => embed(text, tenantId),
+                    model: embedModel,
+                    ...(options.chunkOptions ? { chunkOptions: options.chunkOptions } : {}),
+                });
+            }
+            catch (error) {
+                // Degrade, do not fail — see the SQLite provider's identical rule. The
+                // document IS indexed; failing here would make an embedder outage look
+                // like a write failure to the CDC tap, which retries.
+                options.onVectorError?.(error, 'index');
+            }
+        },
+        async remove(corpus, id, tenantId) {
+            // ONE transaction for both tables: an interrupted removal that dropped the
+            // document but kept its chunks would leave the vector tier ranking a
+            // document that no longer exists. (The FK cascade would handle it, but the
+            // explicit delete keeps the two profiles' behavior identical rather than
+            // resting on a schema detail one of them does not have.)
+            await scoped(tenantId, async (executor) => {
+                await executor.write({
+                    sql: `DELETE FROM search_chunks WHERE doc_id IN (
+                  SELECT id FROM search_docs WHERE corpus = ? AND source_id = ? AND tenant_id = ?)`,
+                    params: [corpus, id, tenantId],
+                });
+                await executor.write({
+                    sql: 'DELETE FROM search_docs WHERE corpus = ? AND source_id = ? AND tenant_id = ?',
+                    params: [corpus, id, tenantId],
+                });
+            });
+        },
+        async query(q, opts) {
+            assertNonEmpty(opts.tenantId, 'tenantId');
+            const trimmed = q.trim();
+            if (trimmed.length === 0)
+                return [];
+            const corpora = opts.corpus ? [...opts.corpus] : [];
+            const limit = clampLimit(opts.limit, defaultLimit, maxLimit);
+            const tierLimit = options.embed ? Math.min(maxLimit, limit * FUSION_DEPTH) : limit;
+            const rows = await scoped(opts.tenantId, (executor) => executor.all(keywordQuery(trimmed, opts.tenantId, corpora, tierLimit)));
+            const keywordHits = rows.map((row) => ({
+                id: row.id,
+                corpus: row.corpus,
+                tenantId: row.tenant_id,
+                // `ts_rank` is already higher-is-better, which is the interface's
+                // contract — so unlike BM25 there is no sign to flip. Said explicitly
+                // because "flip it, the other one does" is the mistake available here.
+                score: Number(row.rank),
+                title: row.title,
+                snippet: row.snippet ?? '',
+            }));
+            if (!options.embed)
+                return keywordHits.slice(0, limit);
+            let vector;
+            try {
+                vector = await rankChunksByVector(perStatementExecutor(opts.tenantId), {
+                    tenantId: opts.tenantId,
+                    corpora,
+                    queryVector: await options.embed(trimmed, opts.tenantId),
+                    model: embedModel,
+                    limit: tierLimit,
+                    ...(options.vectorCandidateLimit !== undefined
+                        ? { candidateLimit: options.vectorCandidateLimit }
+                        : {}),
+                });
+            }
+            catch (error) {
+                options.onVectorError?.(error, 'query');
+                return keywordHits.slice(0, limit);
+            }
+            if (vector.truncated)
+                options.onVectorTruncated?.({ tenantId: opts.tenantId, corpora, scanned: vector.scanned });
+            return await fuseSearchTiers({
+                keyword: keywordHits,
+                vector: vector.chunks,
+                limit,
+                tenantId: opts.tenantId,
+                hydrate: async (keys) => {
+                    // ONE statement for every missing title. A per-document lookup here is
+                    // an N+1 on the hot path, and each of those N is a pool checkout.
+                    const placeholders = keys.map(() => '(?, ?)').join(', ');
+                    const params = keys.flatMap((key) => [key.corpus, key.docId]);
+                    const found = await scoped(opts.tenantId, (executor) => executor.all({
+                        sql: `SELECT corpus, source_id, title FROM search_docs
+                    WHERE tenant_id = ? AND (corpus, source_id) IN (${placeholders})`,
+                        params: [opts.tenantId, ...params],
+                    }));
+                    return new Map(found.map((row) => [docKey(row.corpus, row.source_id), { title: row.title }]));
+                },
+            });
+        },
+        async sweepStale(corpus, tenantId, epoch) {
+            assertNonEmpty(corpus, 'corpus');
+            assertNonEmpty(tenantId, 'tenantId');
+            assertNonEmpty(epoch, 'epoch');
+            return await scoped(tenantId, async (executor) => {
+                // Counted with its own statement rather than read off the DELETE's row
+                // count, so the number the reindex job logs is documents and not
+                // documents-plus-cascaded-chunks. The SQLite provider counts separately
+                // for the same reason (its FTS triggers inflate `changes`).
+                const stale = await executor.get({
+                    sql: 'SELECT COUNT(*) AS n FROM search_docs WHERE corpus = ? AND tenant_id = ? AND indexed_at < ?',
+                    params: [corpus, tenantId, epoch],
+                });
+                await executor.write({
+                    sql: `DELETE FROM search_chunks WHERE doc_id IN (
+                  SELECT id FROM search_docs WHERE corpus = ? AND tenant_id = ? AND indexed_at < ?)`,
+                    params: [corpus, tenantId, epoch],
+                });
+                await executor.write({
+                    sql: 'DELETE FROM search_docs WHERE corpus = ? AND tenant_id = ? AND indexed_at < ?',
+                    params: [corpus, tenantId, epoch],
+                });
+                return Number(stale?.n ?? 0);
+            });
+        },
+        async count(corpus, tenantId) {
+            return await scoped(tenantId, async (executor) => {
+                const row = await executor.get({
+                    sql: 'SELECT COUNT(*) AS n FROM search_docs WHERE corpus = ? AND tenant_id = ?',
+                    params: [corpus, tenantId],
+                });
+                return Number(row?.n ?? 0);
+            });
+        },
+        async countChunks(corpus, tenantId) {
+            return await scoped(tenantId, (executor) => countChunks(executor, corpus, tenantId));
+        },
+    };
+}

package/dist/server/search/pgSearchProvider.pg.test.d.ts

@@ -0,0 +1 @@
+export {};