@farukada/aws-langgraph-dynamodb-ts 1.0.0-rc.1 → 1.0.0

package/dist/cjs/store/internal/table-search.js

@@ -0,0 +1,123 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.collectCandidates = collectCandidates;
+exports.rankInMemory = rankInMemory;
+const clock_js_1 = require("../../shared/clock.js");
+const concurrency_js_1 = require("../../shared/concurrency.js");
+const paginate_js_1 = require("../../shared/dynamodb/paginate.js");
+const retry_js_1 = require("../../shared/dynamodb/retry.js");
+const table_schema_js_1 = require("../../shared/dynamodb/table-schema.js");
+const errors_js_1 = require("../../shared/errors/errors.js");
+const filter_js_1 = require("./filter.js");
+const get_item_js_1 = require("./get-item.js");
+const rows_js_1 = require("./rows.js");
+const semantic_search_js_1 = require("./semantic-search.js");
+function storedVectors(record) {
+    if (record.embeddings)
+        return record.embeddings;
+    return record.embedding ? [record.embedding] : undefined;
+}
+function candidateSource(context, search, signal, now) {
+    return search.namespacePrefix.length > 0
+        ? (0, paginate_js_1.paginateQuery)({
+            retry: (0, retry_js_1.retryFor)(context, signal),
+            signal,
+            client: context.client,
+            params: (0, table_schema_js_1.withoutExpired)((0, rows_js_1.scopedQuery)(context.tableName, search.namespacePrefix), now),
+            maxItems: context.maxScanItems,
+            maxIterations: context.maxIterations,
+        })
+        : (0, paginate_js_1.paginateScan)({
+            retry: (0, retry_js_1.retryFor)(context, signal),
+            signal,
+            client: context.client,
+            params: (0, table_schema_js_1.withoutExpired)((0, rows_js_1.storeScan)(context.tableName), now),
+            maxItems: context.maxScanItems,
+            maxIterations: context.maxIterations,
+        });
+}
+function liveRow(raw, search, now) {
+    const record = (0, rows_js_1.parseWholeStoreRow)(raw);
+    if (!record || (0, table_schema_js_1.isExpiredRow)(record, now))
+        return undefined;
+    return (0, rows_js_1.namespaceMatchesPrefix)(record.namespace, search.namespacePrefix) ? record : undefined;
+}
+async function flush(context, search, state, signal) {
+    if (state.pending.length === 0)
+        return;
+    const batch = state.pending;
+    state.pending = [];
+    const reads = await (0, concurrency_js_1.mapWithConcurrency)(batch, context.readConcurrency ?? concurrency_js_1.DEFAULT_READ_CONCURRENCY, (record) => (0, get_item_js_1.readCandidate)(context, record, signal));
+    for (const read of reads) {
+        if (read !== undefined && (0, filter_js_1.passesFilter)(read.item, search.filter)) {
+            state.collected.push({ item: read.item, embeddings: storedVectors(read.row) });
+        }
+    }
+}
+function batchSize(search, need, collected, limit) {
+    return search.filter === undefined ? Math.min(limit, need - collected) : limit;
+}
+function tooManyCandidates(count, cap) {
+    return (0, errors_js_1.validationError)(`Semantic search candidate set (${count}) exceeds maxSearchCandidates (${cap}); ` +
+        'use a dedicated VectorBackend for large corpora', 'maxSearchCandidates');
+}
+async function collectCandidates(context, search, bound, signal) {
+    const now = (0, clock_js_1.nowSeconds)();
+    const limit = context.readConcurrency ?? concurrency_js_1.DEFAULT_READ_CONCURRENCY;
+    const state = { pending: [], collected: [] };
+    for await (const raw of candidateSource(context, search, signal, now)) {
+        const record = liveRow(raw, search, now);
+        if (!record)
+            continue;
+        state.pending.push(record);
+        if (bound.kind === 'semantic') {
+            if (state.pending.length > bound.cap)
+                throw tooManyCandidates(state.pending.length, bound.cap);
+            continue;
+        }
+        if (state.pending.length < batchSize(search, bound.need, state.collected.length, limit)) {
+            continue;
+        }
+        await flush(context, search, state, signal);
+        if (state.collected.length >= bound.need)
+            return state.collected;
+    }
+    await flush(context, search, state, signal);
+    return state.collected;
+}
+const UNSCORED_RANK = -2;
+function bestScore(vectors, queryVector) {
+    let best;
+    for (const vector of vectors) {
+        if (vector.length !== queryVector.length)
+            continue;
+        const score = (0, semantic_search_js_1.cosineSimilarity)(queryVector, vector);
+        if (best === undefined || score > best)
+            best = score;
+    }
+    return best;
+}
+function isDimensionMismatch(candidate, queryVector) {
+    const vectors = candidate.embeddings;
+    return (vectors !== undefined &&
+        vectors.length > 0 &&
+        vectors.every((vector) => vector.length !== queryVector.length));
+}
+function rankInMemory(candidates, queryVector, maxCandidates, onDimensionMismatch) {
+    if (candidates.length > maxCandidates) {
+        throw (0, errors_js_1.validationError)(`Semantic search candidate set (${candidates.length}) exceeds maxSearchCandidates ` +
+            `(${maxCandidates}); use a dedicated VectorBackend for large corpora`, 'maxSearchCandidates');
+    }
+    const mismatched = candidates.filter((candidate) => isDimensionMismatch(candidate, queryVector)).length;
+    if (mismatched > 0)
+        onDimensionMismatch?.(mismatched);
+    return candidates
+        .map(({ item, embeddings }) => ({
+        ...item,
+        score: embeddings ? bestScore(embeddings, queryVector) : undefined,
+    }))
+        .sort((a, b) => rankValue(b) - rankValue(a));
+}
+function rankValue(item) {
+    return item.score ?? UNSCORED_RANK;
+}

package/dist/cjs/store/internal/vector-index.d.ts

@@ -0,0 +1,258 @@
+/**
+ * Hides the vector backend as a copy of the items' embeddings.
+ *
+ * When a store is given a `VectorBackend`, every item's embedding is kept there
+ * as well as the item in the table, and the table is the truth. When the copy
+ * is written (after the row commits), when an entry is dropped (only once a
+ * fresh read finds the row gone), how a backend's answer is resolved back to
+ * the canonical items and in which direction its scores run, and how the copy
+ * is repaired against the table are decided here, and nothing else calls the
+ * backend.
+ */
+import type { IndexConfig, SearchItem } from '@langchain/langgraph-checkpoint';
+import type { VectorReconcileResult } from '../types.js';
+import type { VectorBackend, VectorMatch, VectorRef, VectorScoreDirection } from '../vector-backend.js';
+import { type Namespace, type ParsedPut, type ParsedSearch, type StoreAddress } from './parse.js';
+import type { StoreContext } from './setup.js';
+/** A store context whose vector copy is configured: an embeddings index and a backend. */
+export type BackendContext = StoreContext & {
+    readonly index: IndexConfig;
+    readonly vectorBackend: VectorBackend;
+};
+/**
+ * Whether the store keeps a vector copy.
+ *
+ * Accepts: `context` — the store's.
+ *
+ * Returns: `true`, narrowing `context`, when both `index` and `vectorBackend`
+ * are configured.
+ *
+ * Throws: nothing.
+ */
+export declare function hasVectorBackend(context: StoreContext): context is BackendContext;
+/**
+ * The one vector a put gives the backend for its item.
+ *
+ * Accepts: `op` — the parsed put. Its `index` is `false` to embed nothing, a
+ * field list to embed those fields, absent to embed the configured ones.
+ *
+ * Returns: the joined embedding, or `undefined` when the store keeps no vector
+ * copy, the put indexes nothing, or the value has no text to embed.
+ *
+ * Throws: `VALIDATION` naming `index.dims` when the model returns a vector of
+ * another length; whatever `embedDocuments` throws.
+ */
+export declare function itemVector(context: StoreContext, op: ParsedPut): Promise<number[] | undefined>;
+/**
+ * Write an item's entry in the vector copy after its row has committed: the
+ * vector when there is one, a delete when there is none. Best-effort sync of
+ * one item's embedding to the vector backend after the canonical DynamoDB
+ * write has already succeeded.
+ *
+ * Accepts: `address` — the item's. `embedding` — from {@link itemVector}:
+ * present upserts it; absent deletes any vector the backend still holds, which
+ * is what a re-put with no indexable text, an `index: false` put and a delete
+ * all mean.
+ *
+ * Returns: nothing, in both the synced and the failed case; without a backend,
+ * nothing is done.
+ *
+ * Throws: nothing. The canonical item is already committed, so failing here
+ * would report a put that in fact succeeded; the drift is logged at `warn` and
+ * `reconcileVectorIndex` repairs it.
+ */
+export declare function syncItemVector(context: StoreContext, address: StoreAddress, embedding: number[] | undefined): Promise<void>;
+/**
+ * Drop the item's vector, but only on a fresh read that finds no row at the
+ * key.
+ *
+ * The question is deliberately **not** "did this call remove the row" — that
+ * one is true in exactly the interleaving that goes wrong. It is "does the key
+ * hold a row *now*", which a racing put that recreated it and a
+ * compare-and-swap that left it alone both answer the same way, and which costs
+ * a point read of this library's own table rather than anything the backend has
+ * to offer. The reconciler asks the same question before pruning a candidate
+ * it never saw; one it did see is pinned to the revision the snapshot read
+ * instead. Either way, the delete path and the reconciler are equally careful.
+ *
+ * A read that itself fails answers "not confirmed" and keeps the vector: a
+ * stale vector for a deleted item, which `reconcileVectorIndex` removes, rather
+ * than a missing one for a live item, which is the defect this exists for.
+ *
+ * Accepts: `address` — the deleted item's; the row read is derived from it.
+ *
+ * Returns: nothing; without a backend, nothing is read or dropped. A row still
+ * at the key keeps its vector and logs one `info`.
+ *
+ * Throws: nothing the backend throws (see {@link syncItemVector}); whatever the
+ * confirmation read throws that `isRowAbsent` does not answer as "not
+ * confirmed".
+ */
+export declare function dropVectorWhenGone(context: StoreContext, address: StoreAddress): Promise<void>;
+/** A canonical item's location plus the embedding recomputed for it. */
+export interface ReconcileTarget {
+    namespace: string[];
+    key: string;
+    embedding: number[] | undefined;
+    /** The revision the snapshot read, which a later put replaces; absent on a row written before revisions. */
+    rev?: string;
+}
+/**
+ * Enumerate the live (unexpired) items under `prefix` and recompute their
+ * embeddings.
+ *
+ * Accepts: `prefix` — at least one element, since the enumeration is one
+ * partition's Query; reconciling is always scoped. `signal` — aborts between
+ * pages.
+ *
+ * Returns: one target per live item, each with the embedding the item's current
+ * value produces — `undefined` when it yields no indexable text, which is the
+ * evidence that lets {@link pruneOrphans} drop a vector that has gone stale.
+ *
+ * Throws: whatever the reads, the decodes and the embeddings model throw; a
+ * failed embedding rejects the whole reconcile, because a skipped item would
+ * leave the live set and {@link selectOrphans} would prune its still-valid
+ * vector. `RESULT_TRUNCATED` when `maxScanItems` or `maxIterations` is
+ * reached while rows remain — reconciling from a partial view would prune
+ * live vectors.
+ *
+ * Guarantees: rows are decoded in bounded batches, so a namespace of offloaded
+ * items costs neither one round-trip at a time nor every payload in memory at
+ * once.
+ */
+export declare function collectReconcileTargets(context: StoreContext, prefix: Namespace, signal?: AbortSignal): Promise<ReconcileTarget[]>;
+/**
+ * Re-push every live embedding to the backend.
+ *
+ * Accepts: `targets` — a target with no embedding is not pushed; its vector is
+ * {@link pruneOrphans}' business, not an upsert of nothing.
+ *
+ * Returns: how many vectors were upserted.
+ *
+ * Throws: whatever the backend throws. The upserts are issued one at a time on
+ * purpose: this is a bulk repair against a third-party index, and the run has
+ * no latency budget worth a thundering herd.
+ */
+export declare function pushEmbeddings(backend: VectorBackend, targets: ReconcileTarget[]): Promise<number>;
+/**
+ * Refs the backend holds that the live set does not account for.
+ *
+ * Accepts: `backendRefs` — whatever the backend lists under the prefix.
+ * `live` — the snapshot; a target with no embedding does not account for a
+ * vector, since its item no longer produces one.
+ *
+ * Returns: the candidates to prune — candidates, not conclusions: the snapshot
+ * and the backend listing are not one point in time (see {@link pruneOrphans}).
+ *
+ * Throws: nothing.
+ */
+export declare function selectOrphans(backendRefs: VectorRef[], live: ReconcileTarget[]): VectorRef[];
+/**
+ * Delete backend vectors with no canonical item.
+ *
+ * Accepts: `live` — the snapshot {@link collectReconcileTargets} took.
+ * `backend.listKeys` — optional; a backend that cannot enumerate its own keys
+ * cannot be pruned, which is reported rather than silently skipped.
+ *
+ * Returns: how many vectors were deleted.
+ *
+ * Throws: whatever the backend and the confirmation read throw.
+ *
+ * Guarantees: a vector is deleted only on evidence that its item is gone — a
+ * fresh strongly-consistent read finds the item still at the revision the
+ * snapshot read — when that snapshot yielded it no embedding — or finds no
+ * row at all, or one past its `ttl` that DynamoDB has yet to sweep. An item
+ * written between the snapshot and the listing looks orphaned and is kept, so
+ * reconciling never drops a just-written item out of semantic search.
+ */
+export declare function pruneOrphans(context: StoreContext, backend: VectorBackend, prefix: Namespace, live: ReconcileTarget[]): Promise<number>;
+/**
+ * Repair the vector copy against the items under a prefix: re-push every live
+ * embedding, then prune entries whose item is confirmed gone.
+ *
+ * Accepts: `prefix` — parsed. `signal` — cancels the table read.
+ *
+ * Returns: how many entries were upserted and pruned.
+ *
+ * Throws: whatever the table read, the embeddings model or the backend throws.
+ */
+export declare function reconcileVectors(context: BackendContext, prefix: Namespace, signal?: AbortSignal): Promise<VectorReconcileResult>;
+/**
+ * Rank a search through a configured vector backend.
+ *
+ * Accepts: `context` — the store's, with the backend searched and the index
+ * whose embeddings model embeds the query. `search.query` — non-empty; the
+ * caller checks that before choosing this path. `search.offset`/`search.limit`
+ * — the page, whose end (`offset + limit`) must fit within
+ * `maxSearchCandidates`, since that many matches have to be fetched to fill it.
+ * `search.filter` — applied to the canonical item, not to whatever the backend
+ * stored, so a filter is never answered from a stale vector.
+ *
+ * Returns: the page's items in the backend's order, each carrying the relevance
+ * score for its vector — once each: a backend holding several vectors for one
+ * item (one per passage, say) can return it more than once in a round, and
+ * the item is answered at the first score it gave, as the reference store and
+ * {@link rankInMemory} answer an item by its best-matching vector. Fewer than
+ * `limit` items means the backend has no more matches under the prefix, not
+ * that the page was cut short. An item the backend holds no vector for — put
+ * with `index: false`, or with no indexable text — is not a match here, where
+ * the in-DynamoDB ranker lists it last with no score.
+ *
+ * Throws: `VALIDATION` naming `index.dims` when the query embeds to a
+ * different width than the index declares, and naming `maxSearchCandidates`
+ * either for a page larger than the cap or when the filter leaves the page short
+ * at the cap — the same answer the in-DynamoDB ranker gives, rather than a
+ * silently short page. Whatever a canonical read throws — a read that did not
+ * happen is not an item that is not there — since a match this store cannot
+ * address is dropped before its read rather than caught after it (see
+ * {@link addressOf}). Whatever the embeddings model and the backend throw.
+ *
+ * Guarantees: DynamoDB stays canonical. A match whose item has since been
+ * deleted or lies outside the prefix is dropped and the search asks the backend
+ * for more, so a stale or over-broad index costs results only in latency. A
+ * match whose read *fails* is not dropped: the page a caller receives is never
+ * shorter than the matches that exist, which is what the in-DynamoDB path
+ * already promises. Items are fetched with the same bounded concurrency as the
+ * in-DynamoDB path, and each distinct match is read once for the whole call
+ * however many rounds it takes (see {@link fetchUnseen}).
+ */
+export declare function searchViaBackend(context: BackendContext, search: ParsedSearch, signal?: AbortSignal): Promise<SearchItem[]>;
+/**
+ * Normalise a backend's matches to the relevance direction upstream
+ * `SearchItem.score` documents — "higher scores indicate better matches",
+ * typically a cosine similarity between -1 and 1.
+ *
+ * A distance is converted by **negation** rather than `1 / (1 + d)`: negation
+ * is monotone over the whole real line, needs no non-negativity precondition,
+ * and is exactly invertible, so the original distance is simply `-score`. A
+ * reciprocal would silently imply a different `(0, 1]` scale and is undefined
+ * at `d === -1`. Negative scores are already in range for this contract, so
+ * nothing is lost by producing them.
+ *
+ * Results are re-sorted after conversion so a backend that returns its matches
+ * in some other order still ranks correctly. Anything that is not `'distance'`
+ * is returned untouched — its own order stays authoritative, exactly as before.
+ *
+ * The test is for `'distance'` rather than against `'relevance'` on purpose:
+ * converting is the destructive branch, so only the exact value that asks for
+ * it may reach it. Testing the other way round made every unrecognised
+ * string — `'Distance'`, a typo, a value read from a config file — reverse the
+ * ranking silently, with no error and no warning (the ascending-score warning
+ * lives downstream of this conversion, so it could never fire). `setUpStore`
+ * rejects such a value outright; this keeps the failure harmless for any
+ * caller that reaches the function some other way.
+ *
+ * Accepts: `matches` — in any order, including empty. `direction` — only the
+ * exact string `'distance'` converts; every other value, recognised or not,
+ * passes the matches through untouched.
+ *
+ * Returns: matches whose `score` follows the relevance direction upstream
+ * documents, highest first. Converting re-sorts, so a backend that returned its
+ * matches in some other order still ranks correctly.
+ *
+ * Throws: nothing.
+ *
+ * Guarantees: the conversion is exactly invertible — the original distance is
+ * `-score` — so nothing about the backend's answer is lost.
+ */
+export declare function toRelevanceScores(matches: VectorMatch[], direction: VectorScoreDirection): VectorMatch[];

package/dist/cjs/store/internal/vector-index.js

@@ -0,0 +1,269 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasVectorBackend = hasVectorBackend;
+exports.itemVector = itemVector;
+exports.syncItemVector = syncItemVector;
+exports.dropVectorWhenGone = dropVectorWhenGone;
+exports.collectReconcileTargets = collectReconcileTargets;
+exports.pushEmbeddings = pushEmbeddings;
+exports.selectOrphans = selectOrphans;
+exports.pruneOrphans = pruneOrphans;
+exports.reconcileVectors = reconcileVectors;
+exports.searchViaBackend = searchViaBackend;
+exports.toRelevanceScores = toRelevanceScores;
+const clock_js_1 = require("../../shared/clock.js");
+const concurrency_js_1 = require("../../shared/concurrency.js");
+const idempotent_write_js_1 = require("../../shared/dynamodb/idempotent-write.js");
+const paginate_js_1 = require("../../shared/dynamodb/paginate.js");
+const retry_js_1 = require("../../shared/dynamodb/retry.js");
+const table_schema_js_1 = require("../../shared/dynamodb/table-schema.js");
+const base_error_js_1 = require("../../shared/errors/base-error.js");
+const errors_js_1 = require("../../shared/errors/errors.js");
+const truncate_js_1 = require("../../shared/logging/truncate.js");
+const filter_js_1 = require("./filter.js");
+const get_item_js_1 = require("./get-item.js");
+const parse_js_1 = require("./parse.js");
+const rows_js_1 = require("./rows.js");
+const semantic_search_js_1 = require("./semantic-search.js");
+function hasVectorBackend(context) {
+    return context.index !== undefined && context.vectorBackend !== undefined;
+}
+async function itemVector(context, op) {
+    if (!context.vectorBackend || op.index === false)
+        return undefined;
+    return (0, semantic_search_js_1.embedValue)(context, op.value, op.index);
+}
+async function syncItemVector(context, address, embedding) {
+    const backend = context.vectorBackend;
+    if (backend === undefined)
+        return;
+    try {
+        if (embedding)
+            await backend.upsert(address.namespace, address.key, embedding);
+        else
+            await backend.delete(address.namespace, address.key);
+    }
+    catch (error) {
+        context.logger.warn('store vector-index sync failed; reconcileVectorIndex will repair', {
+            namespace: address.namespace,
+            key: address.key,
+            operation: embedding ? 'upsert' : 'delete',
+            reason: (0, truncate_js_1.truncateForLog)((0, base_error_js_1.failureLabel)(error)),
+        });
+    }
+}
+async function dropVectorWhenGone(context, address) {
+    if (context.vectorBackend === undefined)
+        return;
+    const key = (0, rows_js_1.itemRowKey)(address);
+    if (!(await (0, idempotent_write_js_1.isRowAbsent)(context, key))) {
+        context.logger.info('store.delete: kept a vector whose item was not confirmed gone', {
+            namespace: address.namespace,
+            key: address.key,
+        });
+        return;
+    }
+    await syncItemVector(context, address, undefined);
+}
+function refIdentity(namespace, key) {
+    return JSON.stringify([...namespace, key]);
+}
+async function drainPending(context, pending, live, signal) {
+    if (pending.length === 0)
+        return;
+    const batch = pending.splice(0, pending.length);
+    const reads = await (0, concurrency_js_1.mapWithConcurrency)(batch, context.readConcurrency ?? concurrency_js_1.DEFAULT_READ_CONCURRENCY, (record) => (0, get_item_js_1.readCandidate)(context, record, signal));
+    for (const read of reads) {
+        if (read === undefined)
+            continue;
+        live.push({
+            namespace: read.row.namespace,
+            key: read.row.key,
+            rev: read.row.rev,
+            value: read.item.value,
+        });
+    }
+}
+async function collectReconcileTargets(context, prefix, signal) {
+    const now = (0, clock_js_1.nowSeconds)();
+    const live = [];
+    const pending = [];
+    const batchLimit = context.readConcurrency ?? concurrency_js_1.DEFAULT_READ_CONCURRENCY;
+    const source = (0, paginate_js_1.paginateQuery)({
+        retry: (0, retry_js_1.retryFor)(context, signal),
+        signal,
+        client: context.client,
+        params: (0, table_schema_js_1.withoutExpired)((0, rows_js_1.scopedQuery)(context.tableName, prefix), now),
+        maxItems: context.maxScanItems,
+        maxIterations: context.maxIterations,
+    });
+    for await (const raw of source) {
+        const record = (0, rows_js_1.parseWholeStoreRow)(raw);
+        if (!record) {
+            context.logger.warn('reconcileVectorIndex: skipped a row that is not a store item', {
+                sortKey: (0, truncate_js_1.truncateForLog)(raw.SK),
+            });
+            continue;
+        }
+        if ((0, table_schema_js_1.isExpiredRow)(record, now) || !(0, rows_js_1.namespaceMatchesPrefix)(record.namespace, prefix))
+            continue;
+        pending.push(record);
+        if (pending.length >= batchLimit)
+            await drainPending(context, pending, live, signal);
+    }
+    await drainPending(context, pending, live, signal);
+    const embeddings = await (0, semantic_search_js_1.embedValues)(context, live.map((entry) => entry.value));
+    return live.map((entry, i) => ({
+        namespace: entry.namespace,
+        key: entry.key,
+        embedding: embeddings[i],
+        rev: entry.rev,
+    }));
+}
+async function pushEmbeddings(backend, targets) {
+    let upserted = 0;
+    for (const target of targets) {
+        if (!target.embedding)
+            continue;
+        await backend.upsert(target.namespace, target.key, target.embedding);
+        upserted += 1;
+    }
+    return upserted;
+}
+function selectOrphans(backendRefs, live) {
+    const liveKeys = new Set(live
+        .filter((target) => target.embedding !== undefined)
+        .map((target) => refIdentity(target.namespace, target.key)));
+    return backendRefs.filter((ref) => !liveKeys.has(refIdentity(ref.namespace, ref.key)));
+}
+async function confirmedGone(context, ref) {
+    const row = await (0, idempotent_write_js_1.readRow)(context, {
+        key: (0, rows_js_1.itemRowKey)(ref),
+        attribute: table_schema_js_1.PARTITION_KEY_ATTRIBUTE,
+        also: ['ttl'],
+    });
+    return row === undefined || (0, table_schema_js_1.isExpiredRow)(row, (0, clock_js_1.nowSeconds)());
+}
+async function unchangedSinceSnapshot(context, ref, rev) {
+    const row = await (0, idempotent_write_js_1.readRow)(context, { key: (0, rows_js_1.itemRowKey)(ref), attribute: rows_js_1.REVISION_ATTRIBUTE });
+    return row === undefined || row[rows_js_1.REVISION_ATTRIBUTE] === rev;
+}
+async function pruneOrphans(context, backend, prefix, live) {
+    if (!backend.listKeys) {
+        context.logger.info('reconcileVectorIndex prune skipped: backend has no listKeys', {
+            prefix: (0, truncate_js_1.truncateLabelsForLog)(prefix),
+        });
+        return 0;
+    }
+    const candidates = selectOrphans(await backend.listKeys(prefix), live);
+    const snapshot = new Map(live.map((target) => [refIdentity(target.namespace, target.key), target.rev]));
+    let pruned = 0;
+    for (const ref of candidates) {
+        const identity = refIdentity(ref.namespace, ref.key);
+        const stale = snapshot.has(identity)
+            ? await unchangedSinceSnapshot(context, ref, snapshot.get(identity))
+            : await confirmedGone(context, ref);
+        if (!stale) {
+            context.logger.info('reconcileVectorIndex: kept a vector whose item reappeared', {
+                namespace: (0, truncate_js_1.truncateLabelsForLog)(ref.namespace),
+                key: (0, truncate_js_1.truncateForLog)(ref.key),
+            });
+            continue;
+        }
+        await backend.delete(ref.namespace, ref.key);
+        pruned += 1;
+    }
+    return pruned;
+}
+async function reconcileVectors(context, prefix, signal) {
+    const targets = await collectReconcileTargets(context, prefix, signal);
+    const upserted = await pushEmbeddings(context.vectorBackend, targets);
+    const pruned = await pruneOrphans(context, context.vectorBackend, prefix, targets);
+    return { upserted, pruned };
+}
+function warnOnNonDescendingScores(context, matches, namespacePrefix) {
+    const descending = matches.every((match, position) => position === 0 || matches[position - 1].score >= match.score);
+    if (descending)
+        return;
+    context.logger.warn('search: vectorBackend returned ascending scores; VectorMatch.score must be a relevance ' +
+        '(higher is better), not a distance — results are forwarded in the order the backend gave', { namespacePrefix: (0, truncate_js_1.truncateLabelsForLog)(namespacePrefix) });
+}
+function addressOf(context, match) {
+    try {
+        return (0, parse_js_1.parseStoreAddress)(match.namespace, match.key);
+    }
+    catch (error) {
+        context.logger.warn('search: skipped an unusable vectorBackend match', {
+            namespace: (0, truncate_js_1.truncateLabelsForLog)(match.namespace),
+            key: (0, truncate_js_1.truncateForLog)(match.key),
+            reason: (0, truncate_js_1.truncateForLog)((0, base_error_js_1.failureLabel)(error)),
+        });
+        return undefined;
+    }
+}
+async function fetchMatch(context, match, signal) {
+    const address = addressOf(context, match);
+    if (address === undefined)
+        return null;
+    return (0, get_item_js_1.getItem)(context, address, signal);
+}
+function matchIdentity(match) {
+    return JSON.stringify([match.namespace, match.key]);
+}
+async function fetchUnseen(context, scoped, fetched, signal) {
+    const unseen = new Map();
+    for (const match of scoped) {
+        const identity = matchIdentity(match);
+        if (!fetched.has(identity))
+            unseen.set(identity, match);
+    }
+    const pending = [...unseen.values()];
+    const items = await (0, concurrency_js_1.mapWithConcurrency)(pending, context.readConcurrency ?? concurrency_js_1.DEFAULT_READ_CONCURRENCY, (match) => fetchMatch(context, match, signal));
+    pending.forEach((match, index) => fetched.set(matchIdentity(match), items[index]));
+}
+async function searchViaBackend(context, search, signal) {
+    const backend = context.vectorBackend;
+    const index = context.index;
+    const { offset, limit } = search;
+    const need = offset + limit;
+    if (need > context.maxSearchCandidates) {
+        throw (0, errors_js_1.validationError)(`Requested page (offset ${offset} + limit ${limit} = ${need}) exceeds maxSearchCandidates ` +
+            `(${context.maxSearchCandidates}); narrow the page or raise maxSearchCandidates`, 'maxSearchCandidates');
+    }
+    const queryVector = await index.embeddings.embedQuery(search.query);
+    (0, semantic_search_js_1.assertVectorDims)(index, queryVector, 'query');
+    let topK = Math.min(need, context.maxSearchCandidates);
+    let results;
+    const fetched = new Map();
+    for (;;) {
+        const matches = toRelevanceScores(await backend.query(search.namespacePrefix, queryVector, topK), context.vectorScoreDirection);
+        warnOnNonDescendingScores(context, matches, search.namespacePrefix);
+        const scoped = matches.filter((match) => (0, rows_js_1.namespaceMatchesPrefix)(match.namespace, search.namespacePrefix));
+        await fetchUnseen(context, scoped, fetched, signal);
+        results = [];
+        const answered = new Set();
+        for (const match of scoped) {
+            const identity = matchIdentity(match);
+            const item = fetched.get(identity);
+            if (!item || answered.has(identity) || !(0, filter_js_1.passesFilter)(item, search.filter))
+                continue;
+            answered.add(identity);
+            results.push({ ...item, score: match.score });
+        }
+        if (results.length >= need || matches.length < topK)
+            break;
+        if (topK >= context.maxSearchCandidates) {
+            throw (0, errors_js_1.validationError)(`Semantic search collected ${results.length} of ${need} matches within maxSearchCandidates ` +
+                `(${context.maxSearchCandidates}); narrow the filter or raise maxSearchCandidates`, 'maxSearchCandidates');
+        }
+        topK = Math.min(topK * 2, context.maxSearchCandidates);
+    }
+    return results;
+}
+function toRelevanceScores(matches, direction) {
+    if (direction !== 'distance')
+        return matches;
+    return matches
+        .map((match) => ({ ...match, score: -match.score }))
+        .sort((left, right) => right.score - left.score);
+}