@seanhogg/builderforce-memory 2026.7.14 → 2026.9.27

package/src/vectorstore/MemoryVectorStore.ts

@@ -0,0 +1,250 @@
+/**
+ * vectorstore/MemoryVectorStore.ts — the in-process reference adapter.
+ *
+ * It is the default store, the test double for every layer above it, and the
+ * conformance benchmark a remote adapter is checked against. It is also the
+ * honest answer for edge deployments (a Worker, a browser tab, an on-prem box)
+ * where standing up a vector database is not on the table.
+ *
+ * Performance notes, because "in-memory" is not an excuse for O(N) per query:
+ *   • The HNSW graph is built ONCE and extended incrementally on upsert. The
+ *     shared `denseSearch` helper rebuilds its index per call, which is right for
+ *     a one-shot rerank over a candidate set and wrong for a store that is queried
+ *     repeatedly — so this adapter owns a persistent index instead.
+ *   • Deletes tombstone rather than rebuild; the graph is rebuilt only once
+ *     tombstones pass a share of the index, amortising the cost.
+ *   • Filtered queries over-fetch from the graph and post-filter, falling back to
+ *     an exact scan only when the filter is selective enough that the graph could
+ *     not fill `topK`. Correct in both regimes, fast in the common one.
+ */
+
+import { bm25Search } from '../retrieval/bm25.js';
+import { HnswIndex, type HnswOptions } from '../retrieval/hnsw.js';
+import { cosineSimilarity } from '../similarity/index.js';
+import { combineFilters, matchesFilter } from './filter.js';
+import type {
+    AccessScope,
+    MetadataFilter,
+    UpsertResult,
+    VectorMatch,
+    VectorQuery,
+    VectorRecord,
+    VectorStore,
+} from './types.js';
+
+export interface MemoryVectorStoreOptions {
+    /** Adapter name for telemetry. Default `'memory'`. */
+    name?: string;
+    /**
+     * Record count at/above which dense search uses the HNSW graph instead of an
+     * exact cosine scan. Below it, a scan is both simpler and faster. Default 256.
+     */
+    annThreshold?: number;
+    /** HNSW construction parameters. */
+    hnsw?: HnswOptions;
+    /**
+     * Over-fetch multiplier for filtered ANN queries — how many graph candidates
+     * to pull per requested result before post-filtering. Default 4.
+     */
+    filterOverfetch?: number;
+    /**
+     * Rebuild the graph once tombstoned nodes exceed this share of it. Default 0.2.
+     */
+    rebuildTombstoneRatio?: number;
+}
+
+export class MemoryVectorStore implements VectorStore {
+    readonly name: string;
+
+    private readonly _records = new Map<string, VectorRecord>();
+    private readonly _annThreshold: number;
+    private readonly _hnswOpts: HnswOptions;
+    private readonly _overfetch: number;
+    private readonly _rebuildRatio: number;
+
+    private _index: HnswIndex | undefined;
+    /** Ids present in the graph but deleted from the store. */
+    private readonly _tombstones = new Set<string>();
+
+    constructor(opts: MemoryVectorStoreOptions = {}) {
+        this.name = opts.name ?? 'memory';
+        this._annThreshold = Math.max(1, opts.annThreshold ?? 256);
+        this._hnswOpts = opts.hnsw ?? {};
+        this._overfetch = Math.max(1, opts.filterOverfetch ?? 4);
+        this._rebuildRatio = opts.rebuildTombstoneRatio ?? 0.2;
+    }
+
+    get size(): number { return this._records.size; }
+
+    async upsert(records: readonly VectorRecord[]): Promise<UpsertResult> {
+        let upserted = 0;
+        for (const record of records) {
+            const previous = this._records.get(record.id);
+            this._records.set(record.id, record);
+            upserted += 1;
+
+            if (!record.vector) continue;
+            // A changed vector under an existing id invalidates its graph node;
+            // tombstone the old one so search cannot return the stale neighbour.
+            if (previous?.vector && this._index?.has(record.id)) {
+                this._tombstones.add(record.id);
+                this._maybeRebuild();
+                continue;
+            }
+            if (this._index && !this._index.has(record.id)) {
+                this._index.add(record.id, record.vector);
+                this._tombstones.delete(record.id);
+            }
+        }
+        return { upserted, skipped: 0 };
+    }
+
+    async fetch(ids: readonly string[]): Promise<VectorRecord[]> {
+        const out: VectorRecord[] = [];
+        for (const id of ids) {
+            const record = this._records.get(id);
+            if (record) out.push(record);
+        }
+        return out;
+    }
+
+    async delete(ids: readonly string[]): Promise<number> {
+        let removed = 0;
+        for (const id of ids) {
+            if (!this._records.delete(id)) continue;
+            removed += 1;
+            if (this._index?.has(id)) this._tombstones.add(id);
+        }
+        this._maybeRebuild();
+        return removed;
+    }
+
+    async deleteByFilter(filter: MetadataFilter, scope?: AccessScope): Promise<number> {
+        const effective = combineFilters(filter, scope);
+        const doomed = [...this._records.values()]
+            .filter((r) => matchesFilter(r.metadata, effective))
+            .map((r) => r.id);
+        return this.delete(doomed);
+    }
+
+    async count(filter?: MetadataFilter, scope?: AccessScope): Promise<number> {
+        const effective = combineFilters(filter, scope);
+        if (!effective) return this._records.size;
+        let n = 0;
+        for (const record of this._records.values()) {
+            if (matchesFilter(record.metadata, effective)) n += 1;
+        }
+        return n;
+    }
+
+    async query(query: VectorQuery): Promise<VectorMatch[]> {
+        const topK = Math.max(1, query.topK ?? 10);
+        const minScore = query.minScore ?? 0;
+        const filter = combineFilters(query.filter, query.scope);
+
+        if (!query.vector) {
+            // No embedding: degrade to a filter-only listing rather than returning
+            // nothing, so metadata-only lookups do not need a second code path.
+            return this._visible(filter)
+                .slice(0, topK)
+                .map((r) => this._toMatch(r, 1, query.includeVectors));
+        }
+
+        const scored = this._denseScore(query.vector, topK, filter);
+        return scored
+            .filter((hit) => hit.score >= minScore)
+            .slice(0, topK)
+            .map(({ record, score }) => this._toMatch(record, score, query.includeVectors));
+    }
+
+    /**
+     * BM25 over the visible corpus. Present because lexical recall catches the
+     * exact identifiers (error codes, SKUs, function names) that embeddings blur —
+     * the failure mode enterprise RAG evaluations reliably surface.
+     */
+    async keywordSearch(query: VectorQuery): Promise<VectorMatch[]> {
+        const text = query.text?.trim();
+        if (!text) return [];
+
+        const topK = Math.max(1, query.topK ?? 10);
+        const filter = combineFilters(query.filter, query.scope);
+        const visible = this._visible(filter);
+        if (visible.length === 0) return [];
+
+        const hits = bm25Search(text, visible.map((r) => ({ id: r.id, text: r.text })));
+        const best = hits[0]?.score ?? 0;
+
+        return hits.slice(0, topK).map((hit) => {
+            const record = this._records.get(hit.id) as VectorRecord;
+            // BM25 scores are unbounded; normalising against the top hit puts them
+            // on the same [0,1] footing as cosine so fusion weights mean something.
+            return this._toMatch(record, best > 0 ? hit.score / best : 0, query.includeVectors);
+        });
+    }
+
+    private _denseScore(
+        vector: Float32Array,
+        topK: number,
+        filter?: MetadataFilter,
+    ): Array<{ record: VectorRecord; score: number }> {
+        const embedded = [...this._records.values()].filter((r) => r.vector);
+
+        const exactScan = () => embedded
+            .filter((r) => matchesFilter(r.metadata, filter))
+            .map((r) => ({ record: r, score: cosineSimilarity(vector, r.vector as Float32Array) }))
+            .sort((a, b) => b.score - a.score);
+
+        if (embedded.length < this._annThreshold) return exactScan();
+
+        this._ensureIndex(embedded);
+        const index = this._index as HnswIndex;
+        const wanted = filter ? topK * this._overfetch : topK;
+        const hits = index.search(vector, wanted + this._tombstones.size);
+
+        const matched: Array<{ record: VectorRecord; score: number }> = [];
+        for (const hit of hits) {
+            if (this._tombstones.has(hit.id)) continue;
+            const record = this._records.get(hit.id);
+            if (!record || !matchesFilter(record.metadata, filter)) continue;
+            matched.push({ record, score: hit.score });
+        }
+
+        // The graph could not fill the page under this filter — the filter is more
+        // selective than the over-fetch anticipated, so pay for an exact scan
+        // rather than return a short, silently-truncated result set.
+        if (matched.length < topK && filter) return exactScan();
+        return matched;
+    }
+
+    private _visible(filter?: MetadataFilter): VectorRecord[] {
+        const all = [...this._records.values()];
+        return filter ? all.filter((r) => matchesFilter(r.metadata, filter)) : all;
+    }
+
+    private _toMatch(record: VectorRecord, score: number, includeVectors?: boolean): VectorMatch {
+        const match: VectorMatch = {
+            id: record.id,
+            text: record.text,
+            score,
+            metadata: record.metadata,
+        };
+        if (includeVectors && record.vector) match.vector = record.vector;
+        return match;
+    }
+
+    private _ensureIndex(embedded: readonly VectorRecord[]): void {
+        if (this._index) return;
+        const index = new HnswIndex(this._hnswOpts);
+        for (const record of embedded) index.add(record.id, record.vector as Float32Array);
+        this._index = index;
+        this._tombstones.clear();
+    }
+
+    private _maybeRebuild(): void {
+        const index = this._index;
+        if (!index || index.size === 0) return;
+        if (this._tombstones.size / index.size < this._rebuildRatio) return;
+        this._index = undefined;
+        this._tombstones.clear();
+    }
+}

package/src/vectorstore/RestVectorStore.ts

@@ -0,0 +1,236 @@
+/**
+ * vectorstore/RestVectorStore.ts — ONE adapter for every HTTP vector database.
+ *
+ * The vendor-specific part is the {@link VectorDialect}; everything that is the
+ * same everywhere lives here exactly once: auth headers, batching, retry with
+ * backoff, residual filtering, capability emulation, and text hydration for stores
+ * that hold vectors only.
+ *
+ * The residual pass is the part that matters for correctness. When a dialect
+ * cannot push a filter clause down, the store over-fetches and applies the
+ * remaining clauses locally with the SAME evaluator the in-process store uses. A
+ * clause is therefore never dropped — which on an ACL or tenant clause would be a
+ * data leak, and is precisely the failure mode a naive adapter ships with.
+ */
+
+import { combineFilters, matchesFilter } from './filter.js';
+import { getDialect, type DialectContext, type HttpRequest, type VectorDialect } from './dialects.js';
+import type {
+    AccessScope,
+    MetadataFilter,
+    UpsertResult,
+    VectorMatch,
+    VectorQuery,
+    VectorRecord,
+    VectorStore,
+} from './types.js';
+
+export interface RestVectorStoreOptions {
+    /** Dialect id (`evermind`, `qdrant`, `pinecone`, `vertex-ai`) or an instance. */
+    dialect: string | VectorDialect;
+    /** Service root, e.g. `https://my-cluster.qdrant.io`. */
+    baseUrl: string;
+    /** Collection / index / namespace. */
+    collection: string;
+    /** Static headers — API keys, `Authorization: Bearer …`. */
+    headers?: Record<string, string>;
+    /**
+     * Per-request auth, for credentials that expire. On GCP this is where a
+     * Workload-Identity access token goes, refreshed by the caller's token source
+     * rather than cached here — the store must never own a credential lifecycle.
+     */
+    authorize?: () => Promise<Record<string, string>> | Record<string, string>;
+    /** Dialect-specific deployment values (`index`, `indexEndpoint`, `deployedIndexId`, …). */
+    options?: Record<string, string>;
+    /** Records per upsert request. Default 100. */
+    batchSize?: number;
+    /** Retries on 429/5xx/network failure. Default 2. */
+    maxRetries?: number;
+    /** Base backoff in ms, doubled per attempt. Default 200. */
+    retryBackoffMs?: number;
+    /** Over-fetch multiplier when a residual filter must be applied locally. Default 4. */
+    residualOverfetch?: number;
+    /**
+     * Supplies chunk text for dialects that store vectors only (Vertex AI). In a
+     * GCP reference architecture this reads the chunk table from Firestore or
+     * BigQuery keyed by the same ids the index holds.
+     */
+    textResolver?: (ids: readonly string[]) => Promise<Map<string, string>>;
+    /** Injected for test. Default global `fetch`. */
+    fetchImpl?: typeof fetch;
+    /** Injected for test so retry backoff does not really sleep. */
+    sleep?: (ms: number) => Promise<void>;
+}
+
+export class RestVectorStore implements VectorStore {
+    readonly name: string;
+    readonly dialect: VectorDialect;
+
+    private readonly _baseUrl: string;
+    private readonly _ctx: DialectContext;
+    private readonly _headers: Record<string, string>;
+    private readonly _authorize: (() => Promise<Record<string, string>> | Record<string, string>) | undefined;
+    private readonly _batchSize: number;
+    private readonly _maxRetries: number;
+    private readonly _backoffMs: number;
+    private readonly _overfetch: number;
+    private readonly _textResolver: RestVectorStoreOptions['textResolver'];
+    private readonly _fetch: typeof fetch;
+    private readonly _sleep: (ms: number) => Promise<void>;
+
+    constructor(opts: RestVectorStoreOptions) {
+        this.dialect = typeof opts.dialect === 'string' ? getDialect(opts.dialect) : opts.dialect;
+        this.name = `rest:${this.dialect.id}`;
+        this._baseUrl = opts.baseUrl.replace(/\/+$/, '');
+        this._ctx = { collection: opts.collection, options: opts.options ?? {} };
+        this._headers = opts.headers ?? {};
+        this._authorize = opts.authorize;
+        this._batchSize = Math.max(1, opts.batchSize ?? 100);
+        this._maxRetries = Math.max(0, opts.maxRetries ?? 2);
+        this._backoffMs = Math.max(0, opts.retryBackoffMs ?? 200);
+        this._overfetch = Math.max(1, opts.residualOverfetch ?? 4);
+        this._textResolver = opts.textResolver;
+        this._fetch = opts.fetchImpl ?? ((...args) => fetch(...args));
+        this._sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    }
+
+    async upsert(records: readonly VectorRecord[]): Promise<UpsertResult> {
+        let upserted = 0;
+        let skipped = 0;
+        // Batched because every vendor caps request size, and a 10k-record ingest
+        // that fails at record 9,999 must not lose the first 9,998.
+        for (let i = 0; i < records.length; i += this._batchSize) {
+            const batch = records.slice(i, i + this._batchSize);
+            const json = await this._send(this.dialect.upsert(this._ctx, batch));
+            const result = this.dialect.parseUpsert(json, batch.length);
+            upserted += result.upserted;
+            skipped += result.skipped;
+        }
+        return { upserted, skipped };
+    }
+
+    async query(query: VectorQuery): Promise<VectorMatch[]> {
+        return this._search(query, (translated, topK) =>
+            this.dialect.query(this._ctx, {
+                ...(query.vector ? { vector: query.vector } : {}),
+                ...(query.text ? { text: query.text } : {}),
+                topK,
+                ...(translated !== undefined ? { filter: translated } : {}),
+                includeVectors: query.includeVectors ?? false,
+            }),
+        );
+    }
+
+    async keywordSearch(query: VectorQuery): Promise<VectorMatch[]> {
+        const text = query.text?.trim();
+        if (!text || !this.dialect.supportsKeywordSearch || !this.dialect.keywordSearch) return [];
+        const build = this.dialect.keywordSearch.bind(this.dialect);
+        return this._search(query, (translated, topK) =>
+            build(this._ctx, { text, topK, ...(translated !== undefined ? { filter: translated } : {}) }),
+        );
+    }
+
+    async fetch(ids: readonly string[]): Promise<VectorRecord[]> {
+        if (ids.length === 0) return [];
+        const json = await this._send(this.dialect.fetch(this._ctx, ids));
+        const records = this.dialect.parseRecords(json);
+        if (this.dialect.storesText || !this._textResolver) return records;
+
+        const texts = await this._textResolver(records.map((r) => r.id));
+        return records.map((r) => ({ ...r, text: texts.get(r.id) ?? r.text }));
+    }
+
+    async delete(ids: readonly string[]): Promise<number> {
+        if (ids.length === 0) return 0;
+        for (let i = 0; i < ids.length; i += this._batchSize) {
+            await this._send(this.dialect.deleteByIds(this._ctx, ids.slice(i, i + this._batchSize)));
+        }
+        return ids.length;
+    }
+
+    async deleteByFilter(filter: MetadataFilter, scope?: AccessScope): Promise<number> {
+        const effective = combineFilters(filter, scope) as MetadataFilter;
+        const translated = this.dialect.translateFilter(effective);
+
+        // Server-side bulk delete is only safe when the dialect can express the
+        // WHOLE predicate. A residual would mean deleting a superset — so the
+        // emulated path (query, filter locally, delete by id) is used instead.
+        if (this.dialect.supportsDeleteByFilter && this.dialect.deleteByFilter && !translated.residual) {
+            await this._send(this.dialect.deleteByFilter(this._ctx, translated.pushed));
+            return -1; // vendor bulk deletes do not report a count
+        }
+
+        const doomed = await this.query({ filter, ...(scope ? { scope } : {}), topK: 10_000 });
+        return this.delete(doomed.map((m) => m.id));
+    }
+
+    async count(filter?: MetadataFilter, scope?: AccessScope): Promise<number> {
+        const effective = combineFilters(filter, scope);
+        const translated = effective ? this.dialect.translateFilter(effective) : undefined;
+
+        if (this.dialect.supportsCount && this.dialect.count && this.dialect.parseCount && !translated?.residual) {
+            const json = await this._send(this.dialect.count(this._ctx, translated?.pushed));
+            return this.dialect.parseCount(json);
+        }
+        const matches = await this.query({ ...(filter ? { filter } : {}), ...(scope ? { scope } : {}), topK: 10_000 });
+        return matches.length;
+    }
+
+    /** Shared query path: translate, over-fetch for the residual, filter, hydrate. */
+    private async _search(
+        query: VectorQuery,
+        build: (translated: unknown, topK: number) => HttpRequest,
+    ): Promise<VectorMatch[]> {
+        const topK = Math.max(1, query.topK ?? 10);
+        const effective = combineFilters(query.filter, query.scope);
+        const translated = effective ? this.dialect.translateFilter(effective) : undefined;
+        const wanted = translated?.residual ? topK * this._overfetch : topK;
+
+        const json = await this._send(build(translated?.pushed, wanted));
+        let matches = this.dialect.parseMatches(json);
+
+        if (translated?.residual) {
+            const residual = translated.residual;
+            matches = matches.filter((m) => matchesFilter(m.metadata, residual));
+        }
+        if (query.minScore !== undefined) {
+            matches = matches.filter((m) => m.score >= (query.minScore as number));
+        }
+        matches = matches.slice(0, topK);
+
+        if (!this.dialect.storesText && this._textResolver && matches.length > 0) {
+            const texts = await this._textResolver(matches.map((m) => m.id));
+            matches = matches.map((m) => ({ ...m, text: texts.get(m.id) ?? m.text }));
+        }
+        return matches;
+    }
+
+    private async _send(request: HttpRequest): Promise<unknown> {
+        const dynamic = this._authorize ? await this._authorize() : {};
+        const headers = { 'Content-Type': 'application/json', ...this._headers, ...dynamic };
+        const url = `${this._baseUrl}${request.path}`;
+
+        let lastError: unknown;
+        for (let attempt = 0; attempt <= this._maxRetries; attempt++) {
+            if (attempt > 0) await this._sleep(this._backoffMs * 2 ** (attempt - 1));
+            try {
+                const res = await this._fetch(url, {
+                    method: request.method,
+                    headers,
+                    ...(request.body !== undefined ? { body: JSON.stringify(request.body) } : {}),
+                });
+                if (res.ok) return await res.json().catch(() => ({}));
+
+                const detail = await res.text().catch(() => '');
+                lastError = new Error(`${this.name} ${request.method} ${request.path} → ${res.status}: ${detail}`);
+                // 4xx other than rate-limiting is a request the server will reject
+                // identically on retry; failing fast keeps a bad filter from
+                // becoming a retry storm.
+                if (res.status !== 429 && res.status < 500) break;
+            } catch (err) {
+                lastError = err;
+            }
+        }
+        throw lastError instanceof Error ? lastError : new Error(String(lastError));
+    }
+}