@seanhogg/builderforce-memory 2026.7.12 → 2026.9.27

package/src/vectorstore/filter.ts

@@ -0,0 +1,148 @@
+/**
+ * vectorstore/filter.ts — the ONE evaluator and the ONE scope compiler.
+ *
+ * Two rules this file exists to enforce:
+ *
+ *   • A filter means the same thing everywhere. The in-process store, the
+ *     post-filter a remote adapter applies when its dialect cannot express a
+ *     clause, the ingest de-duplicator and the eval harness all call
+ *     {@link matchesFilter}. A second implementation would eventually disagree
+ *     about `null` or about a missing field, and a filter that disagrees with
+ *     itself across two code paths is a cross-tenant leak waiting for traffic.
+ *
+ *   • Access scope is compiled, never hand-written. {@link accessFilter} is the
+ *     single definition of "what may this caller see", so a new adapter or a new
+ *     query path inherits the rule instead of re-deriving it.
+ */
+
+import {
+    ACL_FIELD,
+    ACL_PUBLIC,
+    SENSITIVITY_FIELD,
+    TENANT_FIELD,
+    type AccessScope,
+    type MetadataFilter,
+    type MetadataRecord,
+    type MetadataValue,
+} from './types.js';
+
+/**
+ * Compiles an access scope into a filter.
+ *
+ * The ACL clause is `anyOf(acl, [...principals, '*'])`. Records are required to
+ * carry a non-empty `acl` (ingestion defaults it to `['*']`), which is what makes
+ * the rule fail CLOSED: a record written without an ACL matches nothing rather
+ * than matching everyone.
+ */
+export function accessFilter(scope: AccessScope): MetadataFilter {
+    const clauses: MetadataFilter[] = [
+        { op: 'eq', field: TENANT_FIELD, value: scope.tenantId },
+        { op: 'anyOf', field: ACL_FIELD, values: [...(scope.principals ?? []), ACL_PUBLIC] },
+    ];
+
+    if (scope.maxSensitivity !== undefined) {
+        clauses.push({ op: 'lte', field: SENSITIVITY_FIELD, value: scope.maxSensitivity });
+    }
+
+    return { op: 'and', filters: clauses };
+}
+
+/**
+ * ANDs a caller filter with a scope, dropping either when absent.
+ * Returns `undefined` only when neither is present — an unscoped query, which is
+ * legitimate for single-tenant use but never happens by accident, because the
+ * caller had to omit the scope explicitly.
+ */
+export function combineFilters(
+    filter?: MetadataFilter,
+    scope?: AccessScope,
+): MetadataFilter | undefined {
+    const scoped = scope ? accessFilter(scope) : undefined;
+    if (filter && scoped) return { op: 'and', filters: [scoped, filter] };
+    return filter ?? scoped;
+}
+
+/** Evaluates a filter against one record's metadata. */
+export function matchesFilter(metadata: MetadataRecord, filter?: MetadataFilter): boolean {
+    if (!filter) return true;
+
+    switch (filter.op) {
+        case 'and':
+            return filter.filters.every((f) => matchesFilter(metadata, f));
+        case 'or':
+            return filter.filters.some((f) => matchesFilter(metadata, f));
+        case 'not':
+            return !matchesFilter(metadata, filter.filter);
+        case 'exists':
+            return metadata[filter.field] !== undefined && metadata[filter.field] !== null;
+        case 'in':
+            return filter.values.some((v) => scalarEquals(metadata[filter.field], v));
+        case 'nin':
+            return !filter.values.some((v) => scalarEquals(metadata[filter.field], v));
+        case 'anyOf': {
+            const actual = metadata[filter.field];
+            if (actual === undefined || actual === null) return false;
+            const held = Array.isArray(actual) ? actual : [actual];
+            return held.some((h) => filter.values.some((v) => scalarEquals(h, v)));
+        }
+        case 'contains': {
+            const actual = metadata[filter.field];
+            if (typeof actual !== 'string') return false;
+            return actual.toLowerCase().includes(filter.value.toLowerCase());
+        }
+        default:
+            return compare(metadata[filter.field], filter.op, filter.value);
+    }
+}
+
+function scalarEquals(a: MetadataValue | undefined, b: MetadataValue): boolean {
+    if (Array.isArray(a) || Array.isArray(b)) {
+        return Array.isArray(a) && Array.isArray(b)
+            && a.length === b.length
+            && a.every((v, i) => v === b[i]);
+    }
+    return a === b;
+}
+
+function compare(
+    actual: MetadataValue | undefined,
+    op: 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte',
+    expected: MetadataValue,
+): boolean {
+    if (op === 'eq') return scalarEquals(actual, expected);
+    if (op === 'ne') return !scalarEquals(actual, expected);
+
+    // Ordering comparisons on a missing or non-scalar field are false rather than
+    // throwing: a record without the field simply does not satisfy `sensitivity <= n`.
+    if (actual === undefined || actual === null) return false;
+    if (Array.isArray(actual) || Array.isArray(expected) || expected === null) return false;
+    if (typeof actual !== typeof expected) return false;
+    if (typeof actual === 'boolean' || typeof expected === 'boolean') return false;
+
+    switch (op) {
+        case 'gt':  return actual > expected;
+        case 'gte': return actual >= expected;
+        case 'lt':  return actual < expected;
+        default:    return actual <= expected;
+    }
+}
+
+/**
+ * Walks a filter tree, reporting every field it touches. Adapters use this to
+ * decide up front whether a dialect can push the whole filter down, or whether a
+ * post-filter pass is needed.
+ */
+export function filterFields(filter: MetadataFilter, out = new Set<string>()): Set<string> {
+    switch (filter.op) {
+        case 'and':
+        case 'or':
+            for (const f of filter.filters) filterFields(f, out);
+            break;
+        case 'not':
+            filterFields(filter.filter, out);
+            break;
+        default:
+            out.add(filter.field);
+    }
+    return out;
+}

package/src/vectorstore/index.ts

@@ -0,0 +1,53 @@
+/**
+ * vectorstore — the VectorStore port, its declarative filter algebra, and the
+ * adapters. Tenant/ACL scope is compiled into every read, so multi-tenant RAG
+ * fails closed rather than depending on each caller remembering to scope.
+ */
+
+export { MemoryVectorStore } from './MemoryVectorStore.js';
+export type { MemoryVectorStoreOptions } from './MemoryVectorStore.js';
+
+export { RestVectorStore } from './RestVectorStore.js';
+export type { RestVectorStoreOptions } from './RestVectorStore.js';
+
+export {
+    getDialect,
+    listDialects,
+    registerDialect,
+    evermindDialect,
+    qdrantDialect,
+    pineconeDialect,
+    vertexAiDialect,
+} from './dialects.js';
+export type {
+    VectorDialect,
+    DialectContext,
+    HttpRequest,
+    TranslatedFilter,
+} from './dialects.js';
+
+export {
+    accessFilter,
+    combineFilters,
+    matchesFilter,
+    filterFields,
+} from './filter.js';
+
+export {
+    TENANT_FIELD,
+    ACL_FIELD,
+    SENSITIVITY_FIELD,
+    ACL_PUBLIC,
+} from './types.js';
+export type {
+    AccessScope,
+    ComparisonOp,
+    MetadataFilter,
+    MetadataRecord,
+    MetadataValue,
+    UpsertResult,
+    VectorMatch,
+    VectorQuery,
+    VectorRecord,
+    VectorStore,
+} from './types.js';

package/src/vectorstore/types.ts

@@ -0,0 +1,149 @@
+/**
+ * vectorstore/types.ts — the VectorStore port.
+ *
+ * Enterprise deployments do not get to choose the vector database; the customer
+ * already has one (pgvector in their Cloud SQL, Vertex AI Vector Search because
+ * they are a GCP shop, Qdrant because platform standardised on it). So the store
+ * is a PORT, and every adapter is infrastructure behind it. Retrieval, ingestion
+ * and evaluation depend on this interface and never on a vendor.
+ *
+ * Two things make it enterprise-grade rather than a toy vector index:
+ *
+ *   1. **Filters are DATA, not predicates.** A `(record) => boolean` cannot be
+ *      pushed down to a remote database — it forces fetch-everything-then-filter,
+ *      which is both slow and a data-leak: rows the caller may not see are read
+ *      into the process before being dropped. {@link MetadataFilter} is a small
+ *      declarative algebra each adapter translates into its own query language.
+ *
+ *   2. **Access scope is first-class.** Multi-tenant RAG fails closed only if the
+ *      tenant predicate is part of the query the database runs. {@link AccessScope}
+ *      compiles to a filter that is AND-ed into every read, so "forgot to scope the
+ *      query" is not a reachable state.
+ */
+
+export type MetadataValue = string | number | boolean | null | string[];
+
+export type MetadataRecord = Record<string, MetadataValue>;
+
+/** A stored unit of retrievable knowledge. */
+export interface VectorRecord {
+    id: string;
+    /** The chunk text — returned with matches so a retriever can rerank on it. */
+    text: string;
+    /** Embedding. Optional so a store can embed server-side. */
+    vector?: Float32Array;
+    metadata: MetadataRecord;
+}
+
+export interface VectorMatch {
+    id: string;
+    text: string;
+    /** Similarity in [0,1] — cosine for every adapter here. */
+    score: number;
+    metadata: MetadataRecord;
+    vector?: Float32Array;
+}
+
+// ── Filter algebra ────────────────────────────────────────────────────────────
+
+export type ComparisonOp = 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte';
+
+export type MetadataFilter =
+    | { op: ComparisonOp; field: string; value: MetadataValue }
+    /** Field equals any of `values` (scalar field). */
+    | { op: 'in' | 'nin'; field: string; values: MetadataValue[] }
+    /**
+     * Field (an array, or a scalar treated as a one-element array) intersects
+     * `values`. This is the ACL primitive — every mainstream vector database can
+     * express "array field matches any of", which is why ACLs are modelled as an
+     * array rather than as a bitmask or a nested object.
+     */
+    | { op: 'anyOf'; field: string; values: MetadataValue[] }
+    /** Case-insensitive substring match on a string field. */
+    | { op: 'contains'; field: string; value: string }
+    | { op: 'exists'; field: string }
+    // `and` and `or` are separate members rather than one `op: 'and' | 'or'`
+    // variant so that a discriminant check narrows them away cleanly; a union-typed
+    // discriminant is not removable by control flow, which forces casts on every
+    // dialect that walks the tree.
+    | { op: 'and'; filters: MetadataFilter[] }
+    | { op: 'or'; filters: MetadataFilter[] }
+    | { op: 'not'; filter: MetadataFilter };
+
+/**
+ * The non-composite half of the algebra — every variant that names a `field`.
+ * Dialect translation works leaf-by-leaf, so having the type say so removes a
+ * cast from every adapter.
+ */
+export type LeafFilter = Extract<MetadataFilter, { field: string }>;
+
+// ── Access scope ──────────────────────────────────────────────────────────────
+
+/** Metadata keys the access model reserves. Ingestion always populates them. */
+export const TENANT_FIELD = 'tenantId';
+export const ACL_FIELD = 'acl';
+export const SENSITIVITY_FIELD = 'sensitivity';
+/** ACL entry meaning "any principal within the tenant". */
+export const ACL_PUBLIC = '*';
+
+export interface AccessScope {
+    /** Hard tenant boundary. Every read is AND-ed with an equality on this. */
+    tenantId: string;
+    /**
+     * Group / role / user ids the caller holds. A record is visible when its `acl`
+     * array intersects these, or contains {@link ACL_PUBLIC}. Omit for
+     * tenant-wide access to public records only.
+     */
+    principals?: string[];
+    /**
+     * Highest classification level the caller may read (records carry a numeric
+     * `sensitivity`). Omit for no ceiling.
+     */
+    maxSensitivity?: number;
+}
+
+// ── The port ──────────────────────────────────────────────────────────────────
+
+export interface VectorQuery {
+    /** Dense query embedding. Required unless the adapter embeds server-side. */
+    vector?: Float32Array;
+    /** Raw query text — used by `keywordSearch`, and by server-side embedders. */
+    text?: string;
+    /** Results to return. Default 10. */
+    topK?: number;
+    /** Declarative metadata predicate, pushed down to the store. */
+    filter?: MetadataFilter;
+    /** Tenant/ACL scope, AND-ed with `filter`. */
+    scope?: AccessScope;
+    /** Drop matches scoring below this. Default 0. */
+    minScore?: number;
+    /** Return stored vectors on matches (costs bandwidth). Default false. */
+    includeVectors?: boolean;
+}
+
+export interface UpsertResult {
+    upserted: number;
+    /** Ids the store considered unchanged and skipped, when it can tell. */
+    skipped: number;
+}
+
+/**
+ * The port. `keywordSearch` is optional because not every vector database has a
+ * lexical index; retrievers degrade to dense-only and SAY SO on the span rather
+ * than silently returning worse results.
+ */
+export interface VectorStore {
+    /** Adapter name, stamped on telemetry spans. */
+    readonly name: string;
+
+    upsert(records: readonly VectorRecord[]): Promise<UpsertResult>;
+    query(query: VectorQuery): Promise<VectorMatch[]>;
+    delete(ids: readonly string[]): Promise<number>;
+    /** Bulk delete by predicate — how an incremental sync removes a stale source. */
+    deleteByFilter(filter: MetadataFilter, scope?: AccessScope): Promise<number>;
+    fetch(ids: readonly string[]): Promise<VectorRecord[]>;
+    count(filter?: MetadataFilter, scope?: AccessScope): Promise<number>;
+
+    /** Lexical/BM25 search, when the adapter supports one. */
+    keywordSearch?(query: VectorQuery): Promise<VectorMatch[]>;
+}