@nacre.work/core 0.0.0

package/dist/vector/query.js

@@ -0,0 +1,94 @@
+/**
+ * Reciprocal Rank Fusion over a dense branch and a sparse one, then the caller
+ * reranks. Defaults follow docs/architecture.md: 50 candidates per branch.
+ */
+export function buildHybridQuery(options) {
+    const { branches, filter, limit, prefetchLimit = 50, withPayload = true } = options;
+    if (branches.length === 0) {
+        // An empty prefetch list is a query with no constraints on which points it
+        // considers, which is the same failure as a missing filter arriving by a
+        // different route. Refuse rather than emit it.
+        throw new Error('a hybrid query needs at least one branch');
+    }
+    return {
+        prefetch: branches.map((branch) => ({
+            query: branch.kind === 'dense'
+                ? [...branch.vector]
+                : { indices: [...branch.vector.indices], values: [...branch.vector.values] },
+            using: branch.using,
+            // The single source. Every branch gets it; there is no path that skips one.
+            filter,
+            limit: prefetchLimit,
+        })),
+        query: { fusion: 'rrf' },
+        limit,
+        with_payload: withPayload,
+    };
+}
+/**
+ * The payload indexes the permission filter needs.
+ *
+ * Without them the filter is evaluated by scanning, which does not fail a test
+ * — it just gets slow enough to matter at a customer's volume, which is the
+ * worst way to find out. `acl_tags` stays in memory because it participates in
+ * every single query.
+ */
+export const PAYLOAD_INDEXES = [
+    { field_name: 'layer_id', field_schema: 'uuid' },
+    { field_name: 'doc_id', field_schema: 'uuid' },
+    { field_name: 'org_id', field_schema: 'uuid' },
+    { field_name: 'deleted', field_schema: 'bool' },
+    {
+        field_name: 'acl_tags',
+        field_schema: { type: 'keyword', is_tenant: false, on_disk: false },
+    },
+];
+/**
+ * Collection configuration for one organization.
+ *
+ * int8 quantization is not a tuning knob: roughly 4x less memory for a recall
+ * loss inside one percent, which for corporate search is noise. Originals stay
+ * on disk for rescoring.
+ */
+export function collectionConfig(vectorName, size) {
+    return {
+        vectors: {
+            [vectorName]: {
+                size,
+                distance: 'Cosine',
+                hnsw_config: { m: 32, ef_construct: 256 },
+                quantization_config: {
+                    scalar: { type: 'int8', quantile: 0.99, always_ram: true },
+                },
+            },
+        },
+        sparse_vectors: { bm25: {} },
+        optimizers_config: { default_segment_number: 4 },
+        on_disk_payload: true,
+    };
+}
+/**
+ * The named vector for a model and dimension: `v_{model}_{dim}`.
+ *
+ * One function because there are three callers that must agree — the API and
+ * MCP search paths derive it from configuration, and a layer stores it in a
+ * column the worker reads when writing points. They disagreed: layers created
+ * through `POST /v1/layers` got the literal `default`, so the worker wrote to a
+ * vector the collection did not have and every upsert failed with `Bad
+ * Request`, while search looked for the configured name and would have found
+ * nothing even if the write had worked.
+ *
+ * A mismatch here is not a type error and not a test failure; it is an empty
+ * index. Deriving it in more than one place is what made that possible.
+ */
+export function vectorName(model, dimensions) {
+    return `v_${model.replace(/[^a-z0-9]/gi, '_')}_${dimensions}`;
+}
+/** `org_{slug}` — offboarding a tenant is then one delete, with no rows to forget. */
+export function collectionName(orgSlug) {
+    if (!/^[a-z0-9][a-z0-9_-]*$/i.test(orgSlug)) {
+        throw new Error(`refusing to build a collection name from: ${orgSlug}`);
+    }
+    return `org_${orgSlug.replace(/-/g, '_')}`;
+}
+//# sourceMappingURL=query.js.map

package/dist/vector/query.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"query.js","sourceRoot":"","sources":["../../vector/query.ts"],"names":[],"mappings":"AA2DA;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA2B;IAC1D,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,EAAE,EAAE,WAAW,GAAG,IAAI,EAAE,GAAG,OAAO,CAAA;IAEnF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,2EAA2E;QAC3E,yEAAyE;QACzE,+CAA+C;QAC/C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;IAC7D,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAClC,KAAK,EACH,MAAM,CAAC,IAAI,KAAK,OAAO;gBACrB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;gBACpB,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;YAChF,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,4EAA4E;YAC5E,MAAM;YACN,KAAK,EAAE,aAAa;SACrB,CAAC,CAAC;QACH,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;QACxB,KAAK;QACL,YAAY,EAAE,WAAW;KAC1B,CAAA;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE;IAChD,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;IAC9C,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;IAC9C,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,EAAE;IAC/C;QACE,UAAU,EAAE,UAAU;QACtB,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE;KACpE;CACO,CAAA;AAEV;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,UAAkB,EAAE,IAAY;IAC/D,OAAO;QACL,OAAO,EAAE;YACP,CAAC,UAAU,CAAC,EAAE;gBACZ,IAAI;gBACJ,QAAQ,EAAE,QAAiB;gBAC3B,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE;gBACzC,mBAAmB,EAAE;oBACnB,MAAM,EAAE,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;iBACpE;aACF;SACF;QACD,cAAc,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QAC5B,iBAAiB,EAAE,EAAE,sBAAsB,EAAE,CAAC,EAAE;QAChD,eAAe,EAAE,IAAI;KACtB,CAAA;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,UAAU,CAAC,KAAa,EAAE,UAAkB;IAC1D,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,UAAU,EAAE,CAAA;AAC/D,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6CAA6C,OAAO,EAAE,CAAC,CAAA;IACzE,CAAC;IACD,OAAO,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAA;AAC5C,CAAC"}

package/dist/vector/search.d.ts

@@ -0,0 +1,62 @@
+import { type QueryablePlan } from '../authz/filter.js';
+import { type Branch } from './query.js';
+export interface VectorStoreOptions {
+    readonly url: string;
+    readonly apiKey?: string;
+}
+export interface SearchRequest {
+    readonly orgId: string;
+    readonly orgSlug: string;
+    /** What the caller may reach. `none` is not accepted — see buildFilter. */
+    readonly plan: QueryablePlan;
+    readonly branches: readonly Branch[];
+    readonly topK: number;
+}
+export interface Hit {
+    readonly id: string;
+    readonly score: number;
+    readonly payload: Record<string, unknown>;
+}
+/**
+ * The vector store.
+ *
+ * One method matters: `search` takes an `AccessPlan` and never a filter. The
+ * filter is derived here, from the plan, and there is no way to pass one in —
+ * so no caller can assemble a query that reaches the index without going
+ * through the permission model first.
+ *
+ * `top_k` is passed straight through, uncorrected. Asking for more and trimming
+ * is a post-filter that also costs more, and it is the specific mistake
+ * invariant I2 is written against.
+ */
+export declare class VectorStore {
+    #private;
+    constructor(options: VectorStoreOptions);
+    ensureCollection(orgSlug: string, vectorName: string, size: number): Promise<void>;
+    /**
+     * Mark every point of a document deleted.
+     *
+     * A payload write, not a removal: physical deletion is the collector's job
+     * and runs on its own schedule. This is what actually takes the document out
+     * of results, because `deleted = false` is what the pre-filter tests — the
+     * Postgres tombstone alone changes nothing a query looks at.
+     *
+     * It lives on the search client rather than in the worker because the
+     * request that deletes a document is served by the API, and a document that
+     * is tombstoned in Postgres but not here stays searchable until a background
+     * job gets to it. That window is exactly what invariant I5 forbids.
+     */
+    tombstone(orgSlug: string, documentId: string): Promise<void>;
+    search(request: SearchRequest): Promise<readonly Hit[]>;
+    /**
+     * Re-check the tenant on the way out.
+     *
+     * Invariant I1 asks for this explicitly: the filter is checked once more when
+     * a response is serialized, and a mismatch is an error rather than a quiet
+     * drop. Filtering silently here would hide the bug that produced the row —
+     * and a bug that produces one cross-tenant row will produce more.
+     */
+    static assertTenant(orgId: string, hits: readonly Hit[]): readonly Hit[];
+    close(): Promise<void>;
+}
+//# sourceMappingURL=search.d.ts.map

package/dist/vector/search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../vector/search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAA;AACpE,OAAO,EAAuE,KAAK,MAAM,EAAE,MAAM,YAAY,CAAA;AAE7G,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAA;IACpC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,GAAG;IAClB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC1C;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,WAAW;;gBAGV,OAAO,EAAE,kBAAkB;IAWjC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBxF;;;;;;;;;;;;OAYG;IACG,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ7D,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC;IAiB7D;;;;;;;OAOG;IACH,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,EAAE,GAAG,SAAS,GAAG,EAAE;IAYlE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAI7B"}

package/dist/vector/search.js

@@ -0,0 +1,98 @@
+import { QdrantClient } from '@qdrant/js-client-rest';
+import { buildFilter } from '../authz/filter.js';
+import { buildHybridQuery, collectionConfig, collectionName, PAYLOAD_INDEXES } from './query.js';
+/**
+ * The vector store.
+ *
+ * One method matters: `search` takes an `AccessPlan` and never a filter. The
+ * filter is derived here, from the plan, and there is no way to pass one in —
+ * so no caller can assemble a query that reaches the index without going
+ * through the permission model first.
+ *
+ * `top_k` is passed straight through, uncorrected. Asking for more and trimming
+ * is a post-filter that also costs more, and it is the specific mistake
+ * invariant I2 is written against.
+ */
+export class VectorStore {
+    #client;
+    constructor(options) {
+        // exactOptionalPropertyTypes distinguishes "absent" from "present and
+        // undefined", and the client's parameters mean it: passing apiKey:
+        // undefined is not the same as omitting it.
+        this.#client = new QdrantClient(options.apiKey === undefined
+            ? { url: options.url }
+            : { url: options.url, apiKey: options.apiKey });
+    }
+    async ensureCollection(orgSlug, vectorName, size) {
+        const name = collectionName(orgSlug);
+        const existing = await this.#client.getCollections();
+        if (existing.collections.some((c) => c.name === name))
+            return;
+        await this.#client.createCollection(name, collectionConfig(vectorName, size));
+        // Created here rather than left to an operator: a filter that falls back to
+        // a scan is a latency problem that only shows up at a customer's volume.
+        for (const index of PAYLOAD_INDEXES) {
+            await this.#client.createPayloadIndex(name, {
+                field_name: index.field_name,
+                field_schema: index.field_schema,
+                wait: true,
+            });
+        }
+    }
+    /**
+     * Mark every point of a document deleted.
+     *
+     * A payload write, not a removal: physical deletion is the collector's job
+     * and runs on its own schedule. This is what actually takes the document out
+     * of results, because `deleted = false` is what the pre-filter tests — the
+     * Postgres tombstone alone changes nothing a query looks at.
+     *
+     * It lives on the search client rather than in the worker because the
+     * request that deletes a document is served by the API, and a document that
+     * is tombstoned in Postgres but not here stays searchable until a background
+     * job gets to it. That window is exactly what invariant I5 forbids.
+     */
+    async tombstone(orgSlug, documentId) {
+        await this.#client.setPayload(collectionName(orgSlug), {
+            wait: true,
+            payload: { deleted: true },
+            filter: { must: [{ key: 'doc_id', match: { value: documentId } }] },
+        });
+    }
+    async search(request) {
+        const filter = buildFilter(request.orgId, request.plan);
+        const query = buildHybridQuery({
+            branches: request.branches,
+            filter,
+            limit: request.topK,
+        });
+        const result = await this.#client.query(collectionName(request.orgSlug), query);
+        return result.points.map((p) => ({
+            id: String(p.id),
+            score: p.score,
+            payload: (p.payload ?? {}),
+        }));
+    }
+    /**
+     * Re-check the tenant on the way out.
+     *
+     * Invariant I1 asks for this explicitly: the filter is checked once more when
+     * a response is serialized, and a mismatch is an error rather than a quiet
+     * drop. Filtering silently here would hide the bug that produced the row —
+     * and a bug that produces one cross-tenant row will produce more.
+     */
+    static assertTenant(orgId, hits) {
+        for (const hit of hits) {
+            if (hit.payload.org_id !== orgId) {
+                throw new Error(`tenant mismatch on the way out: point ${hit.id} carries org_id ` +
+                    `${String(hit.payload.org_id)} for a request scoped to ${orgId}`);
+            }
+        }
+        return hits;
+    }
+    async close() {
+        // The REST client holds no pool; this exists so callers have one shape to
+        // depend on when it does.
+    }
+}
+//# sourceMappingURL=search.js.map

package/dist/vector/search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.js","sourceRoot":"","sources":["../../vector/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAErD,OAAO,EAAE,WAAW,EAAsB,MAAM,oBAAoB,CAAA;AACpE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,cAAc,EAAE,eAAe,EAAe,MAAM,YAAY,CAAA;AAsB7G;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,WAAW;IACb,OAAO,CAAc;IAE9B,YAAY,OAA2B;QACrC,sEAAsE;QACtE,mEAAmE;QACnE,4CAA4C;QAC5C,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAC7B,OAAO,CAAC,MAAM,KAAK,SAAS;YAC1B,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;YACtB,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CACjD,CAAA;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,EAAE,UAAkB,EAAE,IAAY;QACtE,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;QACpC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAA;QACpD,IAAI,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC;YAAE,OAAM;QAE7D,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,gBAAgB,CAAC,UAAU,EAAE,IAAI,CAAU,CAAC,CAAA;QAEtF,4EAA4E;QAC5E,yEAAyE;QACzE,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE;gBAC1C,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,YAAY,EAAE,KAAK,CAAC,YAAqB;gBACzC,IAAI,EAAE,IAAI;aACX,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,SAAS,CAAC,OAAe,EAAE,UAAkB;QACjD,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE;YACrD,IAAI,EAAE,IAAI;YACV,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;YAC1B,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,EAAE;SAC3D,CAAC,CAAA;IACb,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;QACvD,MAAM,KAAK,GAAG,gBAAgB,CAAC;YAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM;YACN,KAAK,EAAE,OAAO,CAAC,IAAI;SACpB,CAAC,CAAA;QAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,KAAc,CAAC,CAAA;QAExF,OAAO,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAA4B;SACtD,CAAC,CAAC,CAAA;IACL,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,YAAY,CAAC,KAAa,EAAE,IAAoB;QACrD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CACb,yCAAyC,GAAG,CAAC,EAAE,kBAAkB;oBAC/D,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,4BAA4B,KAAK,EAAE,CACnE,CAAA;YACH,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,KAAK,CAAC,KAAK;QACT,0EAA0E;QAC1E,0BAA0B;IAC5B,CAAC;CACF"}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@nacre.work/core",
+  "version": "0.0.0",
+  "description": "Data model, permission resolver, shared types",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/nacre-work/nacre.git",
+    "directory": "packages/core"
+  },
+  "homepage": "https://nacre.work",
+  "license": "Apache-2.0",
+  "type": "module",
+  "scripts": {
+    "build": "tsc -b && node ./scripts/copy-migrations.mjs",
+    "dev": "tsx watch index.ts"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@qdrant/js-client-rest": "^1.18.0",
+    "pg": "^8.22.0"
+  },
+  "devDependencies": {
+    "@types/node": "^26.1.2",
+    "jose": "^6.2.7"
+  }
+}