@nacre.work/api 0.0.0 → 0.2.0

package/dist/adapters.js

@@ -1,11 +1,37 @@
-import { aclTags, buildFilter, effectivePrincipals, loadGrants, loadScopeTree, PostgresGroupGraph, referenceAllows, resolve, VectorStore, vectorName, withOrg, } from '@nacre.work/core';
+import { aclTags, buildFilter, cachedEffectivePrincipals, documentKey, effectivePrincipals, fromStateJson, loadGrants, loadGroupsVersion, loadScopeTree, PostgresGroupGraph, referenceAllows, reindexProgress, resolve, toStateJson, VectorStore, vectorName, withOrg, } from '@nacre.work/core';
 import { createHash } from 'node:crypto';
+import { encodeCursor, pageOf } from './pagination.js';
+import { applyRanking } from './rerank.js';
 export class PostgresDocuments {
     pool;
+    payload;
     role;
-    constructor(pool, role) {
+    principalsCache;
+    presign;
+    constructor(pool, 
+    /**
+     * What writes metadata into the payload of a document's points.
+     *
+     * Narrow on purpose, like the delete path's tombstone port: this endpoint
+     * has no business searching, and a port that could would let a later change
+     * reach the index without a plan.
+     */
+    payload, role, 
+    /** See `principalsFor`. Absent means recompute the closure every time. */
+    principalsCache, 
+    /**
+     * Where a `source_url` comes from, when a deployment has object storage.
+     *
+     * Absent leaves the field off the response entirely, which is what every
+     * deployment without a bucket should see: a document whose bytes are in
+     * `documents.source_ref` has nothing to link to.
+     */
+    presign) {
         this.pool = pool;
+        this.payload = payload;
         this.role = role;
+        this.principalsCache = principalsCache;
+        this.presign = presign;
     }
     /**
      * Undefined for absent, for another organization's, and for one this caller
@@ -27,10 +53,11 @@ export class PostgresDocuments {
             // The plan first. Reading the row and then checking is the same
             // information leak with extra steps if the check is ever forgotten,
             // and the layer a document sits in is what the grant is on.
-            const plan = resolve(await contextFor(client, auth), 'read');
+            const plan = resolve(await contextFor(client, auth, this.principalsCache), 'read');
             if (plan.kind === 'none')
                 return undefined;
-            const { rows } = await client.query(`SELECT d.id, d.title, d.layer_id, l.slug AS layer, d.status, d.chunk_count, d.updated_at
+            const { rows } = await client.query(`SELECT d.id, d.title, d.layer_id, l.slug AS layer, d.status, d.chunk_count,
+                  d.updated_at, d.metadata, d.source_type, d.source_ref
              FROM documents d
              JOIN layers l ON l.id = d.layer_id AND l.org_id = d.org_id
             WHERE d.org_id = $1 AND d.id = $2 AND d.deleted_at IS NULL`, [orgId, documentId]);
@@ -54,53 +81,342 @@ export class PostgresDocuments {
                 status: row.status,
                 chunk_count: Number(row.chunk_count),
                 updated_at: row.updated_at.toISOString(),
+                // Read back, because a caller who tags a document and cannot see the
+                // tag has no way to tell a successful write from a dropped one — and
+                // this field was dropped by the handler for as long as it existed.
+                metadata: (row.metadata ?? {}),
+                // Minted here and nowhere earlier: everything above this line is the
+                // permission check, so a link exists only for a caller who has just
+                // been found to hold `read` on this document. `write` alone does not
+                // reach here — rule 6 — and neither does a denied document.
+                //
+                // Only for `s3`. An `inline` document has no object, and a `url` one
+                // is somebody else's address that this system has no business
+                // signing.
+                ...(this.presign !== undefined && row.source_type === 's3' && row.source_ref !== null
+                    ? { source_url: this.presign.url(row.source_ref) }
+                    : {}),
             };
         }, this.role === undefined ? {} : { role: this.role });
     }
+    /**
+     * Replace a document's metadata without re-embedding it.
+     *
+     * This is the cheap path `docs/api.md` said was not built. Ingest treats a
+     * metadata change as a change: the row and the vector payload would otherwise
+     * disagree and the document would carry a tag it does not answer to, so a
+     * re-tag cost a full re-parse, re-chunk and re-embed. Here the vectors are
+     * untouched and only the payload is rewritten — the same `setPayload` the ACL
+     * retag sweep uses, for the same reason.
+     *
+     * `write`, not `read`, and rule 6 means those are not the same set. The
+     * answer carries no body at all: a caller who may write to a document and not
+     * read it must not learn its title or its layer from a successful PATCH.
+     *
+     * Row first, then payload, both inside one transaction. A payload write that
+     * throws rolls the row back, so the two cannot end up disagreeing in the
+     * direction that matters — the row is what the worker rebuilds the payload
+     * from on the next index, so a row that ran ahead would be reverted while a
+     * payload that ran ahead would not.
+     */
+    async updateMetadata(auth, documentId, metadata) {
+        if (!/^[0-9a-f-]{36}$/i.test(documentId))
+            return false;
+        return withOrg(this.pool, auth.orgId, async (client) => {
+            const { rows: orgs } = await client.query('SELECT vector_collection FROM organizations WHERE id = $1 AND deleted_at IS NULL', [auth.orgId]);
+            const collection = orgs[0]?.vector_collection;
+            if (collection === undefined)
+                return false;
+            const plan = resolve(await contextFor(client, auth, this.principalsCache), 'write');
+            if (plan.kind === 'none')
+                return false;
+            const { rows } = await client.query(`SELECT layer_id FROM documents
+            WHERE org_id = $1 AND id = $2 AND deleted_at IS NULL`, [auth.orgId, documentId]);
+            const layerId = rows[0]?.layer_id;
+            if (layerId === undefined)
+                return false;
+            if (plan.kind === 'scoped') {
+                // deniedDocs first, as on the delete path: a deny beats an allow at
+                // any depth, and checking only `layers` would let a document inside
+                // an allowed layer be retagged despite an explicit deny on it.
+                if (plan.deniedDocs.includes(documentId))
+                    return false;
+                if (!plan.layers.includes(layerId) && !plan.extraDocs.includes(documentId))
+                    return false;
+            }
+            await client.query(`UPDATE documents SET metadata = $3, updated_at = now()
+            WHERE org_id = $1 AND id = $2`, [auth.orgId, documentId, JSON.stringify(metadata)]);
+            // `version` is deliberately not bumped. It counts revisions of the
+            // document's content, and a tag change is not one — bumping it would
+            // make every retag look like an edit to anything watching for one.
+            await this.payload.setMetadata(collection, documentId, metadata);
+            return true;
+        }, this.role === undefined ? {} : { role: this.role });
+    }
 }
 export class NacreSearchService {
     deps;
     constructor(deps) {
         this.deps = deps;
     }
-    async search(auth, query, topK) {
-        const slug = await this.deps.orgSlug(auth.orgId);
-        if (slug === undefined) {
+    async search(auth, query, topK, options = {}) {
+        const context = await withOrg(this.deps.pool, auth.orgId, async (client) => {
+            // The collection, from the column that owns it. Read in the transaction
+            // the plan needs anyway rather than through a cache: a reindex moves
+            // this pointer, and a cached one means the API keeps searching the
+            // collection the worker has stopped writing to. That failure returns
+            // results, just increasingly stale ones, which is the kind nobody
+            // notices.
+            const { rows: orgs } = await client.query('SELECT vector_collection FROM organizations WHERE id = $1 AND deleted_at IS NULL', [auth.orgId]);
+            const collection = orgs[0]?.vector_collection;
+            if (collection === undefined)
+                return undefined;
+            const principals = await principalsFor(client, auth, this.deps.principalsCache);
+            const grants = await loadGrants(client, auth.orgId, principals);
+            const tree = await loadScopeTree(client, auth.orgId, grants.filter((g) => g.scope.type === 'document').map((g) => g.scope.id));
+            const plan = resolve({ orgId: auth.orgId, role: auth.role, principals, grants, tree }, 'read');
+            // Every live layer with the model it is on. Small — an installation
+            // runs tens of layers per organization, not thousands — and it is what
+            // decides how many branches the query has.
+            const { rows: layers } = await client.query(`SELECT l.id, l.vector_name, p.id AS provider_id, p.endpoint, p.model, p.dimensions
+             FROM layers l
+             JOIN embedding_providers p ON p.id = l.provider_id
+            WHERE l.org_id = $1 AND l.deleted_at IS NULL`, [auth.orgId]);
+            return { collection, plan, layers };
+        }, this.deps.role === undefined ? {} : { role: this.deps.role });
+        if (context === undefined) {
             // The token names an organization that is not there. Rule I3: a
             // permission that cannot be evaluated denies.
             return [];
         }
-        const plan = await withOrg(this.deps.pool, auth.orgId, async (client) => {
-            const graph = await PostgresGroupGraph.load(client, auth.orgId);
-            const principals = effectivePrincipals(auth.principal, graph);
-            const grants = await loadGrants(client, auth.orgId, principals);
-            const tree = await loadScopeTree(client, auth.orgId, grants.filter((g) => g.scope.type === 'document').map((g) => g.scope.id));
-            return resolve({ orgId: auth.orgId, role: auth.role, principals, grants, tree }, 'read');
-        }, this.deps.role === undefined ? {} : { role: this.deps.role });
+        const { collection, plan } = context;
         // No access at all. Returning early rather than querying is not an
         // optimization — buildFilter refuses this plan, and it refuses it because
         // there is no filter that means "nothing" without meaning "everything" by
         // one mistake.
         if (plan.kind === 'none')
             return [];
-        const [vector] = await this.deps.embedder.embed([query]);
-        if (vector === undefined)
-            throw new Error('the embedder returned no vector for the query');
+        // The caller's own restriction, resolved before the query.
+        //
+        // `layers` was declared in openapi.yaml and in the MCP tool schema from the
+        // beginning and read by nothing, so a client scoping a search to one layer
+        // silently searched all of them. For a product whose selling point is that
+        // a search returns only what you may see, a scoping parameter that does
+        // nothing is the worst kind of no-op: the caller believes they narrowed it.
+        let narrow;
+        if (options.filters !== undefined && Object.keys(options.filters).length > 0) {
+            // Metadata alone is a valid narrowing — a caller may filter by source
+            // without naming a layer. `buildFilter` turns each entry into a `must`
+            // beside the permission constraint; there is no path from here to a
+            // filter the caller assembled.
+            narrow = { metadata: options.filters };
+        }
+        if (options.layers !== undefined && options.layers.length > 0) {
+            const resolved = await this.layerIds(auth.orgId, options.layers);
+            // Intersected with what the plan actually reaches, before deciding
+            // whether to query at all.
+            //
+            // The filter would already return nothing for a layer this caller cannot
+            // read — `must` on layer_id against a `should` that names other layers
+            // matches no point. Doing it here as well closes a narrower hole: without
+            // it, an unreadable-but-existing layer runs the query and an absent one
+            // does not, so anything that makes the query path *fail* — the vector
+            // store being down — separates the two. That is invariant I4 leaking
+            // through a dependency outage, and it costs six lines to remove. It also
+            // saves a round trip in the ordinary case.
+            //
+            // Not intersected when the plan carries `extraDocs`: those are documents
+            // granted individually and they can sit in any layer, including one the
+            // caller reaches no other way. Dropping such a layer here would answer
+            // empty where the correct answer is "the documents you were granted".
+            const permitted = plan.kind === 'scoped' && plan.extraDocs.length === 0
+                ? resolved.filter((id) => plan.layers.includes(id))
+                : resolved;
+            // Nothing the caller named is reachable. That covers a slug that does not
+            // exist and one that does but sits outside their grants — and it has to,
+            // because invariant I4 makes those the same answer. Empty results, no
+            // error naming the layer, and no query.
+            if (permitted.length === 0)
+                return [];
+            narrow = { ...narrow, layers: permitted };
+        }
+        // One branch per model actually in scope.
+        //
+        // Almost always exactly one: an organization runs a single embedding model
+        // and every layer is on it. Two while a reindex is part-way through, and
+        // two for as long as an operator keeps layers on different providers.
+        // Zero means the organization has no layers, or none the caller can reach,
+        // and there is nothing to ask the index.
+        const groups = this.groupsInScope(context.layers, plan, narrow);
+        if (groups.length === 0)
+            return [];
+        const branches = await Promise.all(groups.map(async (group) => {
+            const [vector] = await this.deps.embedderFor(group.provider).embed([query]);
+            if (vector === undefined) {
+                throw new Error(`the embedder for ${group.provider.model} returned no vector for the query`);
+            }
+            return {
+                kind: 'dense',
+                using: group.vectorName,
+                vector,
+                // Confined to the layers on this model. A reindexed layer keeps its
+                // old vector until the collection is rebuilt without it, so without
+                // this it would match the other model's branch too and fusion would
+                // rank it above everything for no reason but having been migrated.
+                //
+                // Only when there is more than one group: with a single model this
+                // clause is every layer in the organization, which is a longer filter
+                // that removes nothing.
+                ...(groups.length > 1 ? { onlyLayers: group.layerIds } : {}),
+            };
+        }));
+        // Off unless the deployment configured a reranker and the caller left it on.
+        const reranking = this.deps.reranker !== undefined && options.rerank !== false;
         const hits = await this.deps.vectors.search({
             orgId: auth.orgId,
-            orgSlug: slug,
+            collection,
             plan,
-            branches: [{ kind: 'dense', using: this.deps.vectorName, vector }],
-            // Passed through. Asking for more and trimming is a post-filter that also
-            // costs more.
-            topK,
+            branches,
+            // Without a reranker this is passed through uncorrected: asking for more
+            // and trimming would be a post-filter that also costs more.
+            //
+            // With one it is widened, and that is not the same thing. The trim a
+            // reranker performs is on relevance over a set the index has already
+            // filtered by permission, so it changes *which* permitted results come
+            // back and never how many. See rerank.ts — the distinction is the whole
+            // argument for this being allowed at all.
+            topK: reranking ? Math.max(topK, this.deps.rerankCandidates ?? 50) : topK,
+            // A `must` on layer_id inside the traversal, never a trim afterwards.
+            // Narrowing a search is still a pre-filter: `top_k` comes back full from
+            // the smaller set rather than being cut down from the larger one.
+            ...(narrow === undefined ? {} : { narrow }),
         });
         // Invariant I1, checked again on the way out. Raises rather than filtering:
         // silently dropping the row would hide the bug that produced it.
         const checked = VectorStore.assertTenant(auth.orgId, hits);
         if (checked.length === 0)
             return [];
-        return this.hydrate(auth.orgId, checked);
+        const candidates = await this.hydrate(auth.orgId, checked);
+        const ranked = reranking ? await this.rerank(query, candidates, topK) : candidates;
+        // Last, because reranking scores the query against the text. Dropping it
+        // earlier would rerank against nothing.
+        return options.includeContent === false ? ranked.map((hit) => ({ ...hit, text: '' })) : ranked;
+    }
+    /**
+     * The organization's layers, grouped by the model they are indexed with.
+     *
+     * Restricted first to what this search can actually reach — the permission
+     * plan, then the caller's own `layers` narrowing — so a query never embeds
+     * against a provider whose layers contribute nothing. That is not only a
+     * saving: an embedding endpoint that is down would otherwise fail a search
+     * over layers that have no connection to it.
+     *
+     * A plan carrying `extraDocs` keeps every group. Documents granted
+     * individually can sit in any layer, including one the caller reaches no
+     * other way, so dropping a group here would answer empty where the correct
+     * answer is "the documents you were granted".
+     */
+    groupsInScope(layers, plan, narrow) {
+        const reachable = layers.filter((l) => {
+            // Only the layer half narrows the branch set. A metadata restriction says
+            // nothing about which models are in scope: it removes points inside the
+            // query rather than removing layers from it.
+            if (narrow?.layers !== undefined && !narrow.layers.includes(l.id))
+                return false;
+            if (plan.kind === 'all')
+                return true;
+            if (plan.extraDocs.length > 0)
+                return true;
+            return plan.layers.includes(l.id);
+        });
+        const byVector = new Map();
+        for (const layer of reachable) {
+            // `vector_name` and `provider_id` have to agree, because the first says
+            // which slot to search and the second says which model to embed the query
+            // with. They disagreed once already — a reindex switched the vector and
+            // left the provider behind — and the symptom was Qdrant refusing the
+            // whole query on a dimension mismatch, which takes every *other* layer in
+            // the organization down with it. Named here rather than left to arrive as
+            // `Bad Request`, and raised rather than skipped: dropping the layer from
+            // the branch set would answer a search with a silently smaller corpus.
+            const expected = vectorName(layer.model, layer.dimensions);
+            if (expected !== layer.vector_name) {
+                throw new Error(`layer ${layer.id} names vector ${layer.vector_name} but its provider is ` +
+                    `${layer.model}/${layer.dimensions}, which is ${expected}. A reindex left the two ` +
+                    'out of step; the layer cannot be searched until they agree.');
+            }
+            const group = byVector.get(layer.vector_name) ?? {
+                provider: {
+                    id: layer.provider_id,
+                    endpoint: layer.endpoint,
+                    model: layer.model,
+                    dimensions: layer.dimensions,
+                },
+                layerIds: [],
+            };
+            group.layerIds.push(layer.id);
+            byVector.set(layer.vector_name, group);
+        }
+        return [...byVector].map(([vectorName, group]) => ({
+            vectorName,
+            provider: group.provider,
+            layerIds: group.layerIds,
+        }));
+    }
+    /**
+     * Layer slugs to ids, within one organization.
+     *
+     * Scoped through `withOrg` like every other tenant read, so a slug belonging
+     * to another organization resolves to nothing rather than to their layer. The
+     * result is intersected with what the caller may reach by the filter itself —
+     * the ids become a `must` and the permission constraint stays a `should`, so
+     * a layer this caller cannot read contributes no results even when it
+     * resolves here.
+     *
+     * Unknown slugs are dropped rather than reported. Answering "no such layer"
+     * would separate "does not exist" from "you cannot see it", which is exactly
+     * the distinction invariant I4 exists to prevent.
+     */
+    async layerIds(orgId, slugs) {
+        // Bounded, because this is an unauthenticated-shaped input on an
+        // authenticated path: an array of ten thousand slugs is one query with ten
+        // thousand parameters otherwise.
+        const wanted = [...new Set(slugs.map((s) => s.trim().toLowerCase()).filter((s) => s !== ''))].slice(0, 64);
+        if (wanted.length === 0)
+            return [];
+        return withOrg(this.deps.pool, orgId, async (client) => {
+            const { rows } = await client.query('SELECT id FROM layers WHERE org_id = $1 AND slug = ANY($2::text[])', [orgId, wanted]);
+            return rows.map((r) => r.id);
+        }, this.deps.role === undefined ? {} : { role: this.deps.role });
+    }
+    /**
+     * Reorder by cross-encoder score, or answer in fusion order and say so.
+     *
+     * The text comes from the hydration that had to happen anyway. A reranker
+     * scores query against text, and the vector store holds no text — so the
+     * order here is search, hydrate the candidates, rerank, cut. Hydrating fifty
+     * rows rather than ten is one query with a wider `ANY($2)`, which is the cost
+     * of the quality this buys.
+     *
+     * Failure answers with what fusion produced rather than raising. Reranking
+     * decides ordering over candidates that were all permitted before it saw
+     * them, so a reranker being down is a quality degradation and not a
+     * permissions one — and making the product's central operation depend on an
+     * optional model server would be the worse trade. It is reported, never
+     * silent.
+     */
+    async rerank(query, candidates, topK) {
+        const reranker = this.deps.reranker;
+        if (reranker === undefined)
+            return candidates.slice(0, topK);
+        try {
+            const scores = await reranker.rank(query, candidates.map((c) => c.text));
+            return applyRanking(candidates, scores, topK);
+        }
+        catch (error) {
+            this.deps.onRerankFailed?.(error);
+            return candidates.slice(0, topK);
+        }
     }
     /**
      * Turn hits into results a caller can read.
@@ -182,14 +498,22 @@ export class PostgresAudit {
     async write(event) {
         const [actorType, actorId] = event.actor.split(':');
         await withOrg(this.pool, event.orgId, async (client) => {
-            await client.query(`INSERT INTO audit_events
+            await client.query(
+            // `surface` and `target` were literals in this statement — 'api' and
+            // an empty object — so every MCP call was logged as REST and the
+            // `gin (target)` index built for this indexed nothing. The columns
+            // and the schema were right from the first migration; only the write
+            // was not.
+            `INSERT INTO audit_events
              (org_id, actor_type, actor_id, actor_label, action, surface, target, result, detail, request_id)
-           VALUES ($1,$2,$3,$4,$5,'api','{}'::jsonb,$6,$7,$8)`, [
+           VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8,$9,$10)`, [
                 event.orgId,
                 actorType ?? 'unknown',
                 /^[0-9a-f-]{36}$/i.test(actorId ?? '') ? actorId : null,
                 event.actor,
                 event.action,
+                event.surface ?? 'api',
+                JSON.stringify(event.target ?? {}),
                 event.result,
                 JSON.stringify(event.detail),
                 event.requestId,
@@ -208,18 +532,46 @@ export class HttpEmbedder {
     endpoint;
     model;
     dimensions;
-    constructor(endpoint, model, dimensions) {
+    timeoutMs;
+    constructor(endpoint, model, dimensions, 
+    /** One query, one vector, and a caller waiting on it. */
+    timeoutMs = 15_000) {
         this.endpoint = endpoint;
         this.model = model;
         this.dimensions = dimensions;
+        this.timeoutMs = timeoutMs;
+    }
+    /**
+     * One embedder per provider, built once and reused.
+     *
+     * Keyed on the provider's id, so an operator who edits an endpoint has to
+     * restart — which is true of every other piece of configuration here, and the
+     * alternative is a cache that has to be invalidated from a table nothing
+     * watches.
+     */
+    static pool(timeoutMs) {
+        const embedders = new Map();
+        return (provider) => {
+            const cached = embedders.get(provider.id);
+            if (cached !== undefined)
+                return cached;
+            const embedder = new HttpEmbedder(provider.endpoint, provider.model, provider.dimensions, ...(timeoutMs === undefined ? [] : [timeoutMs]));
+            embedders.set(provider.id, embedder);
+            return embedder;
+        };
     }
     async embed(texts) {
         if (texts.length === 0)
             return [];
+        // On the search path, so much tighter than the worker's: a caller is
+        // waiting. Without any bound a wedged embedder held every search open for
+        // undici's 300 s default, which exhausts the connection pool long before
+        // anyone sees an error.
         const response = await fetch(new URL('/embeddings', this.endpoint), {
             method: 'POST',
             headers: { 'content-type': 'application/json' },
             body: JSON.stringify({ model: this.model, input: texts }),
+            signal: AbortSignal.timeout(this.timeoutMs),
         });
         if (!response.ok) {
             throw new Error(`the embedding endpoint answered ${response.status}`);
@@ -255,8 +607,7 @@ export class NacreIngest {
     }
     /** The layers this caller may write to, by slug. */
     async writableLayer(client, auth, layerSlug) {
-        const graph = await PostgresGroupGraph.load(client, auth.orgId);
-        const principals = effectivePrincipals(auth.principal, graph);
+        const principals = await principalsFor(client, auth, this.deps.principalsCache);
         const grants = await loadGrants(client, auth.orgId, principals);
         const tree = await loadScopeTree(client, auth.orgId, grants.filter((g) => g.scope.type === 'document').map((g) => g.scope.id));
         const plan = resolve({ orgId: auth.orgId, role: auth.role, principals, grants, tree }, 'write');
@@ -277,12 +628,32 @@ export class NacreIngest {
             const layerId = await this.writableLayer(client, auth, request.layer);
             if (layerId === undefined)
                 return undefined;
-            const source = request.content !== undefined ? request.content : request.url;
-            const sourceType = request.content !== undefined ? 'inline' : 'url';
+            const inline = request.content !== undefined;
+            const source = inline ? request.content : request.url;
             const hash = createHash('sha256').update(source, 'utf8').digest('hex');
+            // Bytes to object storage before the row, never after.
+            //
+            // The reverse fails unrecoverably, which is the same argument the
+            // delete path is ordered by: a row that names an object which was never
+            // written is a document the worker fails on every attempt, forever,
+            // with the API having answered `queued`. A PUT with no row is a stray
+            // object at a deterministic key that the next ingest overwrites.
+            //
+            // Only for inline content. A `url` document is a reference already, and
+            // copying somebody else's URL into our bucket at ingest time would be
+            // a fetch on the request path — which is exactly what the worker does
+            // later, with a timeout and a sandbox.
+            let sourceType = inline ? 'inline' : 'url';
+            let sourceRef = source;
+            if (inline && this.deps.objects !== undefined) {
+                const key = documentKey(auth.orgId, layerId, request.externalId);
+                await this.deps.objects.put(key, new TextEncoder().encode(source), 'text/plain');
+                sourceType = 's3';
+                sourceRef = key;
+            }
             const { rows } = await client.query(`INSERT INTO documents
-             (org_id, layer_id, external_id, source_type, source_ref, title, content_hash, status)
-           VALUES ($1,$2,$3,$4,$5,$6,$7,'pending')
+             (org_id, layer_id, external_id, source_type, source_ref, title, content_hash, metadata, status)
+           VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'pending')
            ON CONFLICT (layer_id, external_id) DO UPDATE SET
              -- Only touch the row when the content actually changed. A repeat
              -- with the same bytes must not re-queue work: every client that
@@ -291,7 +662,19 @@ export class NacreIngest {
                                  THEN EXCLUDED.source_ref ELSE documents.source_ref END,
              title        = COALESCE(EXCLUDED.title, documents.title),
              content_hash = EXCLUDED.content_hash,
+             metadata     = EXCLUDED.metadata,
+             -- Metadata counts as a change, because it is written into the
+             -- vector payload of every point and a filter reads it from there.
+             -- Updating the row alone would leave the two disagreeing: the
+             -- document would carry a tag it does not answer to, which is the
+             -- silent shape of failure this repository keeps removing.
+             --
+             -- The cost is real — this re-embeds a document whose bytes did not
+             -- change — and the cheaper path is a payload-only write like the
+             -- ACL retag sweep does. Not built; docs/api.md says so rather than
+             -- letting an operator discover it from a bill.
              status       = CASE WHEN documents.content_hash IS DISTINCT FROM EXCLUDED.content_hash
+                                   OR documents.metadata IS DISTINCT FROM EXCLUDED.metadata
                                  THEN 'pending' ELSE documents.status END,
              deleted_at   = NULL,
              updated_at   = now()
@@ -300,9 +683,10 @@ export class NacreIngest {
                 layerId,
                 request.externalId,
                 sourceType,
-                source,
+                sourceRef,
                 request.title ?? null,
                 `sha256:${hash}`,
+                JSON.stringify(request.metadata),
             ]);
             const row = rows[0];
             if (row === undefined)
@@ -317,15 +701,16 @@ export class NacreIngest {
     async remove(auth, documentId) {
         if (!/^[0-9a-f-]{36}$/i.test(documentId))
             return false;
-        const slug = await this.deps.orgSlug(auth.orgId);
-        // The token names an organization that is not there. Refusing rather than
-        // deleting only in Postgres: a tombstone we cannot mirror into the index is
-        // a document that stays searchable while the API reports it gone.
-        if (slug === undefined)
-            return false;
         return withOrg(this.pool, auth.orgId, async (client) => {
-            const graph = await PostgresGroupGraph.load(client, auth.orgId);
-            const principals = effectivePrincipals(auth.principal, graph);
+            // The collection, from the column that owns it, inside the same
+            // transaction as the permission check. Refusing rather than deleting
+            // only in Postgres: a tombstone we cannot mirror into the index is a
+            // document that stays searchable while the API reports it gone.
+            const { rows: orgs } = await client.query('SELECT vector_collection FROM organizations WHERE id = $1 AND deleted_at IS NULL', [auth.orgId]);
+            const collection = orgs[0]?.vector_collection;
+            if (collection === undefined)
+                return false;
+            const principals = await principalsFor(client, auth, this.deps.principalsCache);
             const grants = await loadGrants(client, auth.orgId, principals);
             const tree = await loadScopeTree(client, auth.orgId, [documentId]);
             const plan = resolve({ orgId: auth.orgId, role: auth.role, principals, grants, tree }, 'write');
@@ -363,7 +748,7 @@ export class NacreIngest {
             // which purges rather than repairs. Index-first fails the other way: the
             // points are marked, the row is not, the caller sees an error and
             // retries, and in the meantime the document is already invisible.
-            await this.deps.tombstone.tombstone(slug, documentId);
+            await this.deps.tombstone.tombstone(collection, documentId);
             // A tombstone, not a delete, on this side too. Physical removal of the
             // points is the collector's job, and `deleted_at` is what puts the
             // document in its queue.
@@ -384,9 +769,13 @@ export class NacreIngest {
 export class PostgresJobs {
     pool;
     role;
-    constructor(pool, role) {
+    principalsCache;
+    constructor(pool, role, 
+    /** See `principalsFor`. Absent means recompute the closure every time. */
+    principalsCache) {
         this.pool = pool;
         this.role = role;
+        this.principalsCache = principalsCache;
     }
     async read(auth, jobId) {
         const orgId = auth.orgId;
@@ -397,7 +786,7 @@ export class PostgresJobs {
             // different projection — and needs the same check. The status and the
             // error string are both facts about a document the caller may not be
             // allowed to know exists.
-            const plan = resolve(await contextFor(client, auth), 'read');
+            const plan = resolve(await contextFor(client, auth, this.principalsCache), 'read');
             if (plan.kind === 'none')
                 return undefined;
             const { rows } = await client.query(`SELECT id, status, error, chunk_count, layer_id FROM documents
@@ -430,20 +819,52 @@ export class PostgresJobs {
         }, this.role === undefined ? {} : { role: this.role });
     }
 }
+async function principalsFor(client, auth, cache) {
+    if (cache === undefined) {
+        return effectivePrincipals(auth.principal, await PostgresGroupGraph.load(client, auth.orgId));
+    }
+    // In the same transaction as the grants that follow, so the key and the grant
+    // set describe one instant. Read separately, there is a window where the
+    // version says "fresh" about a set loaded before a change.
+    const groupsVersion = await loadGroupsVersion(client, auth.orgId);
+    return cachedEffectivePrincipals({
+        orgId: auth.orgId,
+        principal: auth.principal,
+        groupsVersion,
+        ttlSeconds: cache.ttlSeconds,
+    }, cache.store, () => PostgresGroupGraph.load(client, auth.orgId));
+}
 /** The per-request permission context, loaded once and asked several questions. */
-async function contextFor(client, auth) {
-    const graph = await PostgresGroupGraph.load(client, auth.orgId);
-    const principals = effectivePrincipals(auth.principal, graph);
+async function contextFor(client, auth, cache) {
+    const principals = await principalsFor(client, auth, cache);
     const grants = await loadGrants(client, auth.orgId, principals);
     const tree = await loadScopeTree(client, auth.orgId, grants.filter((g) => g.scope.type === 'document').map((g) => g.scope.id));
     return { orgId: auth.orgId, role: auth.role, principals, grants, tree };
 }
 export class PostgresLayers {
     pool;
+    vectors;
     role;
-    constructor(pool, role) {
+    principalsCache;
+    constructor(pool, 
+    /**
+     * Only to ask which named vectors the organization's collection has.
+     *
+     * A layer is created against a provider, and the provider decides which
+     * named vector the layer's points go into. Qdrant fixes that set when the
+     * collection is created and has no way to extend it, so a layer on a
+     * provider the collection has no slot for is a layer whose every document
+     * fails in the worker with `Not existing vector name` — after the API has
+     * answered 202 and the row says `pending`. Checking here is what turns that
+     * into a collection rebuild instead.
+     */
+    vectors, role, 
+    /** See `principalsFor`. Absent means recompute the closure every time. */
+    principalsCache) {
         this.pool = pool;
+        this.vectors = vectors;
         this.role = role;
+        this.principalsCache = principalsCache;
     }
     get scope() {
         return this.role === undefined ? {} : { role: this.role };
@@ -456,11 +877,11 @@ export class PostgresLayers {
      * break invariant I2 — but writing it the other way keeps one habit for both
      * and removes the question of which endpoints are allowed to be sloppy.
      */
-    async list(auth) {
+    async list(auth, page) {
         return withOrg(this.pool, auth.orgId, async (client) => {
-            const plan = resolve(await contextFor(client, auth), 'read');
+            const plan = resolve(await contextFor(client, auth, this.principalsCache), 'read');
             if (plan.kind === 'none')
-                return [];
+                return { items: [], nextCursor: null };
             // The count comes from the same statement. It is what a catalog is
             // for — an agent choosing where to search, or a person deciding
             // whether a layer is worth opening — and the MCP catalog has always
@@ -472,30 +893,45 @@ export class PostgresLayers {
             // grants exist — a commercial module — this becomes a number that
             // discloses more than the caller can reach, and it has to be recomputed
             // per caller or dropped.
-            const projection = `l.id, l.slug, l.name, l.workspace_id, l.description,
+            const projection = `l.id, l.slug, l.name, l.workspace_id, l.description, l.created_at,
               (SELECT count(*) FROM documents d
                 WHERE d.layer_id = l.id AND d.deleted_at IS NULL) AS document_count`;
+            // Ordered by (created_at, id) rather than by slug, because that is the
+            // cursor's sort key and a page has to be stable under inserts. Sorting
+            // by a mutable column would move rows between pages when one is renamed.
+            const order = 'ORDER BY l.created_at, l.id';
+            const after = page?.after;
+            const seek = after === undefined ? '' : ' AND (l.created_at, l.id) > ($3::timestamptz, $4::uuid)';
+            const cap = page === undefined ? '' : ` LIMIT ${page.limit}`;
             const { rows } = plan.kind === 'all'
                 ? await client.query(`SELECT ${projection} FROM layers l
-                  WHERE l.org_id = $1 AND l.deleted_at IS NULL ORDER BY l.slug`, [auth.orgId])
+                  WHERE l.org_id = $1 AND l.deleted_at IS NULL${seek.replace('$3', '$2').replace('$4', '$3')}
+                  ${order}${cap}`, after === undefined ? [auth.orgId] : [auth.orgId, after.createdAt, after.id])
                 : await client.query(`SELECT ${projection} FROM layers l
-                  WHERE l.org_id = $1 AND l.deleted_at IS NULL AND l.id = ANY($2::uuid[])
-                  ORDER BY l.slug`, [auth.orgId, [...plan.layers]]);
-            return rows.map((r) => ({
+                  WHERE l.org_id = $1 AND l.deleted_at IS NULL AND l.id = ANY($2::uuid[])${seek}
+                  ${order}${cap}`, after === undefined
+                    ? [auth.orgId, [...plan.layers]]
+                    : [auth.orgId, [...plan.layers], after.createdAt, after.id]);
+            const layers = rows.map((r) => ({
                 id: r.id,
                 slug: r.slug,
                 name: r.name,
                 workspaceId: r.workspace_id,
                 description: r.description ?? '',
                 documentCount: Number(r.document_count),
+                createdAt: r.created_at.toISOString(),
             }));
+            return pageOf(layers, page, (l) => ({ createdAt: l.createdAt, id: l.id }));
         }, this.scope);
     }
+    async update(auth, layerId, input) {
+        return updateLayer(this.pool, auth, layerId, input, this.role, this.principalsCache);
+    }
     async create(auth, input) {
         if (!/^[0-9a-f-]{36}$/i.test(input.workspaceId))
             return { kind: 'denied' };
         return withOrg(this.pool, auth.orgId, async (client) => {
-            const context = await contextFor(client, auth);
+            const context = await contextFor(client, auth, this.principalsCache);
             // `referenceAllows` and not `resolve`: the question is "may they
             // administer this one workspace", which is what the reference answers
             // directly. Reading it out of a flattened plan would mean inferring a
@@ -509,9 +945,35 @@ export class PostgresLayers {
             const { rows: workspaces } = await client.query(`SELECT id FROM workspaces WHERE org_id = $1 AND id = $2 AND deleted_at IS NULL`, [auth.orgId, input.workspaceId]);
             if (workspaces[0] === undefined)
                 return { kind: 'denied' };
-            const { rows: providers } = await client.query(`SELECT id, model, dimensions FROM embedding_providers
-            WHERE org_id = $1 OR org_id IS NULL ORDER BY org_id NULLS LAST LIMIT 1`, [auth.orgId]);
-            const provider = providers[0];
+            // The caller's choice, or the organization's if there is no ambiguity.
+            //
+            // `ORDER BY org_id NULLS LAST LIMIT 1` was the whole of this, which is
+            // fine while an organization has one provider and arbitrary the moment
+            // it has two — two rows with the same non-null `org_id` come back in
+            // whatever order the database felt like, so the model a layer was
+            // created against was a coin toss with no way for the caller to call
+            // it. Now they can, and where they do not, an ambiguous choice is
+            // refused rather than guessed.
+            const { rows: providers } = await client.query(`SELECT id, model, dimensions, (org_id IS NOT NULL) AS own
+             FROM embedding_providers
+            WHERE (org_id = $1 OR org_id IS NULL)
+              AND ($2::uuid IS NULL OR id = $2::uuid)
+            ORDER BY org_id NULLS LAST, id`, [auth.orgId, input.providerId ?? null]);
+            if (input.providerId !== undefined && providers[0] === undefined) {
+                return {
+                    kind: 'provider',
+                    detail: 'No embedding provider with that id is available to this organization.',
+                };
+            }
+            const own = providers.filter((p) => p.own);
+            if (input.providerId === undefined && own.length > 1) {
+                return {
+                    kind: 'provider',
+                    detail: `This organization has ${own.length} embedding providers, so which one a layer ` +
+                        "is indexed with is not implied. Name it with 'provider_id'.",
+                };
+            }
+            const provider = own[0] ?? providers[0];
             if (provider === undefined) {
                 throw new Error('no embedding provider is configured; a layer cannot be created without one');
             }
@@ -521,10 +983,46 @@ export class PostgresLayers {
             // collection does not have, which fails every upsert with `Bad
             // Request` and would return nothing even if it did not.
             const vector = vectorName(provider.model, provider.dimensions);
-            const { rows } = await client.query(`INSERT INTO layers (org_id, workspace_id, slug, name, provider_id, vector_name)
-           VALUES ($1,$2,$3,$4,$5,$6)
+            // Does the collection have room for this model?
+            //
+            // Usually yes: `init` creates the collection and the global provider
+            // from the same configuration, so every layer lands on the slot that is
+            // already there. An operator who adds a second provider is the case
+            // this exists for — `embedding_providers` has carried an `org_id` and a
+            // documented "NULL = global default" since the first migration, and a
+            // layer created against an organization's own provider used to accept
+            // documents and fail every one of them, forever, with the API reporting
+            // `queued` throughout.
+            const { rows: orgs } = await client.query('SELECT vector_collection FROM organizations WHERE id = $1 AND deleted_at IS NULL', [auth.orgId]);
+            const collection = orgs[0]?.vector_collection;
+            if (collection === undefined)
+                return { kind: 'denied' };
+            const present = await this.vectors.vectorsOf(collection);
+            // A copy in flight if the slot is missing. The layer is created either
+            // way — it is a real layer with a real provider — and the worker holds
+            // its documents at `pending` until the collection that can hold them
+            // exists. See `claimNext`.
+            //
+            // Reusing the reindex's own state rather than inventing a second
+            // mechanism: this *is* the copy phase of a reindex, with no documents
+            // to re-embed afterwards, so `finishCopy` and `finishReindexIfDone`
+            // close it out with nothing added.
+            const state = vector in present
+                ? null
+                : toStateJson({
+                    status: 'running',
+                    phase: 'copying',
+                    shadowVector: vector,
+                    providerId: provider.id,
+                    startedAt: new Date().toISOString(),
+                    total: 0,
+                    done: 0,
+                    failed: 0,
+                });
+            const { rows } = await client.query(`INSERT INTO layers (org_id, workspace_id, slug, name, provider_id, vector_name, reindex_state)
+           VALUES ($1,$2,$3,$4,$5,$6,$7)
            ON CONFLICT DO NOTHING
-           RETURNING id, slug, name, workspace_id, description`, [auth.orgId, input.workspaceId, input.slug, input.name, provider.id, vector]);
+           RETURNING id, slug, name, workspace_id, description, created_at`, [auth.orgId, input.workspaceId, input.slug, input.name, provider.id, vector, state]);
             const row = rows[0];
             // ON CONFLICT DO NOTHING returns nothing, and by here the caller has
             // already proved admin on the workspace — so the only way to get here
@@ -541,17 +1039,189 @@ export class PostgresLayers {
                     description: row.description ?? '',
                     // Just created, so this is a fact rather than a query.
                     documentCount: 0,
+                    createdAt: row.created_at.toISOString(),
+                },
+            };
+        }, this.scope);
+    }
+}
+/**
+ * Workspaces.
+ *
+ * The missing link in the API's own flow: creating a layer takes a
+ * `workspace_id`, and until this existed the only way to have one was the line
+ * `init` printed. `docs/quickstart.md` said as much — "the workspace id is the
+ * one `init` printed" — so a second administrator, a client library, or anyone
+ * who closed that terminal had no way forward that did not involve reading the
+ * database directly.
+ */
+export class PostgresWorkspaces {
+    pool;
+    role;
+    principalsCache;
+    constructor(pool, role, 
+    /** See `principalsFor`. Absent means recompute the closure every time. */
+    principalsCache) {
+        this.pool = pool;
+        this.role = role;
+        this.principalsCache = principalsCache;
+    }
+    get scope() {
+        return this.role === undefined ? {} : { role: this.role };
+    }
+    /**
+     * The workspaces this caller can reach, newest last, cursor-paged.
+     *
+     * Two ways to reach one, and the second is the reason this endpoint had to
+     * exist at all:
+     *
+     * 1. it holds a layer the read plan reaches — the ordinary case, and the same
+     *    narrowing the layer listing does;
+     * 2. the caller holds a grant on the workspace itself. An **empty** workspace
+     *    has no layers to be reached through, so without this an administrator
+     *    who had just been granted one could not see it, and therefore could not
+     *    create the first layer in it. That is the exact deadlock this endpoint
+     *    is here to break.
+     *
+     * Filtered in memory rather than in SQL because the second rule is the
+     * resolver's to answer, not a join's, and an organization has tens of
+     * workspaces rather than thousands. The listing is narrowed before it is
+     * paged — a page assembled first and filtered second would leak the count.
+     */
+    async list(auth, page) {
+        return withOrg(this.pool, auth.orgId, async (client) => {
+            const context = await contextFor(client, auth, this.principalsCache);
+            // No early return on `plan.kind === 'none'`, and that is the whole
+            // point of this endpoint.
+            //
+            // `resolve` flattens a grant set to the layers it reaches, so a
+            // principal whose only grant is `admin` on an **empty** workspace
+            // resolves to `none` — there are no layers for it to name. Bailing out
+            // here left exactly that caller unable to list the workspace they
+            // administer, and therefore unable to create the first layer in it,
+            // which is the deadlock this endpoint was added to break. It was
+            // written that way first and caught by running it.
+            const plan = resolve(context, 'read');
+            const { rows } = await client.query(`SELECT w.id, w.slug, w.name, w.created_at,
+                  (SELECT array_agg(l.id) FROM layers l
+                    WHERE l.workspace_id = w.id AND l.deleted_at IS NULL) AS layer_ids
+             FROM workspaces w
+            WHERE w.org_id = $1 AND w.deleted_at IS NULL
+            ORDER BY w.created_at, w.id`, [auth.orgId]);
+            const reachable = rows.filter((w) => {
+                if (plan.kind === 'all')
+                    return true;
+                if (plan.kind === 'scoped' &&
+                    (w.layer_ids ?? []).some((id) => plan.layers.includes(id))) {
+                    return true;
+                }
+                // The grant on the workspace itself, which `admin` satisfies for
+                // `read` — invariant 6's other half, and the only thing that answers
+                // for a workspace with nothing in it yet.
+                return referenceAllows(context, { type: 'workspace', id: w.id }, 'read');
+            });
+            const items = reachable.map((w) => ({
+                id: w.id,
+                slug: w.slug,
+                name: w.name,
+                layerCount: (w.layer_ids ?? []).length,
+                createdAt: w.created_at.toISOString(),
+            }));
+            return pageOf(items, page, (w) => ({ createdAt: w.createdAt, id: w.id }));
+        }, this.scope);
+    }
+    /**
+     * Create a workspace. `org_admin` only.
+     *
+     * There is no scope above a workspace to hold a grant on — `scope_type` is
+     * one of workspace, layer, document — so there is no one to ask except the
+     * role. `referenceAllows` would answer this with `role === 'org_admin'` and
+     * nothing else, so the check is written plainly rather than routed through a
+     * resolver call that can only return one thing.
+     *
+     * `platform_admin` is refused along with everyone else. Rule 2: it
+     * administers organizations and reads no documents, and a workspace is where
+     * documents live.
+     */
+    async create(auth, input) {
+        if (auth.role !== 'org_admin')
+            return { kind: 'denied' };
+        return withOrg(this.pool, auth.orgId, async (client) => {
+            const { rows } = await client.query(`INSERT INTO workspaces (org_id, slug, name) VALUES ($1,$2,$3)
+           ON CONFLICT DO NOTHING
+           RETURNING id, slug, name, created_at`, [auth.orgId, input.slug, input.name]);
+            const row = rows[0];
+            // ON CONFLICT DO NOTHING returns nothing, and by here the caller is
+            // org_admin — so the only way to get here is a slug already in use.
+            // 409 rather than 404, as on layer create: the caller has proved they
+            // administer the organization, so this is a fact about the resource
+            // rather than about what they can see.
+            if (row === undefined)
+                return { kind: 'conflict' };
+            return {
+                kind: 'created',
+                workspace: {
+                    id: row.id,
+                    slug: row.slug,
+                    name: row.name,
+                    layerCount: 0,
+                    createdAt: row.created_at.toISOString(),
                 },
             };
         }, this.scope);
     }
 }
+/**
+ * Rename a layer, or change what it says it is for.
+ *
+ * `admin` on the workspace, which is what creating one takes — the same
+ * question asked of the same resolver, so the two cannot drift into "who may
+ * create" and "who may rename" meaning different things.
+ *
+ * The description is user-facing copy: it is what the MCP tool's generated
+ * description is built from, so an agent's picture of what a layer holds
+ * changes with it. That is the reason it is editable at all.
+ */
+async function updateLayer(pool, auth, layerId, input, role, principalsCache) {
+    if (!/^[0-9a-f-]{36}$/i.test(layerId))
+        return false;
+    if (input.name === undefined && input.description === undefined)
+        return false;
+    return withOrg(pool, auth.orgId, async (client) => {
+        const context = await contextFor(client, auth, principalsCache);
+        // The layer's workspace, before anything is decided — the grant is on the
+        // workspace, so there is nothing to ask until we know which one.
+        const { rows } = await client.query(`SELECT workspace_id FROM layers
+          WHERE org_id = $1 AND id = $2 AND deleted_at IS NULL`, [auth.orgId, layerId]);
+        const workspaceId = rows[0]?.workspace_id;
+        if (workspaceId === undefined)
+            return false;
+        // `referenceAllows` on the workspace rather than a flattened plan, for
+        // the reason layer creation gives: the question is "may this caller
+        // administer this workspace", which the reference answers directly, and
+        // inferring it from the layers a plan happens to contain gets an empty
+        // workspace wrong.
+        if (!referenceAllows(context, { type: 'workspace', id: workspaceId }, 'admin'))
+            return false;
+        // COALESCE, so an absent field is left alone rather than blanked. PATCH
+        // changes what it names; a missing key is not an instruction to erase.
+        await client.query(`UPDATE layers
+            SET name        = COALESCE($3, name),
+                description = COALESCE($4, description)
+          WHERE org_id = $1 AND id = $2`, [auth.orgId, layerId, input.name ?? null, input.description ?? null]);
+        return true;
+    }, role === undefined ? {} : { role });
+}
 export class PostgresGrants {
     pool;
     role;
-    constructor(pool, role) {
+    principalsCache;
+    constructor(pool, role, 
+    /** See `principalsFor`. Absent means recompute the closure every time. */
+    principalsCache) {
         this.pool = pool;
         this.role = role;
+        this.principalsCache = principalsCache;
     }
     get scope() {
         return this.role === undefined ? {} : { role: this.role };
@@ -564,15 +1234,32 @@ export class PostgresGrants {
      * than none — an administrator who cannot see an existing grant revokes the
      * wrong thing.
      */
-    async list(auth) {
+    /**
+     * Paginated, and the page may be short.
+     *
+     * Which grants a caller may see is decided by `referenceAllows` against the
+     * scope tree, which SQL cannot express — so rows are fetched by cursor and
+     * filtered afterwards. That makes `items.length < limit` normal rather than a
+     * signal, and `next_cursor` the only thing that says whether more exist.
+     *
+     * The cursor comes from the last row **fetched**, not the last returned.
+     * Deriving it from the last returned row would skip everything the filter
+     * removed after it, which is a page of grants silently missing from an
+     * administrator's view of who can reach what.
+     */
+    async list(auth, page) {
         return withOrg(this.pool, auth.orgId, async (client) => {
-            const context = await contextFor(client, auth);
+            const context = await contextFor(client, auth, this.principalsCache);
             const plan = resolve(context, 'admin');
             if (plan.kind === 'none')
-                return [];
-            const { rows } = await client.query(`SELECT id, principal_type, principal_id, scope_type, scope_id, permission, effect, source
-             FROM grants WHERE org_id = $1 ORDER BY created_at`, [auth.orgId]);
-            return rows
+                return { items: [], nextCursor: null };
+            const after = page?.after;
+            const seek = after === undefined ? '' : ' AND (created_at, id) > ($2::timestamptz, $3::uuid)';
+            const cap = page === undefined ? '' : ` LIMIT ${page.limit}`;
+            const { rows } = await client.query(`SELECT id, principal_type, principal_id, scope_type, scope_id, permission, effect, source,
+                  created_at
+             FROM grants WHERE org_id = $1${seek} ORDER BY created_at, id${cap}`, after === undefined ? [auth.orgId] : [auth.orgId, after.createdAt, after.id]);
+            const visible = rows
                 .filter((r) => referenceAllows(context, { type: r.scope_type, id: r.scope_id }, 'admin'))
                 .map((r) => ({
                 id: r.id,
@@ -584,6 +1271,15 @@ export class PostgresGrants {
                 effect: r.effect,
                 source: r.source,
             }));
+            // From the last row fetched rather than the last returned. The filter
+            // above removes rows the caller may not administer, and taking the
+            // cursor from a survivor would skip every row the filter dropped after
+            // it — a page of grants silently missing from an administrator's view.
+            const lastFetched = rows[rows.length - 1];
+            const nextCursor = page !== undefined && rows.length >= page.limit && lastFetched !== undefined
+                ? encodeCursor({ createdAt: lastFetched.created_at.toISOString(), id: lastFetched.id })
+                : null;
+            return { items: visible, nextCursor };
         }, this.scope);
     }
     async issue(auth, input) {
@@ -591,7 +1287,7 @@ export class PostgresGrants {
             return undefined;
         }
         return withOrg(this.pool, auth.orgId, async (client) => {
-            const context = await contextFor(client, auth);
+            const context = await contextFor(client, auth, this.principalsCache);
             // Admin on the scope being granted, not admin in general. Otherwise
             // anyone holding admin on one layer could grant themselves another.
             if (!referenceAllows(context, { type: input.scopeType, id: input.scopeId }, 'admin')) {
@@ -650,7 +1346,7 @@ export class PostgresGrants {
             // because the object is a grant rather than a document.
             if (row === undefined)
                 return false;
-            const context = await contextFor(client, auth);
+            const context = await contextFor(client, auth, this.principalsCache);
             const scope = {
                 type: row.scope_type,
                 id: row.scope_id,
@@ -669,4 +1365,328 @@ export class PostgresGrants {
         }, this.scope);
     }
 }
+/**
+ * Reading the access log back.
+ *
+ * Scoped through `withOrg` like every other tenant read, so the row-level
+ * policy on `audit_events` is the second line of defence behind the role check
+ * in the handler. `SELECT` is the only grant the application role has on this
+ * table and that stays true — nothing here writes.
+ *
+ * ## Newest first, and what that does to the cursor
+ *
+ * Every other paged collection here reads oldest-first, because a catalog is a
+ * set and the order is arbitrary. A log is not: the useful end is the recent
+ * one, and an auditor paging forward from the beginning of a 400-day retention
+ * window to find yesterday is not a feature. So this orders
+ * `(occurred_at, id) DESC` and the seek predicate is `<` rather than `>`.
+ *
+ * The cursor still means "everything after the last row you saw", which is the
+ * same contract read in the other direction. It is worth being explicit because
+ * copying the seek clause from the layers adapter — where it is `>` — would
+ * produce a first page that works and a second page that silently returns the
+ * beginning of the log.
+ */
+export class PostgresAuditReader {
+    pool;
+    role;
+    principalsCache;
+    constructor(pool, role, 
+    /** See `principalsFor`. Absent means recompute the closure every time. */
+    principalsCache) {
+        this.pool = pool;
+        this.role = role;
+        this.principalsCache = principalsCache;
+    }
+    /**
+     * Actions that record a substantive access to a document's contents.
+     *
+     * Withheld from `platform_admin`, per rule 2 in docs/authz.md as
+     * docs/audit.md applies it to the journal. Named as a deny-list rather than
+     * an allow-list of administrative actions, and that is the uncomfortable
+     * direction: a new action defaults to *visible* to a platform administrator
+     * rather than hidden.
+     *
+     * Chosen anyway, because the alternative fails worse. An allow-list means a
+     * new administrative action is invisible to the operator who administers the
+     * installation until someone remembers to add it — a silent gap in an
+     * operational tool. This way a new *access* action is visible until someone
+     * adds it here, which is a disclosure to an already highly-privileged role
+     * within one installation. Both are bugs; only one of them is quiet.
+     *
+     * The `audit-event` skill's checklist is where this list is kept in step.
+     */
+    static DOCUMENT_ACCESS = ['search', 'document.read', 'document.get', 'chunk.read'];
+    async read(auth, query, page) {
+        return withOrg(this.pool, auth.orgId, async (client) => {
+            const where = ['org_id = $1'];
+            const params = [auth.orgId];
+            const bind = (value) => {
+                params.push(value);
+                return `$${params.length}`;
+            };
+            if (query.from !== undefined)
+                where.push(`occurred_at >= ${bind(query.from)}::timestamptz`);
+            if (query.to !== undefined)
+                where.push(`occurred_at < ${bind(query.to)}::timestamptz`);
+            if (query.actorId !== undefined)
+                where.push(`actor_id = ${bind(query.actorId)}::uuid`);
+            if (query.action !== undefined)
+                where.push(`action = ${bind(query.action)}`);
+            if (query.result !== undefined)
+                where.push(`result = ${bind(query.result)}`);
+            if (query.administrativeOnly === true) {
+                where.push(`action <> ALL(${bind(PostgresAuditReader.DOCUMENT_ACCESS)}::text[])`);
+            }
+            // Descending, and `<` to match. See the note on the class.
+            if (page.after !== undefined) {
+                where.push(`(occurred_at, id) < (${bind(page.after.createdAt)}::timestamptz, ${bind(page.after.id)}::bigint)`);
+            }
+            // One more than asked for, so "is there another page" is answered
+            // without a second query and without a count — a count over a
+            // retention window is the expensive thing this endpoint must not do on
+            // every request.
+            const { rows } = await client.query(`SELECT id::text, occurred_at, actor_type, actor_id, actor_label, action,
+                  surface, client, target, result, detail, request_id
+             FROM audit_events
+            WHERE ${where.join(' AND ')}
+            ORDER BY occurred_at DESC, id DESC
+            LIMIT ${page.limit + 1}`, params);
+            const items = rows.slice(0, page.limit).map((r) => ({
+                id: r.id,
+                occurredAt: r.occurred_at.toISOString(),
+                actorType: r.actor_type,
+                actorId: r.actor_id,
+                actorLabel: r.actor_label,
+                action: r.action,
+                surface: r.surface,
+                client: r.client,
+                target: r.target ?? {},
+                result: r.result,
+                detail: r.detail ?? {},
+                requestId: r.request_id,
+            }));
+            const last = items[items.length - 1];
+            const nextCursor = rows.length > page.limit && last !== undefined
+                ? encodeCursor({ createdAt: last.occurredAt, id: last.id })
+                : null;
+            return { items, nextCursor };
+        }, this.scopeFor());
+    }
+    scopeFor() {
+        return this.role === undefined ? {} : { role: this.role };
+    }
+}
+/**
+ * Starting a reindex, and reporting on one.
+ *
+ * The work is the worker's. This writes the intent — `layers.reindex_state` —
+ * counts what there is to do, and declares the new named vector on the
+ * collection.
+ *
+ * That last part belongs here and an earlier version of this comment said the
+ * opposite: that Qdrant creates a named vector when a point is first written
+ * with it, so the worker's first batch would bring it into existence. It does
+ * not. `updateVectors` against an undeclared name answers
+ * `Not existing vector name error`, and the reindex that assumed otherwise
+ * started cleanly, reported `running`, and failed every document forever with
+ * the progress gauge sitting at zero — which is step 1 of the sequence in
+ * docs/architecture.md being skipped, and the document has it first for a
+ * reason.
+ *
+ * Doing it here rather than in the worker also puts the failure where somebody
+ * is looking: an operator who cannot create the vector finds out from the
+ * response to their own request, not by watching a counter not move.
+ */
+export class PostgresReindex {
+    pool;
+    vectors;
+    role;
+    principalsCache;
+    constructor(pool, vectors, role, 
+    /** See `principalsFor`. Absent means recompute the closure every time. */
+    principalsCache) {
+        this.pool = pool;
+        this.vectors = vectors;
+        this.role = role;
+        this.principalsCache = principalsCache;
+    }
+    get scope() {
+        return this.role === undefined ? {} : { role: this.role };
+    }
+    async start(auth, layerId, providerId) {
+        return withOrg(this.pool, auth.orgId, async (client) => {
+            // Administering a layer, not reading it. A reindex rewrites every
+            // vector in the layer and changes which model answers every future
+            // query — `read` is nowhere near enough, and `write` is about putting
+            // documents in rather than about the layer itself.
+            const plan = resolve(await contextFor(client, auth, this.principalsCache), 'admin');
+            if (plan.kind === 'none')
+                return undefined;
+            // Locked for the whole decision. Two starts arriving together would
+            // both read "no reindex running" and both write a state, and the loser
+            // would leave a shadow vector nothing is filling.
+            const { rows: layers } = await client.query(`SELECT id, vector_name, reindex_state FROM layers
+            WHERE org_id = $1 AND id = $2 AND deleted_at IS NULL
+            FOR UPDATE`, [auth.orgId, layerId]);
+            const layer = layers[0];
+            if (layer === undefined)
+                return undefined;
+            if (plan.kind === 'scoped' && !plan.layers.includes(layer.id))
+                return undefined;
+            const running = fromStateJson(layer.reindex_state);
+            if (running?.status === 'running') {
+                // The live remaining count, not zero. A caller who collides with a
+                // running reindex is asking "how far has it got" as much as "why was
+                // I refused", and the answer they get should be the same one `GET`
+                // gives — not a placeholder that reads as finished.
+                const remaining = await this.remaining(client, auth.orgId, layerId, running.shadowVector);
+                return {
+                    kind: 'conflict',
+                    status: this.statusOf(layerId, layer.vector_name, running, remaining),
+                };
+            }
+            // The provider may be the organization's own or the installation-wide
+            // default, which is the one row with a NULL org_id. Same rule the
+            // policy on that table encodes; repeated here because a caller naming
+            // another tenant's provider must get "no such provider" rather than a
+            // reindex onto a model they cannot reach.
+            const { rows: providers } = await client.query(`SELECT model, dimensions FROM embedding_providers
+            WHERE id = $1 AND (org_id = $2 OR org_id IS NULL)`, [providerId, auth.orgId]);
+            const provider = providers[0];
+            if (provider === undefined)
+                return { kind: 'unknown_provider' };
+            const shadow = vectorName(provider.model, provider.dimensions);
+            if (shadow === layer.vector_name) {
+                return { kind: 'already_current', vectorName: shadow };
+            }
+            // Counted now, and it is a starting figure rather than a bound.
+            // Documents ingested during the reindex have no shadow vector either,
+            // so the same pass picks them up and `done` can pass `total` — which is
+            // why the progress ratio is clamped rather than trusted.
+            // Which half this layer starts in.
+            //
+            // A named vector cannot be added to a live Qdrant collection, so the
+            // slot has to already exist. If the organization's collection has it —
+            // because an earlier reindex made it — this layer goes straight to
+            // embedding. If not, the collection has to be rebuilt with room for it
+            // first, and that is org-wide work the worker does.
+            const { rows: orgs } = await client.query('SELECT vector_collection FROM organizations WHERE id = $1', [auth.orgId]);
+            const collection = orgs[0]?.vector_collection;
+            if (collection === undefined)
+                return undefined;
+            const present = await this.vectors.vectorsOf(collection);
+            const width = present[shadow];
+            if (width !== undefined && width !== provider.dimensions) {
+                // Same name, different width. The name is derived from model and
+                // dimensions so this cannot come from the API — it means the vector
+                // was made by hand, and writing 768 values into a 1024-wide slot is
+                // the kind of failure that surfaces months later as bad recall.
+                return { kind: 'unknown_provider' };
+            }
+            const phase = width === undefined ? 'copying' : 'embedding';
+            // One copy at a time for an organization. Two layers starting reindexes
+            // onto two different models would otherwise each rebuild the
+            // collection, and the second would throw away the first's work along
+            // with the pointer that made it live.
+            if (phase === 'copying') {
+                const { rows: busy } = await client.query(`SELECT count(*)::text AS n FROM layers
+              WHERE org_id = $1 AND deleted_at IS NULL
+                AND reindex_state ->> 'status' = 'running'
+                AND reindex_state ->> 'phase' = 'copying'`, [auth.orgId]);
+                if (Number(busy[0]?.n ?? 0) > 0) {
+                    const other = fromStateJson(layer.reindex_state) ?? {
+                        status: 'running',
+                        phase: 'copying',
+                        shadowVector: shadow,
+                        providerId,
+                        startedAt: new Date().toISOString(),
+                        total: 0,
+                        done: 0,
+                        failed: 0,
+                    };
+                    return { kind: 'conflict', status: this.statusOf(layerId, layer.vector_name, other, 0) };
+                }
+            }
+            const total = await this.remaining(client, auth.orgId, layerId, shadow);
+            const state = {
+                status: 'running',
+                phase,
+                shadowVector: shadow,
+                providerId,
+                startedAt: new Date().toISOString(),
+                total,
+                done: 0,
+                failed: 0,
+            };
+            await client.query('UPDATE layers SET reindex_state = $3::jsonb WHERE org_id = $1 AND id = $2', [auth.orgId, layerId, JSON.stringify(toStateJson(state))]);
+            // `remaining = total`: nothing has been reindexed yet, so `done` is 0.
+            // Passing 0 here said `done: total, progress: 1` on the 202 — a reindex
+            // reporting itself finished in the same breath as starting.
+            return {
+                kind: 'started',
+                status: this.statusOf(layerId, layer.vector_name, state, total),
+            };
+        }, this.scope);
+    }
+    async status(auth, layerId) {
+        return withOrg(this.pool, auth.orgId, async (client) => {
+            // `read` here, not `admin`. Knowing that the layer you can search is
+            // being migrated is not an administrative fact — it explains why a
+            // result set moved — and hiding it from someone who can already read
+            // the layer buys nothing.
+            const plan = resolve(await contextFor(client, auth, this.principalsCache), 'read');
+            if (plan.kind === 'none')
+                return undefined;
+            const { rows } = await client.query(`SELECT id, vector_name, reindex_state FROM layers
+            WHERE org_id = $1 AND id = $2 AND deleted_at IS NULL`, [auth.orgId, layerId]);
+            const layer = rows[0];
+            if (layer === undefined)
+                return undefined;
+            if (plan.kind === 'scoped' && !plan.layers.includes(layer.id))
+                return undefined;
+            const state = fromStateJson(layer.reindex_state);
+            if (state === undefined)
+                return undefined;
+            // The live remaining count rather than a stored `done`, because the
+            // worker never writes one — the number is derived, so there is nothing
+            // to go stale between batches.
+            const left = await this.remaining(client, auth.orgId, layerId, state.shadowVector);
+            return this.statusOf(layerId, layer.vector_name, state, left);
+        }, this.scope);
+    }
+    /**
+     * Live documents in the layer that do not yet carry this vector.
+     *
+     * The same predicate the worker's claim uses and the same one the switch is
+     * guarded by. Three places, one meaning — and if they ever disagree, the
+     * switch is the one that is right, because it is the only one inside the
+     * statement that performs it.
+     */
+    async remaining(client, orgId, layerId, shadowVector) {
+        const { rows } = await client.query(`SELECT count(*)::text AS n FROM documents
+        WHERE org_id = $1 AND layer_id = $2 AND deleted_at IS NULL AND chunk_count > 0
+          AND reindexed_vector IS DISTINCT FROM $3`, [orgId, layerId, shadowVector]);
+        return Number(rows[0]?.n ?? 0);
+    }
+    statusOf(layerId, currentVector, state, remaining) {
+        const done = state.status === 'complete' ? state.total : Math.max(0, state.total - remaining);
+        const live = { ...state, done };
+        return {
+            layerId,
+            status: state.status,
+            phase: state.phase,
+            shadowVector: state.shadowVector,
+            currentVector,
+            providerId: state.providerId,
+            startedAt: state.startedAt,
+            finishedAt: state.finishedAt ?? null,
+            total: state.total,
+            done,
+            failed: state.failed,
+            progress: reindexProgress(live),
+            error: state.error ?? null,
+        };
+    }
+}
 //# sourceMappingURL=adapters.js.map