@nacre.work/core 0.0.0 → 0.2.0

package/dist/migrations/0012_prune.sql

@@ -0,0 +1,118 @@
+-- Two things that grew forever, and the decision about the second one.
+--
+-- ─── refresh tokens ───
+--
+-- `0009` said, in a comment on the index built for it: "Expired rows are
+-- deleted by a sweep rather than kept forever; ordering by expiry is what makes
+-- that sweep cheap." There was no sweep. Every login and every rotation inserts
+-- a row and nothing has ever removed one, so the table grows at the rate people
+-- sign in — and grows fastest on the deployments that rotate most often, which
+-- is to say the ones following the advice.
+--
+-- Nothing schema-side is needed for that: `nacre_worker` already holds DELETE
+-- on `refresh_tokens` from `0009`. The sweep is in the worker. It is mentioned
+-- here because this file is where somebody will come looking for it.
+--
+-- ─── the audit log, which is the actual decision ───
+--
+-- `NACRE_AUDIT_RETENTION_DAYS` has been validated at startup and read by
+-- nothing since it was written, and `docs/audit.md` recorded why: `0002`
+-- revokes DELETE on `audit_events` from `nacre_app` on purpose, so retention
+-- "has to run as the owner. That is a decision still to be made rather than a
+-- line still to be written."
+--
+-- This is the decision. Retention is implemented, and the append-only guarantee
+-- is not weakened, because the two are not in tension once you say what
+-- append-only is protecting: it stops history being *rewritten* — a specific
+-- inconvenient event erased, a result flipped from deny to allow. Expiring
+-- everything older than a published horizon is the opposite of that. It is
+-- indiscriminate by construction, it is announced in the configuration, and an
+-- auditor can compute exactly which window survives.
+--
+-- So the grant stays revoked and the pruning goes through one SECURITY DEFINER
+-- function whose *shape* is the guarantee, because its shape is the only part
+-- that holds against every caller:
+--
+--   * it takes a retention in days, never a predicate, so no caller can name a
+--     row, an actor, an organization, or a result;
+--   * it refuses a retention below 30 days, so it cannot be turned into "delete
+--     everything" or "delete this morning";
+--   * it deletes at most `max_rows` per call, so retention on a long-neglected
+--     table is many small transactions rather than one that holds locks for an
+--     hour;
+--   * it returns the count, so the worker can log what it did and a run that
+--     deleted a surprising number is visible rather than silent.
+--
+-- ─── what the EXECUTE grant below does and does not buy ───
+--
+-- It states intent and it stops third parties; it does **not** fence the
+-- application off, and claiming otherwise here would be worse than not saying
+-- it. `0008` grants `nacre_worker` to `nacre_app`, and membership inherits by
+-- default, so the API's role can call this function without so much as a
+-- SET ROLE. Verified rather than assumed — `SET ROLE nacre_app;
+-- SELECT prune_audit_events(400, 10)` succeeds.
+--
+-- That is fine, and it is fine for a reason worth writing down: the four
+-- properties above do not depend on who is calling. `nacre_app` reaching this
+-- function can expire a window past a published horizon, which the operator
+-- configured and an auditor can compute — and it still cannot erase a chosen
+-- event, which is the whole content of the append-only guarantee. Its direct
+-- DELETE stays revoked, so the arbitrary path remains closed.
+--
+-- Anyone with the owner's credentials can still do anything at all. That was
+-- already true and no function changes it.
+
+-- 30 days. Below this, retention stops being retention and becomes a way to
+-- make recent events go away, which is the thing append-only exists to prevent.
+-- Deliberately not configurable: a floor an operator can lower is not a floor.
+CREATE FUNCTION prune_audit_events(retention_days integer, max_rows integer)
+RETURNS integer
+LANGUAGE plpgsql
+SECURITY DEFINER
+-- Pinned, for the reason 0010 exists: an unqualified name in a definer function
+-- resolves through the caller's search_path, and any role may create objects in
+-- pg_temp. A shadowed `audit_events` here would be a caller choosing which rows
+-- a privileged function deletes.
+SET search_path = pg_catalog, public
+AS $$
+DECLARE
+    removed integer;
+BEGIN
+    IF retention_days < 30 THEN
+        RAISE EXCEPTION 'retention below the 30 day floor: %', retention_days
+            USING ERRCODE = 'invalid_parameter_value';
+    END IF;
+    IF max_rows < 1 OR max_rows > 100000 THEN
+        RAISE EXCEPTION 'max_rows out of range: %', max_rows
+            USING ERRCODE = 'invalid_parameter_value';
+    END IF;
+
+    -- By ctid over a bounded, ordered subquery: the index is
+    -- (org_id, occurred_at DESC), so an unbounded `DELETE ... WHERE
+    -- occurred_at < cutoff` on a table nobody has pruned in a year is one
+    -- statement scanning the lot and holding every row lock it takes until it
+    -- commits. Oldest first so repeated calls converge.
+    WITH doomed AS (
+        SELECT ctid
+          FROM audit_events
+         WHERE occurred_at < now() - make_interval(days => retention_days)
+         ORDER BY occurred_at
+         LIMIT max_rows
+    )
+    DELETE FROM audit_events a USING doomed d WHERE a.ctid = d.ctid;
+
+    GET DIAGNOSTICS removed = ROW_COUNT;
+    RETURN removed;
+END
+$$;
+
+-- The worker, and only the worker. `nacre_app` is the role on the process that
+-- serves requests; it has no business pruning anything, and 0002 revoked its
+-- DELETE for exactly that reason.
+REVOKE ALL ON FUNCTION prune_audit_events(integer, integer) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION prune_audit_events(integer, integer) TO nacre_worker;
+
+-- Expiry alone is not enough to find the sweep's rows cheaply once a family has
+-- been revoked for a month and a half: 0009's index is on `expires_at`, which
+-- is what the sweep orders by, and that one already exists. Nothing further is
+-- needed here — noted so the next person does not add a redundant index.

package/dist/migrations/0013_reindex.sql

@@ -0,0 +1,65 @@
+-- Reindexing a layer onto a different embedding model.
+--
+-- `layers.reindex_state` has been in the schema since 0001 with a comment
+-- describing its shape, and nothing has ever written it. This is what writes it,
+-- plus the one column the pass needs to know what is left to do.
+--
+-- ─── what a reindex is, mechanically ───
+--
+-- Chunks already hold their text and their Qdrant point id. So reindexing is
+-- not re-parsing anything: it is reading stored chunk text, embedding it with
+-- the new model, and adding a second **named vector** to the points that are
+-- already there. The old vector stays where it is and search keeps using it,
+-- which is what "search stays available throughout" in docs/architecture.md
+-- means in practice.
+--
+-- When every live document in the layer carries the new vector, `vector_name`
+-- switches in one statement and the next query uses it.
+--
+-- ─── one column, and why not dual-write ───
+--
+-- `reindexed_vector` records which named vector a document has most recently
+-- been reindexed onto. The pass selects documents in a reindexing layer where
+-- it is distinct from the shadow vector; the switch happens when that set is
+-- empty.
+--
+-- docs/architecture.md step 2 says "put the layer in dual-write: new documents
+-- are indexed by both models". This does not do that, and the deviation is
+-- deliberate. Dual-write means the ingest path carries reindex awareness
+-- permanently — a branch on layer state in the hot path, for a condition that
+-- is true for a few hours in a layer's life. Selecting on "lacks the shadow
+-- vector" gets the same guarantee out of machinery that already exists: a
+-- document ingested mid-reindex simply has no shadow vector yet, so the same
+-- pass picks it up, and the switch cannot happen while it is outstanding.
+--
+-- The resulting invariant is stronger and easier to check than dual-write's:
+-- **`vector_name` only ever changes when no live document in the layer lacks
+-- the new vector.** One predicate, verifiable in SQL, rather than a promise
+-- about two write paths staying in step.
+
+ALTER TABLE documents ADD COLUMN reindexed_vector text;
+
+-- Partial and on the layer, because the pass always asks the same question:
+-- "in this layer, which documents do not yet carry this vector". Live documents
+-- with chunks are the only ones that can — the rest have no points to add a
+-- vector to.
+CREATE INDEX documents_reindex_pending ON documents (layer_id, reindexed_vector)
+    WHERE deleted_at IS NULL AND chunk_count > 0;
+
+-- ─── the state, and the one shape it is allowed to take ───
+--
+-- A check constraint rather than a convention. `reindex_state` is jsonb, which
+-- means anything at all fits in the column, and the failure mode of a free-form
+-- status field is a typo that reads as "not running" — leaving a reindex that
+-- has stopped looking exactly like one that never started.
+--
+-- NULL means no reindex has ever run, which is different from one that finished
+-- and is why the column is nullable rather than defaulted.
+ALTER TABLE layers ADD CONSTRAINT layers_reindex_state_shape CHECK (
+    reindex_state IS NULL
+    OR (
+        reindex_state ? 'status'
+        AND reindex_state ->> 'status' IN ('running', 'complete', 'failed')
+        AND reindex_state ? 'shadow_vector'
+    )
+);

package/dist/migrations/0014_retired_collections.sql

@@ -0,0 +1,68 @@
+-- The collections a reindex leaves behind.
+--
+-- Every model migration builds a new collection, copies every point across, and
+-- moves `organizations.vector_collection` at the end. The old collection is
+-- still there afterwards, holding a full copy of the organization's vectors,
+-- and nothing has ever removed one. `runbooks/rollback-layer-reindex.md` says
+-- so in as many words — "каждая переиндексация оставляет коллекцию … ничего их
+-- не подчищает" — and hands the operator a shell loop.
+--
+-- That loop is also the wrong rule. It reads every collection Qdrant has,
+-- subtracts the ones an organization points at, and calls the difference
+-- orphaned. During a copy the *target* is exactly that: built, filling up, and
+-- not yet pointed at by anything. Running the documented cleanup while a
+-- migration is copying deletes the migration.
+--
+-- ─── so the record is kept, not derived ───
+--
+-- A row is written when the pointer moves, naming the collection it moved away
+-- from. Nothing else is ever a candidate for deletion. A copy in progress has
+-- no row, because the row is written by the same statement that supersedes it,
+-- so the dangerous case is not merely avoided — it is unrepresentable.
+--
+-- The delay is the point of the window, not an implementation detail: D2 in the
+-- rollback runbook is "move the pointer back", which is instant and only works
+-- while the old collection still exists. `NACRE_COLLECTION_RETENTION_DAYS` is
+-- how long that stays possible.
+--
+-- ─── what this does not cover ───
+--
+-- A copy that was abandoned — the reindex failed partway — leaves a target
+-- collection with no row and no pointer, and the sweep will not touch it. That
+-- is deliberate for now: the safe rule is "delete only what a successful
+-- migration superseded", and widening it to "delete anything unreferenced"
+-- brings back the race above. The runbook keeps its manual procedure for that
+-- case alone.
+
+CREATE TABLE retired_collections (
+    org_id     uuid        NOT NULL REFERENCES organizations (id) ON DELETE CASCADE,
+    -- The Qdrant collection name, which is the whole payload of a row. Not a
+    -- reference to anything: by the time this matters the collection may
+    -- already be gone, and a dangling name is the normal end state.
+    name       text        NOT NULL,
+    retired_at timestamptz NOT NULL DEFAULT now(),
+
+    -- One row per name per organization. A second migration away from the same
+    -- collection cannot happen — the pointer only ever leaves a name once — but
+    -- an idempotent insert is cheaper to reason about than a proof that it
+    -- cannot repeat.
+    PRIMARY KEY (org_id, name)
+);
+
+-- The sweep asks one question: which rows are older than the window. It asks it
+-- across organizations, so the index is on the timestamp alone.
+CREATE INDEX retired_collections_due ON retired_collections (retired_at);
+
+-- Tenant data, so RLS, and FORCE because ENABLE alone does not apply to the
+-- owner and migrations run as the owner.
+ALTER TABLE retired_collections ENABLE ROW LEVEL SECURITY;
+ALTER TABLE retired_collections FORCE  ROW LEVEL SECURITY;
+
+CREATE POLICY org_isolation ON retired_collections
+    USING (org_id = current_setting('app.current_org')::uuid);
+
+-- `nacre_app` inserts, because the pointer moves under `withOrg` in the worker's
+-- copy phase and that runs as the application role. `nacre_worker` deletes,
+-- because the sweep runs across organizations.
+GRANT SELECT, INSERT ON retired_collections TO nacre_app;
+GRANT SELECT, INSERT, DELETE ON retired_collections TO nacre_worker;

package/dist/oauth.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Protected resource metadata — RFC 9728.
+ *
+ * Every `401` from the MCP transport carries
+ * `WWW-Authenticate: Bearer resource_metadata="…/.well-known/oauth-protected-resource"`,
+ * and for as long as that header existed nothing served the path it named. A
+ * client doing exactly what the header told it to got a `404`, which is the
+ * same failure as a search parameter read by nothing: the server described a
+ * capability and did not have it.
+ *
+ * ## This is a resource server, not an authorization server
+ *
+ * `docs/mcp.md` says so and the code agrees: tokens are verified locally
+ * against a key, and nothing here issues one through an OAuth flow.
+ * `/v1/auth/login` is email and password, and a service account key is a random
+ * string matched against a hash — neither is an OAuth grant.
+ *
+ * So `authorization_servers` is **absent unless an operator names one**. The
+ * field is optional in RFC 9728, and an installation that has not put an
+ * identity provider in front of Nacre has no authorization server to name.
+ * Pointing the field at ourselves would be the same lie the 404 was, one
+ * redirect further along: a client would fetch our metadata, learn we are our
+ * own issuer, ask us for a token endpoint, and find nothing again.
+ *
+ * What the document is still worth without that field is the part clients
+ * actually need first: the canonical resource identifier every token must be
+ * audience-bound to, and where to read about the credential that does work
+ * today.
+ */
+export interface ProtectedResourceMetadata {
+    readonly resource: string;
+    readonly authorization_servers?: readonly string[];
+    readonly bearer_methods_supported: readonly string[];
+    readonly resource_documentation: string;
+    readonly scopes_supported: readonly string[];
+}
+/** The path RFC 9728 fixes, and the one the `WWW-Authenticate` header names. */
+export declare const PROTECTED_RESOURCE_PATH = "/.well-known/oauth-protected-resource";
+export declare function protectedResourceMetadata(input: {
+    /** `NACRE_CANONICAL_URL`. Baked into the issuer of every token ever minted. */
+    readonly canonicalUrl: string;
+    /** `NACRE_OAUTH_AUTHORIZATION_SERVER`, when a deployment has one. */
+    readonly authorizationServer?: string;
+}): ProtectedResourceMetadata;
+//# sourceMappingURL=oauth.d.ts.map

package/dist/oauth.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"oauth.d.ts","sourceRoot":"","sources":["../oauth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAClD,QAAQ,CAAC,wBAAwB,EAAE,SAAS,MAAM,EAAE,CAAA;IACpD,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAA;IACvC,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAA;CAC7C;AAED,gFAAgF;AAChF,eAAO,MAAM,uBAAuB,0CAA0C,CAAA;AAE9E,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,+EAA+E;IAC/E,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAA;CACtC,GAAG,yBAAyB,CAoB5B"}

package/dist/oauth.js

@@ -0,0 +1,53 @@
+/**
+ * Protected resource metadata — RFC 9728.
+ *
+ * Every `401` from the MCP transport carries
+ * `WWW-Authenticate: Bearer resource_metadata="…/.well-known/oauth-protected-resource"`,
+ * and for as long as that header existed nothing served the path it named. A
+ * client doing exactly what the header told it to got a `404`, which is the
+ * same failure as a search parameter read by nothing: the server described a
+ * capability and did not have it.
+ *
+ * ## This is a resource server, not an authorization server
+ *
+ * `docs/mcp.md` says so and the code agrees: tokens are verified locally
+ * against a key, and nothing here issues one through an OAuth flow.
+ * `/v1/auth/login` is email and password, and a service account key is a random
+ * string matched against a hash — neither is an OAuth grant.
+ *
+ * So `authorization_servers` is **absent unless an operator names one**. The
+ * field is optional in RFC 9728, and an installation that has not put an
+ * identity provider in front of Nacre has no authorization server to name.
+ * Pointing the field at ourselves would be the same lie the 404 was, one
+ * redirect further along: a client would fetch our metadata, learn we are our
+ * own issuer, ask us for a token endpoint, and find nothing again.
+ *
+ * What the document is still worth without that field is the part clients
+ * actually need first: the canonical resource identifier every token must be
+ * audience-bound to, and where to read about the credential that does work
+ * today.
+ */
+/** The path RFC 9728 fixes, and the one the `WWW-Authenticate` header names. */
+export const PROTECTED_RESOURCE_PATH = '/.well-known/oauth-protected-resource';
+export function protectedResourceMetadata(input) {
+    return {
+        // Without a trailing slash, because this is the audience value a token is
+        // bound to and `https://api.example.com` and `https://api.example.com/` are
+        // different strings to every audience check ever written.
+        resource: input.canonicalUrl.replace(/\/+$/, ''),
+        ...(input.authorizationServer === undefined
+            ? {}
+            : { authorization_servers: [input.authorizationServer.replace(/\/+$/, '')] }),
+        // Header only. A bearer token in a query string ends up in access logs,
+        // proxy logs and browser history, and RFC 6750 has deprecated that form for
+        // long enough that offering it is a liability rather than a courtesy.
+        bearer_methods_supported: ['header'],
+        resource_documentation: 'https://github.com/nacre-work/nacre/blob/main/docs/mcp.md',
+        // Nacre does not scope tokens. Permission is computed per call against the
+        // grant graph, so a token carries an organization and a role and never a
+        // list of what it may reach — which is why this is empty rather than
+        // inventing names no code reads. See docs/authz.md.
+        scopes_supported: [],
+    };
+}
+//# sourceMappingURL=oauth.js.map

package/dist/oauth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"oauth.js","sourceRoot":"","sources":["../oauth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAUH,gFAAgF;AAChF,MAAM,CAAC,MAAM,uBAAuB,GAAG,uCAAuC,CAAA;AAE9E,MAAM,UAAU,yBAAyB,CAAC,KAKzC;IACC,OAAO;QACL,0EAA0E;QAC1E,4EAA4E;QAC5E,0DAA0D;QAC1D,QAAQ,EAAE,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAChD,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,qBAAqB,EAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;QAC/E,wEAAwE;QACxE,4EAA4E;QAC5E,sEAAsE;QACtE,wBAAwB,EAAE,CAAC,QAAQ,CAAC;QACpC,sBAAsB,EAAE,2DAA2D;QACnF,2EAA2E;QAC3E,yEAAyE;QACzE,qEAAqE;QACrE,oDAAoD;QACpD,gBAAgB,EAAE,EAAE;KACrB,CAAA;AACH,CAAC"}

package/dist/observability.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observability.d.ts","sourceRoot":"","sources":["../observability.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAA;AAG9B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAE3C;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAoFtG"}
+{"version":3,"file":"observability.d.ts","sourceRoot":"","sources":["../observability.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAA;AAI9B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAE3C;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAkKtG"}

package/dist/observability.js

@@ -1,4 +1,5 @@
 import { withOrg } from './db/client.js';
+import { fromStateJson, reindexProgress } from './reindex.js';
 /**
  * The gauges that are queries rather than counters.
  *
@@ -18,20 +19,105 @@ export function collectDatabaseGauges(pool, metrics, role) {
             metrics.documents.reset();
             metrics.tombstonesPending.reset();
             metrics.aclPropagationLag.reset();
+            metrics.reindexProgress.reset();
+            metrics.collectionsRetired.reset();
+            // One series per layer that has ever been reindexed, labelled by slug
+            // like every other per-tenant gauge here. Layers with no reindex produce
+            // no row and therefore no series: inventing a zero for them would mean
+            // every layer in the installation permanently reading "reindex started,
+            // gone nowhere".
+            //
+            // Across organizations in one query rather than per tenant, because this
+            // reads `layers` and not `documents` — a handful of rows per tenant at
+            // most, and the whole point of the collapse in this collector was that a
+            // scrape must not cost a query per organization.
+            const { rows: reindexing } = await client.query(`SELECT o.slug, l.slug AS layer, l.reindex_state AS state
+           FROM layers l
+           JOIN organizations o ON o.id = l.org_id
+          WHERE l.reindex_state IS NOT NULL
+            AND l.deleted_at IS NULL AND o.deleted_at IS NULL`);
+            for (const row of reindexing) {
+                const state = fromStateJson(row.state);
+                if (state === undefined)
+                    continue;
+                metrics.reindexProgress.set(reindexProgress(state), { org: row.slug, layer: row.layer });
+            }
+            // Collections a finished migration is still holding, for the same reason
+            // and in the same shape as the reindex gauge above: one query across
+            // organizations, because `retired_collections` holds a handful of rows in
+            // total rather than a handful per tenant, and a scrape must not cost a
+            // query per organization.
+            //
+            // Only organizations that have one produce a series. A zero for everybody
+            // else would be true and useless — the number that matters is how much
+            // disk a completed migration has not given back, and the alertable shape
+            // is one that stops falling, which needs a series to exist at all rather
+            // than a floor under every tenant.
+            const { rows: retained } = await client.query(`SELECT o.slug, count(*) AS n
+           FROM retired_collections r
+           JOIN organizations o ON o.id = r.org_id
+          GROUP BY o.slug`);
+            for (const row of retained) {
+                metrics.collectionsRetired.set(Number(row.n), { org: row.slug });
+            }
             for (const org of orgs) {
-                const counts = await withOrg(pool, org.id, async (c) => (await c.query(`SELECT status, count(*)::text AS n FROM documents
-                  WHERE org_id = $1 AND deleted_at IS NULL GROUP BY status`, [org.id])).rows, scope);
-                for (const row of counts) {
-                    metrics.documents.set(Number(row.n), { org: org.slug, status: row.status });
-                }
-                const pending = await withOrg(pool, org.id, async (c) => (await c.query(`SELECT count(*)::text AS n FROM documents
-                  WHERE org_id = $1 AND deleted_at IS NOT NULL AND vectors_purged_at IS NULL`, [org.id])).rows[0]?.n ?? '0', scope);
-                metrics.tombstonesPending.set(Number(pending), { org: org.slug });
-                // The oldest document whose tags predate the current groups_version.
-                // Zero when everything is caught up; the age of the laggard otherwise.
-                const lag = await withOrg(pool, org.id, async (c) => (await c.query(`SELECT COALESCE(EXTRACT(EPOCH FROM (now() - min(COALESCE(acl_tagged_at, created_at)))), 0)::text AS lag
+                // One query per organization, not three.
+                //
+                // It was three — document counts, pending tombstones, propagation lag —
+                // each a separate round trip inside its own `withOrg`, so a scrape cost
+                // 3N+1 queries and each of those three also paid for a connection
+                // checkout and a `SET LOCAL app.current_org`. At five hundred tenants
+                // that is fifteen hundred queries every fifteen seconds, on the same
+                // pool the request path uses, triggered by an endpoint that anyone who
+                // can reach the port may call as fast as they like.
+                //
+                // All three read `documents` for one organization, so they are one
+                // grouped query with FILTERed aggregates. Still per organization and
+                // still inside `withOrg`: collapsing across tenants would mean reading
+                // every tenant's documents from the API process, and the row-level
+                // policies are not something to route around for a gauge.
+                const rows = await withOrg(pool, org.id, async (c) => (await c.query(`SELECT status,
+                        count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
+                        count(*) FILTER (
+                          WHERE deleted_at IS NOT NULL AND vectors_purged_at IS NULL
+                        )::text AS tombstoned,
+                        -- The oldest document whose tags predate the current
+                        -- groups_version. chunk_count > 0 is the whole
+                        -- correctness of this one, and it was missing once: the
+                        -- number is the age of the oldest document still
+                        -- carrying tags built from a superseded grant set, which
+                        -- only a document with points in the index can do.
+                        -- Without the clause a single failed or pending
+                        -- document pinned the gauge at its own age forever,
+                        -- because the retag loop claims neither and nothing else
+                        -- can clear it — the one metric with an alert on it
+                        -- became a stuck alert, which is how it gets muted.
+                        --
+                        -- Deliberately not status = 'indexed'. A document that
+                        -- indexed once and failed on a later pass still has the
+                        -- earlier pass's points and still carries their tags,
+                        -- and is exactly the case this is meant to catch.
+                        -- claimStale uses the same predicate; the two must
+                        -- agree, or the gauge counts what the loop cannot claim.
+                        min(COALESCE(acl_tagged_at, created_at)) FILTER (
+                          WHERE deleted_at IS NULL AND chunk_count > 0
+                            AND acl_version < $2
+                        )::text AS oldest_stale
                    FROM documents
-                  WHERE org_id = $1 AND deleted_at IS NULL AND acl_version < $2`, [org.id, Number(org.groups_version)])).rows[0]?.lag ?? '0', scope);
+                  WHERE org_id = $1
+                  GROUP BY status`, [org.id, Number(org.groups_version)])).rows, scope);
+                let tombstoned = 0;
+                let oldest;
+                for (const row of rows) {
+                    metrics.documents.set(Number(row.live), { org: org.slug, status: row.status });
+                    tombstoned += Number(row.tombstoned);
+                    if (row.oldest_stale !== null) {
+                        const at = Date.parse(row.oldest_stale);
+                        if (!Number.isNaN(at) && (oldest === undefined || at < oldest))
+                            oldest = at;
+                    }
+                }
+                metrics.tombstonesPending.set(tombstoned, { org: org.slug });
                 // Per organization, and set even when it is zero.
                 //
                 // A single worst-across-tenants number was the first shape of this and
@@ -44,7 +130,8 @@ export function collectDatabaseGauges(pool, metrics, role) {
                 // Zero is written rather than omitted: an absent series and a zero one
                 // mean "caught up" and "not being measured", and those must not look
                 // alike on the one metric that evidences I4.
-                metrics.aclPropagationLag.set(Number(lag), { org: org.slug });
+                const lag = oldest === undefined ? 0 : Math.max(0, (Date.now() - oldest) / 1000);
+                metrics.aclPropagationLag.set(lag, { org: org.slug });
             }
         }
         finally {

package/dist/observability.js.map

@@ -1 +1 @@
-{"version":3,"file":"observability.js","sourceRoot":"","sources":["../observability.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AAGxC;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAU,EAAE,OAAgB,EAAE,IAAa;IAC/E,MAAM,KAAK,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;IAEhD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACnC,IAAI,CAAC;YACH,uEAAuE;YACvE,wEAAwE;YACxE,yBAAyB;YACzB,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACvC,6EAA6E,CAC9E,CAAA;YAED,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;YACzB,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;YACjC,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;YAEjC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,MAAM,OAAO,CAC1B,IAAI,EACJ,GAAG,CAAC,EAAE,EACN,KAAK,EAAE,CAAC,EAAE,EAAE,CACV,CACE,MAAM,CAAC,CAAC,KAAK,CACX;2EAC2D,EAC3D,CAAC,GAAG,CAAC,EAAE,CAAC,CACT,CACF,CAAC,IAAI,EACR,KAAK,CACN,CAAA;gBACD,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;oBACzB,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC7E,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAC3B,IAAI,EACJ,GAAG,CAAC,EAAE,EACN,KAAK,EAAE,CAAC,EAAE,EAAE,CACV,CACE,MAAM,CAAC,CAAC,KAAK,CACX;6FAC6E,EAC7E,CAAC,GAAG,CAAC,EAAE,CAAC,CACT,CACF,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,GAAG,EACrB,KAAK,CACN,CAAA;gBACD,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;gBAEjE,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,GAAG,GAAG,MAAM,OAAO,CACvB,IAAI,EACJ,GAAG,CAAC,EAAE,EACN,KAAK,EAAE,CAAC,EAAE,EAAE,CACV,CACE,MAAM,CAAC,CAAC,KAAK,CACX;;gFAEgE,EAChE,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CACrC,CACF,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,GAAG,EACvB,KAAK,CACN,CAAA;gBACD,kDAAkD;gBAClD,EAAE;gBACF,uEAAuE;gBACvE,yEAAyE;gBACzE,yEAAyE;gBACzE,qEAAqE;gBACrE,mEAAmE;gBACnE,6CAA6C;gBAC7C,EAAE;gBACF,uEAAuE;gBACvE,qEAAqE;gBACrE,6CAA6C;gBAC7C,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YAC/D,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,OAAO,EAAE,CAAA;QAClB,CAAC;IACH,CAAC,CAAA;AACH,CAAC"}
+{"version":3,"file":"observability.js","sourceRoot":"","sources":["../observability.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAG7D;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAU,EAAE,OAAgB,EAAE,IAAa;IAC/E,MAAM,KAAK,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;IAEhD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACnC,IAAI,CAAC;YACH,uEAAuE;YACvE,wEAAwE;YACxE,yBAAyB;YACzB,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACvC,6EAA6E,CAC9E,CAAA;YAED,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;YACzB,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;YACjC,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;YACjC,OAAO,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;YAC/B,OAAO,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAA;YAElC,sEAAsE;YACtE,yEAAyE;YACzE,uEAAuE;YACvE,wEAAwE;YACxE,iBAAiB;YACjB,EAAE;YACF,yEAAyE;YACzE,uEAAuE;YACvE,yEAAyE;YACzE,iDAAiD;YACjD,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAK7C;;;;8DAIsD,CACvD,CAAA;YACD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;gBACtC,IAAI,KAAK,KAAK,SAAS;oBAAE,SAAQ;gBACjC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAA;YAC1F,CAAC;YAED,yEAAyE;YACzE,qEAAqE;YACrE,0EAA0E;YAC1E,uEAAuE;YACvE,0BAA0B;YAC1B,EAAE;YACF,0EAA0E;YAC1E,uEAAuE;YACvE,yEAAyE;YACzE,yEAAyE;YACzE,mCAAmC;YACnC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAC3C;;;0BAGkB,CACnB,CAAA;YACD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YAClE,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,yCAAyC;gBACzC,EAAE;gBACF,wEAAwE;gBACxE,wEAAwE;gBACxE,kEAAkE;gBAClE,sEAAsE;gBACtE,qEAAqE;gBACrE,uEAAuE;gBACvE,oDAAoD;gBACpD,EAAE;gBACF,mEAAmE;gBACnE,qEAAqE;gBACrE,uEAAuE;gBACvE,mEAAmE;gBACnE,0DAA0D;gBAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CACxB,IAAI,EACJ,GAAG,CAAC,EAAE,EACN,KAAK,EAAE,CAAC,EAAE,EAAE,CACV,CACE,MAAM,CAAC,CAAC,KAAK,CAMX;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kCA6BkB,EAClB,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CACrC,CACF,CAAC,IAAI,EACR,KAAK,CACN,CAAA;gBAED,IAAI,UAAU,GAAG,CAAC,CAAA;gBAClB,IAAI,MAA0B,CAAA;gBAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;oBAC9E,UAAU,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;oBACpC,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;wBAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;wBACvC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,GAAG,MAAM,CAAC;4BAAE,MAAM,GAAG,EAAE,CAAA;oBAC7E,CAAC;gBACH,CAAC;gBAED,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;gBAE5D,kDAAkD;gBAClD,EAAE;gBACF,uEAAuE;gBACvE,yEAAyE;gBACzE,yEAAyE;gBACzE,qEAAqE;gBACrE,mEAAmE;gBACnE,6CAA6C;gBAC7C,EAAE;gBACF,uEAAuE;gBACvE,qEAAqE;gBACrE,6CAA6C;gBAC7C,MAAM,GAAG,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAA;gBAChF,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YACvD,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,OAAO,EAAE,CAAA;QAClB,CAAC;IACH,CAAC,CAAA;AACH,CAAC"}

package/dist/passwords.d.ts

@@ -0,0 +1,37 @@
+/** Raised when the process is already verifying as many passwords as it will. */
+export declare class TooBusy extends Error {
+    constructor();
+}
+/** For a metric and for tests. Not a decision anything makes. */
+export declare const hashingLoad: () => {
+    active: number;
+    queued: number;
+    limit: number;
+};
+/**
+ * `scrypt$N$r$p$salt$hash`, base64url, parameters first.
+ *
+ * Self-describing on purpose. A bare hash cannot be verified after the cost is
+ * raised, so raising it means invalidating every password in the database;
+ * carrying the parameters means old hashes keep verifying and get rewritten as
+ * people sign in.
+ */
+export declare function hashPassword(password: string): Promise<string>;
+/**
+ * Verify, in time that does not depend on whether the password was right.
+ *
+ * A malformed or absent hash answers false rather than throwing: the caller
+ * turns every failure into the same refusal, and an exception on one of them
+ * would be a way to tell them apart from the outside.
+ */
+export declare function verifyPassword(password: string, encoded: string): Promise<boolean>;
+/**
+ * Whether a stored hash was made with parameters weaker than the current ones.
+ *
+ * The caller rehashes on a successful sign-in, which is the only moment the
+ * plaintext exists. Without this, raising the cost protects nobody who already
+ * has an account.
+ */
+export declare function needsRehash(encoded: string): boolean;
+export declare function spendVerificationTime(password: string): Promise<false>;
+//# sourceMappingURL=passwords.d.ts.map

package/dist/passwords.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"passwords.d.ts","sourceRoot":"","sources":["../passwords.ts"],"names":[],"mappings":"AA8FA,iFAAiF;AACjF,qBAAa,OAAQ,SAAQ,KAAK;;CAKjC;AAyCD,iEAAiE;AACjE,eAAO,MAAM,WAAW,QAAO;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAI5E,CAAA;AAEF;;;;;;;GAOG;AACH,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAWpE;AAmCD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAexF;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAIpD;AAgBD,wBAAsB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAI5E"}