@nacre.work/core 0.2.0 → 0.4.0

package/dist/migrations/0016_drop_acl_tag_cache.sql

@@ -0,0 +1,62 @@
+-- 0016 — remove the ACL tag cache.
+--
+-- `docs/authz.md` 3.6 specified `acl_tags` in the vector payload as a second
+-- filter alongside the layer bound: "both the layer filter and the tag filter
+-- apply". The filter was never built. `buildFilter` is the only filter builder
+-- in the codebase and it emits `org_id`, `deleted`, the plan's layers and
+-- documents as a `should`, a `must_not` on denied documents, and the caller's
+-- metadata narrowing — and no tag clause. So the whole subsystem kept a payload
+-- field fresh that no query read.
+--
+-- Two things settled it, and only the second is decisive.
+--
+-- **It saves nothing.** The tags were justified as a speed-up over "joining
+-- back to Postgres". Nothing joins back: the resolver computes the caller's
+-- whole permitted set from `grants` on every request, so the expensive part has
+-- already happened by the time a filter is built. A tag clause would be a
+-- second constraint derived from the same table, one request later.
+--
+-- **It cannot express what the product now sells.** `tagsForLayer` read
+-- `effect = 'allow'` grants on the layer and its workspace — document-scoped
+-- grants were invisible to it, and so were denies. Applied as the `must` the
+-- specification asks for, a caller reaching a document through a
+-- document-scoped grant would have been filtered *out*: their principal is not
+-- in that layer's tag set. Document-scoped grants are issuable now, through
+-- `@nacre.work/enterprise-acl-advanced`, so the specified design and a shipped
+-- feature could not both be true.
+--
+-- Making tags per-document would have fixed that and left the other cost: the
+-- tag filter is stale by construction, so a *newly granted* document would stay
+-- invisible until the sweep caught up. Revocation is already immediate — the
+-- plan is computed per request, and the effective-principals cache is keyed on
+-- `organizations.groups_version`, which bumps on every write to `grants` — so
+-- the trade was a delay on grants in exchange for nothing on revocations.
+--
+-- What goes with it: the retag sweep, its lease, the propagation gauge and its
+-- alert, `NACRE_ACL_TAG_HASH_BYTES` and `NACRE_ACL_PROPAGATION_SLA`. Invariant
+-- I4 is unchanged and is now argued structurally rather than measured
+-- temporally: there is no cache between a grant change and the next request.
+--
+-- `organizations.groups_version` stays. It is the effective-principals cache
+-- key, which is read on every request, and its triggers are what make that
+-- cache safe.
+
+ALTER TABLE documents DROP COLUMN IF EXISTS acl_version;
+ALTER TABLE documents DROP COLUMN IF EXISTS acl_tagged_at;
+
+-- Dropped with its column. Named explicitly because a partial index over a
+-- dropped column disappears silently, and a reader of 0004 should be able to
+-- find where it went.
+DROP INDEX IF EXISTS documents_acl_version_idx;
+
+-- `sweep_claimed_at` **stays**, and 0011's name for it is now misleading rather
+-- than wrong. It was added for the retag sweep and the collector took it as
+-- well: `claimPurgeable` sets it to lease a tombstone so two workers do not
+-- purge the same points, and `markPurged` clears it. Dropping it here removed a
+-- column the collector reads — caught by the garbage-collection tests against a
+-- real database, which is the only place it could have been caught.
+COMMENT ON COLUMN documents.sweep_claimed_at IS
+  'Lease held by the garbage collector while it purges a tombstone''s vectors. Cleared by markPurged, and reclaimed after NACRE_INDEX_LEASE if the worker holding it dies. Shared with the retag sweep until 0016 removed that sweep.';
+
+COMMENT ON COLUMN organizations.groups_version IS
+  'Permission epoch. Bumped by triggers on groups, group_members and grants. Read as the effective-principals cache key — which is what makes caching a permission input safe, since a change composes a different key rather than expiring an old one.';

package/dist/migrations/0017_drop_workspace_admin_role.sql

@@ -0,0 +1,30 @@
+-- 0017 — `users.role` loses `workspace_admin`.
+--
+-- 0001 wrote `CHECK (role IN ('platform_admin','org_admin','workspace_admin','member'))`
+-- and `OrgRole` in the code has always been three values. A row carrying the
+-- fourth read into a type that does not admit it and then behaved as `member`,
+-- because it matches neither role branch in `resolve` and falls through to
+-- grant-based evaluation. Safe — it denies rather than widens — and still a
+-- role the schema offered, nothing implemented, and nothing refused: an
+-- operator who set it got a silently demoted user and no error.
+--
+-- Removed rather than implemented, because it is a category error and not a
+-- missing feature. `users.role` is organization-wide; "administers a
+-- workspace" is scoped to one. The permission model already expresses it, and
+-- expresses it better: `admin` on a `workspace` scope, which a person can hold
+-- on several workspaces at once and which a single column cannot say. Adding
+-- the role would have created a second way to spell one thing, with the
+-- resolver having to decide which wins.
+--
+-- Existing rows are moved to `member`, which is what they already behaved as.
+-- The UPDATE runs before the constraint is replaced, or the constraint would
+-- refuse the rows it exists to remove.
+
+UPDATE users SET role = 'member' WHERE role = 'workspace_admin';
+
+ALTER TABLE users DROP CONSTRAINT IF EXISTS users_role_check;
+ALTER TABLE users ADD CONSTRAINT users_role_check
+    CHECK (role IN ('platform_admin', 'org_admin', 'member'));
+
+COMMENT ON COLUMN users.role IS
+  'Organization-wide role. Anything scoped to a workspace, a layer or a document is a grant, not a role — see docs/authz.md rules 2 and 3.';

package/dist/migrations/0018_group_members_nulls_not_distinct.sql

@@ -0,0 +1,46 @@
+-- 0018 — `group_members` can actually hold a member only once.
+--
+-- 0001 wrote `UNIQUE (group_id, member_user, member_group)` with a CHECK that
+-- exactly one of the two member columns is non-null. **In Postgres a unique
+-- index treats NULLs as distinct by default**, so two rows with the same
+-- `group_id` and `member_user` and a NULL `member_group` do not conflict — and
+-- every row this table holds has one of the two columns NULL, by construction.
+--
+-- The constraint therefore deduplicated nothing at all, in either direction.
+-- `ON CONFLICT DO NOTHING` against it is a no-op, which is exactly how it was
+-- found: a SCIM group sync adding the same member twice produced two rows and
+-- an idempotent re-sync doubled a group's membership every time it ran.
+--
+-- Not a permission bug. `effectivePrincipals` walks the graph into a `Set`, so
+-- a duplicated edge resolves to the same principal set and grants nothing
+-- extra. What it is: a table that grows without bound under a directory sync,
+-- a membership count that is wrong wherever one is shown, and — worse than
+-- either — a uniqueness guarantee the schema appears to make and does not, so
+-- the next writer reaches for `ON CONFLICT` and gets silence.
+--
+-- `NULLS NOT DISTINCT` needs Postgres 15; the Compose image is 17 and
+-- docs/config.md has always named 17. Dropping to a pair of partial unique
+-- indexes would work on older versions and is not worth carrying a version
+-- floor nothing else needs.
+--
+-- Duplicates are collapsed before the constraint is replaced, or the constraint
+-- refuses the rows it exists to prevent. `ctid` is the only thing that
+-- distinguishes them, which is what "duplicate" means here.
+
+DELETE FROM group_members a
+ USING group_members b
+ WHERE a.ctid > b.ctid
+   AND a.group_id = b.group_id
+   AND a.org_id = b.org_id
+   AND a.member_user IS NOT DISTINCT FROM b.member_user
+   AND a.member_group IS NOT DISTINCT FROM b.member_group;
+
+ALTER TABLE group_members
+    DROP CONSTRAINT IF EXISTS group_members_group_id_member_user_member_group_key;
+
+ALTER TABLE group_members
+    ADD CONSTRAINT group_members_member_unique
+    UNIQUE NULLS NOT DISTINCT (group_id, member_user, member_group);
+
+COMMENT ON CONSTRAINT group_members_member_unique ON group_members IS
+  'NULLS NOT DISTINCT, deliberately: exactly one of member_user and member_group is non-null on every row, so the default NULLS DISTINCT made this constraint match nothing.';

package/dist/migrations/0019_embedding_provider_write_guard.sql

@@ -0,0 +1,58 @@
+-- 0019 — a tenant may read the installation-wide embedding provider, never write it.
+--
+-- `embedding_providers` is the one table where a NULL `org_id` is legitimate: it
+-- is the installation-wide default model, and 0002's `org_isolation` lets every
+-- organization *read* it, on purpose. 0002 also carried this, two lines down:
+--
+--     -- Writing a global default is not something a tenant may do.
+--     CREATE POLICY org_writes_own ON embedding_providers
+--       FOR INSERT WITH CHECK (org_id = current_setting('app.current_org')::uuid);
+--
+-- and that policy does nothing. A PERMISSIVE policy is OR'd with the other
+-- PERMISSIVE policy for the same command, so on INSERT the effective check is
+-- `org_writes_own` OR `org_isolation` — and `org_isolation`'s check admits
+-- `org_id IS NULL`. `org_writes_own` is a strict subset of what was already
+-- allowed, so adding it changed nothing. It was verified inert against a real
+-- database: a connection scoped to org A, as `nacre_app`, could INSERT, UPDATE
+-- and DELETE the NULL-`org_id` row every other tenant reads as its default.
+--
+-- The consequence is the shape the whole schema's FORCE RLS exists to prevent,
+-- one table over from where it was checked: repoint that endpoint and every
+-- other tenant's documents are embedded by an attacker's service; delete it and
+-- every tenant relying on the default stops resolving a model. No core endpoint
+-- writes this table, so this is the second line of defence rather than a
+-- reachable exploit today — but the second line is exactly what a NULL-`org_id`
+-- row needs, because the first line (a handler setting `org_id` from the token)
+-- cannot cover a row whose `org_id` is meant to be NULL.
+--
+-- The fix is RESTRICTIVE policies on the three writing commands. A RESTRICTIVE
+-- policy is AND'd with the permissive result, so it can only ever remove access
+-- — it cannot widen a read. SELECT carries none, so the global row stays
+-- readable by every tenant exactly as before.
+
+DROP POLICY IF EXISTS org_writes_own ON embedding_providers;
+
+-- Each names the writing role's own organization and nothing else. `org_id`
+-- NULL fails the equality (NULL = uuid is NULL, not true), which is what keeps
+-- the global row out of a tenant's reach on every one of the three commands.
+CREATE POLICY tenant_writes_own_insert ON embedding_providers
+  AS RESTRICTIVE FOR INSERT
+  WITH CHECK (org_id = current_setting('app.current_org')::uuid);
+
+CREATE POLICY tenant_writes_own_update ON embedding_providers
+  AS RESTRICTIVE FOR UPDATE
+  USING (org_id = current_setting('app.current_org')::uuid)
+  WITH CHECK (org_id = current_setting('app.current_org')::uuid);
+
+CREATE POLICY tenant_writes_own_delete ON embedding_providers
+  AS RESTRICTIVE FOR DELETE
+  USING (org_id = current_setting('app.current_org')::uuid);
+
+-- The installation-wide default is still written once, by `init`, and it is a
+-- NULL-`org_id` row — so it now goes in under `nacre_worker`, the same
+-- BYPASSRLS role the worker's cross-tenant queue uses. `init` is the only
+-- writer of that row (a per-organization provider carries the org's own id and
+-- passes the policies above unchanged), which is why the exception is one
+-- statement in one bootstrap command rather than a hole in the policy.
+COMMENT ON POLICY tenant_writes_own_insert ON embedding_providers IS
+  'A tenant writes only its own providers. The NULL-org_id installation default is written by init under nacre_worker; see 0019.';

package/dist/migrations/0020_document_content_type.sql

@@ -0,0 +1,19 @@
+-- 0020 — what the stored bytes are, recorded where the truth lives.
+--
+-- Binary ingest (docs/architecture.md, "Binary ingest") stores a PDF's bytes
+-- in the bucket and needs the worker to know they are a PDF before it decides
+-- how to hand them to the parser. The S3 object carries a Content-Type of its
+-- own, but reading the answer from object metadata would make a bucket restore
+-- load-bearing for correctness — and Postgres is the source of truth, which is
+-- the whole restore doctrine. So the type is a column.
+--
+-- The default is the entire migration story: every existing document is UTF-8
+-- text, because until this landed the edge refused anything else. No backfill,
+-- no RLS change — the table's policies exist and a new column inherits them.
+
+ALTER TABLE documents
+    ADD COLUMN content_type text NOT NULL DEFAULT 'text/plain'
+    CHECK (content_type IN ('text/plain', 'application/pdf'));
+
+COMMENT ON COLUMN documents.content_type IS
+  'What the stored bytes are. text/plain lives inline or in the bucket as UTF-8; application/pdf lives only in the bucket. The CHECK is the list of formats ingest accepts — extending it is a migration, deliberately.';

package/dist/multipart.d.ts

@@ -0,0 +1,55 @@
+/**
+ * `multipart/form-data`, parsed strictly.
+ *
+ * `POST /v1/documents` has declared this in `docs/openapi.yaml` since before
+ * there was a server, and `docs/api.md` has listed it as not built for just as
+ * long. This is it.
+ *
+ * ─── why by hand, and what that obliges ───
+ *
+ * This parses attacker-supplied bytes. The usual answer is a library, and the
+ * usual argument against one applies here more than anywhere else in this
+ * repository: the request path of a document index is exactly where a
+ * dependency's parser bugs become yours. The parser sidecar makes the same
+ * choice for the same reason and says so.
+ *
+ * The obligation that comes with it is to be strict rather than clever. Every
+ * bound below is a refusal, not a truncation, and none of them is
+ * configurable — a caller who trips one gets `400` and a reason:
+ *
+ * - the boundary must match RFC 2046's grammar, so it cannot carry a regular
+ *   expression, a newline, or a thousand characters;
+ * - a bounded number of parts, so a body of ten thousand empty ones is a
+ *   refusal rather than ten thousand allocations;
+ * - a bounded header block per part, so a part that is all headers cannot be
+ *   scanned forever;
+ * - `filename` is read and never used to build anything. Object keys come from
+ *   `documentKey`, which hashes the external id; a filename reaches a database
+ *   column and nothing else.
+ *
+ * What it deliberately does not do: nested multipart, `Content-Transfer-
+ * Encoding`, and charsets other than UTF-8. Each is a real part of the
+ * specification and none is something a document upload needs, so each is a
+ * refusal rather than a branch.
+ */
+export declare class MultipartError extends Error {
+    constructor(message: string);
+}
+export interface MultipartPart {
+    readonly name: string;
+    /** Present when the part carried one. Never used to build a path or a key. */
+    readonly filename?: string;
+    readonly contentType?: string;
+    readonly bytes: Uint8Array;
+}
+/** Enough for a document plus its fields, and small enough to bound the work. */
+export declare const MAX_PARTS = 16;
+/** A part's headers. Anything longer is not a header block, it is an attack. */
+export declare const MAX_PART_HEADER_BYTES: number;
+/**
+ * The boundary from a `content-type`, or `undefined` when this is not
+ * multipart at all — which is not an error, it is how a caller says "JSON".
+ */
+export declare function multipartBoundary(contentType: string | undefined): string | undefined;
+export declare function parseMultipart(body: Uint8Array, boundary: string): readonly MultipartPart[];
+//# sourceMappingURL=multipart.d.ts.map

package/dist/multipart.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multipart.d.ts","sourceRoot":"","sources":["../multipart.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAA;CAC3B;AAED,iFAAiF;AACjF,eAAO,MAAM,SAAS,KAAK,CAAA;AAC3B,gFAAgF;AAChF,eAAO,MAAM,qBAAqB,QAAW,CAAA;AAK7C;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAqBrF;AAaD,wBAAgB,cAAc,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS,aAAa,EAAE,CA+E3F"}

package/dist/multipart.js

@@ -0,0 +1,158 @@
+/**
+ * `multipart/form-data`, parsed strictly.
+ *
+ * `POST /v1/documents` has declared this in `docs/openapi.yaml` since before
+ * there was a server, and `docs/api.md` has listed it as not built for just as
+ * long. This is it.
+ *
+ * ─── why by hand, and what that obliges ───
+ *
+ * This parses attacker-supplied bytes. The usual answer is a library, and the
+ * usual argument against one applies here more than anywhere else in this
+ * repository: the request path of a document index is exactly where a
+ * dependency's parser bugs become yours. The parser sidecar makes the same
+ * choice for the same reason and says so.
+ *
+ * The obligation that comes with it is to be strict rather than clever. Every
+ * bound below is a refusal, not a truncation, and none of them is
+ * configurable — a caller who trips one gets `400` and a reason:
+ *
+ * - the boundary must match RFC 2046's grammar, so it cannot carry a regular
+ *   expression, a newline, or a thousand characters;
+ * - a bounded number of parts, so a body of ten thousand empty ones is a
+ *   refusal rather than ten thousand allocations;
+ * - a bounded header block per part, so a part that is all headers cannot be
+ *   scanned forever;
+ * - `filename` is read and never used to build anything. Object keys come from
+ *   `documentKey`, which hashes the external id; a filename reaches a database
+ *   column and nothing else.
+ *
+ * What it deliberately does not do: nested multipart, `Content-Transfer-
+ * Encoding`, and charsets other than UTF-8. Each is a real part of the
+ * specification and none is something a document upload needs, so each is a
+ * refusal rather than a branch.
+ */
+export class MultipartError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'MultipartError';
+    }
+}
+/** Enough for a document plus its fields, and small enough to bound the work. */
+export const MAX_PARTS = 16;
+/** A part's headers. Anything longer is not a header block, it is an attack. */
+export const MAX_PART_HEADER_BYTES = 8 * 1024;
+/** RFC 2046: 1-70 characters from a fixed set, not ending in a space. */
+const BOUNDARY = /^[A-Za-z0-9'()+_,\-./:=?]([A-Za-z0-9'()+_,\-./:=? ]{0,68}[A-Za-z0-9'()+_,\-./:=?])?$/;
+/**
+ * The boundary from a `content-type`, or `undefined` when this is not
+ * multipart at all — which is not an error, it is how a caller says "JSON".
+ */
+export function multipartBoundary(contentType) {
+    if (contentType === undefined)
+        return undefined;
+    const [type, ...rest] = contentType.split(';');
+    if ((type ?? '').trim().toLowerCase() !== 'multipart/form-data')
+        return undefined;
+    for (const parameter of rest) {
+        const eq = parameter.indexOf('=');
+        if (eq === -1)
+            continue;
+        if (parameter.slice(0, eq).trim().toLowerCase() !== 'boundary')
+            continue;
+        let value = parameter.slice(eq + 1).trim();
+        if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) {
+            value = value.slice(1, -1);
+        }
+        if (!BOUNDARY.test(value)) {
+            throw new MultipartError('the multipart boundary is not a valid one');
+        }
+        return value;
+    }
+    throw new MultipartError('multipart/form-data with no boundary parameter');
+}
+const CRLF = Buffer.from('\r\n');
+const DOUBLE_CRLF = Buffer.from('\r\n\r\n');
+/** `name="x"; filename="y"` — one parameter, unescaped the way browsers send it. */
+function parameter(header, key) {
+    const match = new RegExp(`(?:^|;)\\s*${key}\\s*=\\s*("([^"]*)"|[^;]*)`, 'i').exec(header);
+    if (match === null)
+        return undefined;
+    const value = match[2] ?? match[1] ?? '';
+    return value.trim();
+}
+export function parseMultipart(body, boundary) {
+    const buffer = Buffer.from(body.buffer, body.byteOffset, body.byteLength);
+    const delimiter = Buffer.from(`--${boundary}`);
+    let cursor = buffer.indexOf(delimiter);
+    if (cursor === -1)
+        throw new MultipartError('the body contains no multipart boundary');
+    const parts = [];
+    while (cursor !== -1) {
+        let start = cursor + delimiter.length;
+        // `--boundary--` closes the body. Anything after it is epilogue and
+        // ignored, which is what the specification says and what every client does.
+        if (buffer.slice(start, start + 2).toString() === '--')
+            break;
+        // The delimiter is followed by CRLF. Tolerating a bare LF here would mean
+        // accepting a body no conforming client sends, so it does not.
+        if (!buffer.slice(start, start + 2).equals(CRLF)) {
+            throw new MultipartError('a multipart boundary is not followed by CRLF');
+        }
+        start += 2;
+        const headerEnd = buffer.indexOf(DOUBLE_CRLF, start);
+        if (headerEnd === -1)
+            throw new MultipartError('a multipart part has no header block');
+        if (headerEnd - start > MAX_PART_HEADER_BYTES) {
+            throw new MultipartError('a multipart part has an implausibly large header block');
+        }
+        const headers = buffer.slice(start, headerEnd).toString('utf8');
+        const bodyStart = headerEnd + DOUBLE_CRLF.length;
+        const next = buffer.indexOf(delimiter, bodyStart);
+        if (next === -1)
+            throw new MultipartError('a multipart part is not terminated');
+        // The CRLF immediately before the next delimiter belongs to the delimiter,
+        // not to the content. Keeping it appends two bytes to every uploaded file.
+        const end = next >= 2 && buffer.slice(next - 2, next).equals(CRLF) ? next - 2 : next;
+        let disposition;
+        let contentType;
+        for (const line of headers.split('\r\n')) {
+            const colon = line.indexOf(':');
+            if (colon === -1)
+                continue;
+            const key = line.slice(0, colon).trim().toLowerCase();
+            const value = line.slice(colon + 1).trim();
+            if (key === 'content-disposition')
+                disposition = value;
+            else if (key === 'content-type')
+                contentType = value;
+            else if (key === 'content-transfer-encoding' && value.toLowerCase() !== 'binary') {
+                // Refused rather than decoded. base64 here is legal and archaic, and a
+                // decoder is another parser of hostile input for a case no browser and
+                // no HTTP client produces.
+                throw new MultipartError(`content-transfer-encoding ${value} is not supported`);
+            }
+        }
+        if (disposition === undefined)
+            throw new MultipartError('a multipart part has no content-disposition');
+        const name = parameter(disposition, 'name');
+        if (name === undefined || name === '') {
+            throw new MultipartError('a multipart part has no name');
+        }
+        const filename = parameter(disposition, 'filename');
+        parts.push({
+            name,
+            ...(filename === undefined || filename === '' ? {} : { filename }),
+            ...(contentType === undefined ? {} : { contentType }),
+            bytes: new Uint8Array(buffer.subarray(bodyStart, end)),
+        });
+        if (parts.length > MAX_PARTS) {
+            throw new MultipartError(`a multipart body may carry at most ${MAX_PARTS} parts`);
+        }
+        cursor = next;
+    }
+    if (parts.length === 0)
+        throw new MultipartError('the multipart body carries no parts');
+    return parts;
+}
+//# sourceMappingURL=multipart.js.map

package/dist/multipart.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"multipart.js","sourceRoot":"","sources":["../multipart.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAA;IAC9B,CAAC;CACF;AAUD,iFAAiF;AACjF,MAAM,CAAC,MAAM,SAAS,GAAG,EAAE,CAAA;AAC3B,gFAAgF;AAChF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,GAAG,IAAI,CAAA;AAE7C,yEAAyE;AACzE,MAAM,QAAQ,GAAG,sFAAsF,CAAA;AAEvG;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,WAA+B;IAC/D,IAAI,WAAW,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IAC/C,MAAM,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC9C,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,qBAAqB;QAAE,OAAO,SAAS,CAAA;IAEjF,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACjC,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,SAAQ;QACvB,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,UAAU;YAAE,SAAQ;QAExE,IAAI,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAC1C,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC5B,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,cAAc,CAAC,2CAA2C,CAAC,CAAA;QACvE,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,IAAI,cAAc,CAAC,gDAAgD,CAAC,CAAA;AAC5E,CAAC;AAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;AAChC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;AAE3C,oFAAoF;AACpF,SAAS,SAAS,CAAC,MAAc,EAAE,GAAW;IAC5C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,cAAc,GAAG,4BAA4B,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACzF,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAA;IACpC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACxC,OAAO,KAAK,CAAC,IAAI,EAAE,CAAA;AACrB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAgB,EAAE,QAAgB;IAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;IACzE,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAA;IAE9C,IAAI,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACtC,IAAI,MAAM,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,cAAc,CAAC,yCAAyC,CAAC,CAAA;IAEtF,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,OAAO,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACrB,IAAI,KAAK,GAAG,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;QAErC,oEAAoE;QACpE,4EAA4E;QAC5E,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,IAAI;YAAE,MAAK;QAE7D,0EAA0E;QAC1E,+DAA+D;QAC/D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,cAAc,CAAC,8CAA8C,CAAC,CAAA;QAC1E,CAAC;QACD,KAAK,IAAI,CAAC,CAAA;QAEV,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;QACpD,IAAI,SAAS,KAAK,CAAC,CAAC;YAAE,MAAM,IAAI,cAAc,CAAC,sCAAsC,CAAC,CAAA;QACtF,IAAI,SAAS,GAAG,KAAK,GAAG,qBAAqB,EAAE,CAAC;YAC9C,MAAM,IAAI,cAAc,CAAC,wDAAwD,CAAC,CAAA;QACpF,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;QAC/D,MAAM,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC,MAAM,CAAA;QAEhD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;QACjD,IAAI,IAAI,KAAK,CAAC,CAAC;YAAE,MAAM,IAAI,cAAc,CAAC,oCAAoC,CAAC,CAAA;QAE/E,2EAA2E;QAC3E,2EAA2E;QAC3E,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAEpF,IAAI,WAA+B,CAAA;QACnC,IAAI,WAA+B,CAAA;QACnC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YAC/B,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,SAAQ;YAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YACrD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAC1C,IAAI,GAAG,KAAK,qBAAqB;gBAAE,WAAW,GAAG,KAAK,CAAA;iBACjD,IAAI,GAAG,KAAK,cAAc;gBAAE,WAAW,GAAG,KAAK,CAAA;iBAC/C,IAAI,GAAG,KAAK,2BAA2B,IAAI,KAAK,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE,CAAC;gBACjF,uEAAuE;gBACvE,uEAAuE;gBACvE,2BAA2B;gBAC3B,MAAM,IAAI,cAAc,CAAC,6BAA6B,KAAK,mBAAmB,CAAC,CAAA;YACjF,CAAC;QACH,CAAC;QAED,IAAI,WAAW,KAAK,SAAS;YAAE,MAAM,IAAI,cAAc,CAAC,6CAA6C,CAAC,CAAA;QACtG,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;QAC3C,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,cAAc,CAAC,8BAA8B,CAAC,CAAA;QAC1D,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,WAAW,EAAE,UAAU,CAAC,CAAA;QACnD,KAAK,CAAC,IAAI,CAAC;YACT,IAAI;YACJ,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;YAClE,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;YACrD,KAAK,EAAE,IAAI,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;SACvD,CAAC,CAAA;QAEF,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,cAAc,CAAC,sCAAsC,SAAS,QAAQ,CAAC,CAAA;QACnF,CAAC;QAED,MAAM,GAAG,IAAI,CAAA;IACf,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,cAAc,CAAC,qCAAqC,CAAC,CAAA;IACvF,OAAO,KAAK,CAAA;AACd,CAAC"}

package/dist/oauth.d.ts

@@ -36,6 +36,16 @@ export interface ProtectedResourceMetadata {
 }
 /** The path RFC 9728 fixes, and the one the `WWW-Authenticate` header names. */
 export declare const PROTECTED_RESOURCE_PATH = "/.well-known/oauth-protected-resource";
+/**
+ * Where the public signing key is published — RFC 7517, and the conventional
+ * path every JWT library looks at first.
+ *
+ * Served only by a deployment that signs asymmetrically. With
+ * `NACRE_JWT_SECRET` there is nothing to publish and this answers `404`: a
+ * shared secret has no public half, and an endpoint that produced one anyway
+ * would be handing out the key that mints tokens.
+ */
+export declare const JWKS_PATH = "/.well-known/jwks.json";
 export declare function protectedResourceMetadata(input: {
     /** `NACRE_CANONICAL_URL`. Baked into the issuer of every token ever minted. */
     readonly canonicalUrl: string;

package/dist/oauth.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"oauth.d.ts","sourceRoot":"","sources":["../oauth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAClD,QAAQ,CAAC,wBAAwB,EAAE,SAAS,MAAM,EAAE,CAAA;IACpD,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAA;IACvC,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAA;CAC7C;AAED,gFAAgF;AAChF,eAAO,MAAM,uBAAuB,0CAA0C,CAAA;AAE9E,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,+EAA+E;IAC/E,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAA;CACtC,GAAG,yBAAyB,CAoB5B"}
+{"version":3,"file":"oauth.d.ts","sourceRoot":"","sources":["../oauth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAClD,QAAQ,CAAC,wBAAwB,EAAE,SAAS,MAAM,EAAE,CAAA;IACpD,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAA;IACvC,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAA;CAC7C;AAED,gFAAgF;AAChF,eAAO,MAAM,uBAAuB,0CAA0C,CAAA;AAE9E;;;;;;;;GAQG;AACH,eAAO,MAAM,SAAS,2BAA2B,CAAA;AAEjD,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,+EAA+E;IAC/E,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,qEAAqE;IACrE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAA;CACtC,GAAG,yBAAyB,CAoB5B"}

package/dist/oauth.js

@@ -29,6 +29,16 @@
  */
 /** The path RFC 9728 fixes, and the one the `WWW-Authenticate` header names. */
 export const PROTECTED_RESOURCE_PATH = '/.well-known/oauth-protected-resource';
+/**
+ * Where the public signing key is published — RFC 7517, and the conventional
+ * path every JWT library looks at first.
+ *
+ * Served only by a deployment that signs asymmetrically. With
+ * `NACRE_JWT_SECRET` there is nothing to publish and this answers `404`: a
+ * shared secret has no public half, and an endpoint that produced one anyway
+ * would be handing out the key that mints tokens.
+ */
+export const JWKS_PATH = '/.well-known/jwks.json';
 export function protectedResourceMetadata(input) {
     return {
         // Without a trailing slash, because this is the audience value a token is

package/dist/oauth.js.map

@@ -1 +1 @@
-{"version":3,"file":"oauth.js","sourceRoot":"","sources":["../oauth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAUH,gFAAgF;AAChF,MAAM,CAAC,MAAM,uBAAuB,GAAG,uCAAuC,CAAA;AAE9E,MAAM,UAAU,yBAAyB,CAAC,KAKzC;IACC,OAAO;QACL,0EAA0E;QAC1E,4EAA4E;QAC5E,0DAA0D;QAC1D,QAAQ,EAAE,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAChD,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,qBAAqB,EAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;QAC/E,wEAAwE;QACxE,4EAA4E;QAC5E,sEAAsE;QACtE,wBAAwB,EAAE,CAAC,QAAQ,CAAC;QACpC,sBAAsB,EAAE,2DAA2D;QACnF,2EAA2E;QAC3E,yEAAyE;QACzE,qEAAqE;QACrE,oDAAoD;QACpD,gBAAgB,EAAE,EAAE;KACrB,CAAA;AACH,CAAC"}
+{"version":3,"file":"oauth.js","sourceRoot":"","sources":["../oauth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAUH,gFAAgF;AAChF,MAAM,CAAC,MAAM,uBAAuB,GAAG,uCAAuC,CAAA;AAE9E;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,wBAAwB,CAAA;AAEjD,MAAM,UAAU,yBAAyB,CAAC,KAKzC;IACC,OAAO;QACL,0EAA0E;QAC1E,4EAA4E;QAC5E,0DAA0D;QAC1D,QAAQ,EAAE,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAChD,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,qBAAqB,EAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;QAC/E,wEAAwE;QACxE,4EAA4E;QAC5E,sEAAsE;QACtE,wBAAwB,EAAE,CAAC,QAAQ,CAAC;QACpC,sBAAsB,EAAE,2DAA2D;QACnF,2EAA2E;QAC3E,yEAAyE;QACzE,qEAAqE;QACrE,oDAAoD;QACpD,gBAAgB,EAAE,EAAE;KACrB,CAAA;AACH,CAAC"}

package/dist/observability.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observability.d.ts","sourceRoot":"","sources":["../observability.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAA;AAI9B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAE3C;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAkKtG"}
+{"version":3,"file":"observability.d.ts","sourceRoot":"","sources":["../observability.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAA;AAI9B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAE3C;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAyItG"}

package/dist/observability.js

@@ -15,10 +15,10 @@ export function collectDatabaseGauges(pool, metrics, role) {
             // Across organizations, so it runs outside withOrg under the role that
             // may see them all. Every query below names org_id in its output rather
             // than filtering to one.
-            const { rows: orgs } = await client.query('SELECT id, slug, groups_version FROM organizations WHERE deleted_at IS NULL');
+            const { rows: orgs } = await client.query('SELECT id, slug FROM organizations WHERE deleted_at IS NULL');
             metrics.documents.reset();
             metrics.tombstonesPending.reset();
-            metrics.aclPropagationLag.reset();
+            metrics.processingAge.reset();
             metrics.reindexProgress.reset();
             metrics.collectionsRetired.reset();
             // One series per layer that has ever been reindexed, labelled by slug
@@ -76,62 +76,40 @@ export function collectDatabaseGauges(pool, metrics, role) {
                 // still inside `withOrg`: collapsing across tenants would mean reading
                 // every tenant's documents from the API process, and the row-level
                 // policies are not something to route around for a gauge.
-                const rows = await withOrg(pool, org.id, async (c) => (await c.query(`SELECT status,
+                const rows = await withOrg(pool, org.id, async (c) => (await c.query(
+                // claim_age rides this query rather than adding one: claimed_at
+                // is on the same documents rows, non-null only while a document
+                // is being indexed, and cleared on success or failure. The max
+                // per status is the oldest in-flight document; the max across
+                // statuses is taken below, so a worker wedged in either
+                // `parsing` or `indexing` shows up.
+                `SELECT status,
                         count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
                         count(*) FILTER (
                           WHERE deleted_at IS NOT NULL AND vectors_purged_at IS NULL
                         )::text AS tombstoned,
-                        -- The oldest document whose tags predate the current
-                        -- groups_version. chunk_count > 0 is the whole
-                        -- correctness of this one, and it was missing once: the
-                        -- number is the age of the oldest document still
-                        -- carrying tags built from a superseded grant set, which
-                        -- only a document with points in the index can do.
-                        -- Without the clause a single failed or pending
-                        -- document pinned the gauge at its own age forever,
-                        -- because the retag loop claims neither and nothing else
-                        -- can clear it — the one metric with an alert on it
-                        -- became a stuck alert, which is how it gets muted.
-                        --
-                        -- Deliberately not status = 'indexed'. A document that
-                        -- indexed once and failed on a later pass still has the
-                        -- earlier pass's points and still carries their tags,
-                        -- and is exactly the case this is meant to catch.
-                        -- claimStale uses the same predicate; the two must
-                        -- agree, or the gauge counts what the loop cannot claim.
-                        min(COALESCE(acl_tagged_at, created_at)) FILTER (
-                          WHERE deleted_at IS NULL AND chunk_count > 0
-                            AND acl_version < $2
-                        )::text AS oldest_stale
+                        extract(
+                          epoch FROM max(now() - claimed_at) FILTER (WHERE claimed_at IS NOT NULL)
+                        ) AS claim_age
                    FROM documents
                   WHERE org_id = $1
-                  GROUP BY status`, [org.id, Number(org.groups_version)])).rows, scope);
+                  GROUP BY status`, [org.id])).rows, scope);
                 let tombstoned = 0;
-                let oldest;
+                let claimAge = 0;
                 for (const row of rows) {
                     metrics.documents.set(Number(row.live), { org: org.slug, status: row.status });
                     tombstoned += Number(row.tombstoned);
-                    if (row.oldest_stale !== null) {
-                        const at = Date.parse(row.oldest_stale);
-                        if (!Number.isNaN(at) && (oldest === undefined || at < oldest))
-                            oldest = at;
-                    }
+                    if (row.claim_age !== null)
+                        claimAge = Math.max(claimAge, Number(row.claim_age));
                 }
                 metrics.tombstonesPending.set(tombstoned, { org: org.slug });
-                // Per organization, and set even when it is zero.
-                //
-                // A single worst-across-tenants number was the first shape of this and
-                // it was wrong twice over: one neglected tenant pins the gauge and hides
-                // every other tenant behind it, and when the alert does fire it does not
-                // say who. `max(nacre_acl_propagation_lag_seconds) > 60` is the same
-                // alert and is still independent of how many tenants exist, so the
-                // aggregate cost nothing and bought nothing.
-                //
-                // Zero is written rather than omitted: an absent series and a zero one
-                // mean "caught up" and "not being measured", and those must not look
-                // alike on the one metric that evidences I4.
-                const lag = oldest === undefined ? 0 : Math.max(0, (Date.now() - oldest) / 1000);
-                metrics.aclPropagationLag.set(lag, { org: org.slug });
+                // Only when something is in flight. A zero for every idle tenant would
+                // be true and useless; the alertable shape is a series that exists and
+                // climbs, so no claimed document means no series, the same way the
+                // reindex and retired-collection gauges stay silent about tenants with
+                // nothing to report.
+                if (claimAge > 0)
+                    metrics.processingAge.set(claimAge, { org: org.slug });
             }
         }
         finally {

package/dist/observability.js.map

@@ -1 +1 @@
-{"version":3,"file":"observability.js","sourceRoot":"","sources":["../observability.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAG7D;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAU,EAAE,OAAgB,EAAE,IAAa;IAC/E,MAAM,KAAK,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;IAEhD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACnC,IAAI,CAAC;YACH,uEAAuE;YACvE,wEAAwE;YACxE,yBAAyB;YACzB,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACvC,6EAA6E,CAC9E,CAAA;YAED,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;YACzB,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;YACjC,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;YACjC,OAAO,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;YAC/B,OAAO,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAA;YAElC,sEAAsE;YACtE,yEAAyE;YACzE,uEAAuE;YACvE,wEAAwE;YACxE,iBAAiB;YACjB,EAAE;YACF,yEAAyE;YACzE,uEAAuE;YACvE,yEAAyE;YACzE,iDAAiD;YACjD,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAK7C;;;;8DAIsD,CACvD,CAAA;YACD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;gBACtC,IAAI,KAAK,KAAK,SAAS;oBAAE,SAAQ;gBACjC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAA;YAC1F,CAAC;YAED,yEAAyE;YACzE,qEAAqE;YACrE,0EAA0E;YAC1E,uEAAuE;YACvE,0BAA0B;YAC1B,EAAE;YACF,0EAA0E;YAC1E,uEAAuE;YACvE,yEAAyE;YACzE,yEAAyE;YACzE,mCAAmC;YACnC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAC3C;;;0BAGkB,CACnB,CAAA;YACD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YAClE,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,yCAAyC;gBACzC,EAAE;gBACF,wEAAwE;gBACxE,wEAAwE;gBACxE,kEAAkE;gBAClE,sEAAsE;gBACtE,qEAAqE;gBACrE,uEAAuE;gBACvE,oDAAoD;gBACpD,EAAE;gBACF,mEAAmE;gBACnE,qEAAqE;gBACrE,uEAAuE;gBACvE,mEAAmE;gBACnE,0DAA0D;gBAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CACxB,IAAI,EACJ,GAAG,CAAC,EAAE,EACN,KAAK,EAAE,CAAC,EAAE,EAAE,CACV,CACE,MAAM,CAAC,CAAC,KAAK,CAMX;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kCA6BkB,EAClB,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CACrC,CACF,CAAC,IAAI,EACR,KAAK,CACN,CAAA;gBAED,IAAI,UAAU,GAAG,CAAC,CAAA;gBAClB,IAAI,MAA0B,CAAA;gBAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;oBAC9E,UAAU,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;oBACpC,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;wBAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;wBACvC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,GAAG,MAAM,CAAC;4BAAE,MAAM,GAAG,EAAE,CAAA;oBAC7E,CAAC;gBACH,CAAC;gBAED,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;gBAE5D,kDAAkD;gBAClD,EAAE;gBACF,uEAAuE;gBACvE,yEAAyE;gBACzE,yEAAyE;gBACzE,qEAAqE;gBACrE,mEAAmE;gBACnE,6CAA6C;gBAC7C,EAAE;gBACF,uEAAuE;gBACvE,qEAAqE;gBACrE,6CAA6C;gBAC7C,MAAM,GAAG,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAA;gBAChF,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YACvD,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,OAAO,EAAE,CAAA;QAClB,CAAC;IACH,CAAC,CAAA;AACH,CAAC"}
+{"version":3,"file":"observability.js","sourceRoot":"","sources":["../observability.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAG7D;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAU,EAAE,OAAgB,EAAE,IAAa;IAC/E,MAAM,KAAK,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;IAEhD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACnC,IAAI,CAAC;YACH,uEAAuE;YACvE,wEAAwE;YACxE,yBAAyB;YACzB,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACvC,6DAA6D,CAC9D,CAAA;YAED,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;YACzB,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;YACjC,OAAO,CAAC,aAAa,CAAC,KAAK,EAAE,CAAA;YAC7B,OAAO,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;YAC/B,OAAO,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAA;YAElC,sEAAsE;YACtE,yEAAyE;YACzE,uEAAuE;YACvE,wEAAwE;YACxE,iBAAiB;YACjB,EAAE;YACF,yEAAyE;YACzE,uEAAuE;YACvE,yEAAyE;YACzE,iDAAiD;YACjD,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAK7C;;;;8DAIsD,CACvD,CAAA;YACD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;gBACtC,IAAI,KAAK,KAAK,SAAS;oBAAE,SAAQ;gBACjC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAA;YAC1F,CAAC;YAED,yEAAyE;YACzE,qEAAqE;YACrE,0EAA0E;YAC1E,uEAAuE;YACvE,0BAA0B;YAC1B,EAAE;YACF,0EAA0E;YAC1E,uEAAuE;YACvE,yEAAyE;YACzE,yEAAyE;YACzE,mCAAmC;YACnC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAC3C;;;0BAGkB,CACnB,CAAA;YACD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YAClE,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,yCAAyC;gBACzC,EAAE;gBACF,wEAAwE;gBACxE,wEAAwE;gBACxE,kEAAkE;gBAClE,sEAAsE;gBACtE,qEAAqE;gBACrE,uEAAuE;gBACvE,oDAAoD;gBACpD,EAAE;gBACF,mEAAmE;gBACnE,qEAAqE;gBACrE,uEAAuE;gBACvE,mEAAmE;gBACnE,0DAA0D;gBAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CACxB,IAAI,EACJ,GAAG,CAAC,EAAE,EACN,KAAK,EAAE,CAAC,EAAE,EAAE,CACV,CACE,MAAM,CAAC,CAAC,KAAK;gBAMX,gEAAgE;gBAChE,gEAAgE;gBAChE,+DAA+D;gBAC/D,8DAA8D;gBAC9D,wDAAwD;gBACxD,oCAAoC;gBACpC;;;;;;;;;;kCAUkB,EAClB,CAAC,GAAG,CAAC,EAAE,CAAC,CACT,CACF,CAAC,IAAI,EACR,KAAK,CACN,CAAA;gBAED,IAAI,UAAU,GAAG,CAAC,CAAA;gBAClB,IAAI,QAAQ,GAAG,CAAC,CAAA;gBAChB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;oBAC9E,UAAU,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;oBACpC,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;wBAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAA;gBAClF,CAAC;gBAED,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;gBAC5D,uEAAuE;gBACvE,uEAAuE;gBACvE,mEAAmE;gBACnE,uEAAuE;gBACvE,qBAAqB;gBACrB,IAAI,QAAQ,GAAG,CAAC;oBAAE,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YAC1E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,OAAO,EAAE,CAAA;QAClB,CAAC;IACH,CAAC,CAAA;AACH,CAAC"}