@nacre.work/core 0.1.0 → 0.3.0

package/dist/observability.js

@@ -15,11 +15,12 @@ export function collectDatabaseGauges(pool, metrics, role) {
             // Across organizations, so it runs outside withOrg under the role that
             // may see them all. Every query below names org_id in its output rather
             // than filtering to one.
-            const { rows: orgs } = await client.query('SELECT id, slug, groups_version FROM organizations WHERE deleted_at IS NULL');
+            const { rows: orgs } = await client.query('SELECT id, slug FROM organizations WHERE deleted_at IS NULL');
             metrics.documents.reset();
             metrics.tombstonesPending.reset();
-            metrics.aclPropagationLag.reset();
+            metrics.processingAge.reset();
             metrics.reindexProgress.reset();
+            metrics.collectionsRetired.reset();
             // One series per layer that has ever been reindexed, labelled by slug
             // like every other per-tenant gauge here. Layers with no reindex produce
             // no row and therefore no series: inventing a zero for them would mean
@@ -41,6 +42,24 @@ export function collectDatabaseGauges(pool, metrics, role) {
                     continue;
                 metrics.reindexProgress.set(reindexProgress(state), { org: row.slug, layer: row.layer });
             }
+            // Collections a finished migration is still holding, for the same reason
+            // and in the same shape as the reindex gauge above: one query across
+            // organizations, because `retired_collections` holds a handful of rows in
+            // total rather than a handful per tenant, and a scrape must not cost a
+            // query per organization.
+            //
+            // Only organizations that have one produce a series. A zero for everybody
+            // else would be true and useless — the number that matters is how much
+            // disk a completed migration has not given back, and the alertable shape
+            // is one that stops falling, which needs a series to exist at all rather
+            // than a floor under every tenant.
+            const { rows: retained } = await client.query(`SELECT o.slug, count(*) AS n
+           FROM retired_collections r
+           JOIN organizations o ON o.id = r.org_id
+          GROUP BY o.slug`);
+            for (const row of retained) {
+                metrics.collectionsRetired.set(Number(row.n), { org: row.slug });
+            }
             for (const org of orgs) {
                 // One query per organization, not three.
                 //
@@ -57,62 +76,40 @@ export function collectDatabaseGauges(pool, metrics, role) {
                 // still inside `withOrg`: collapsing across tenants would mean reading
                 // every tenant's documents from the API process, and the row-level
                 // policies are not something to route around for a gauge.
-                const rows = await withOrg(pool, org.id, async (c) => (await c.query(`SELECT status,
+                const rows = await withOrg(pool, org.id, async (c) => (await c.query(
+                // claim_age rides this query rather than adding one: claimed_at
+                // is on the same documents rows, non-null only while a document
+                // is being indexed, and cleared on success or failure. The max
+                // per status is the oldest in-flight document; the max across
+                // statuses is taken below, so a worker wedged in either
+                // `parsing` or `indexing` shows up.
+                `SELECT status,
                         count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
                         count(*) FILTER (
                           WHERE deleted_at IS NOT NULL AND vectors_purged_at IS NULL
                         )::text AS tombstoned,
-                        -- The oldest document whose tags predate the current
-                        -- groups_version. chunk_count > 0 is the whole
-                        -- correctness of this one, and it was missing once: the
-                        -- number is the age of the oldest document still
-                        -- carrying tags built from a superseded grant set, which
-                        -- only a document with points in the index can do.
-                        -- Without the clause a single failed or pending
-                        -- document pinned the gauge at its own age forever,
-                        -- because the retag loop claims neither and nothing else
-                        -- can clear it — the one metric with an alert on it
-                        -- became a stuck alert, which is how it gets muted.
-                        --
-                        -- Deliberately not status = 'indexed'. A document that
-                        -- indexed once and failed on a later pass still has the
-                        -- earlier pass's points and still carries their tags,
-                        -- and is exactly the case this is meant to catch.
-                        -- claimStale uses the same predicate; the two must
-                        -- agree, or the gauge counts what the loop cannot claim.
-                        min(COALESCE(acl_tagged_at, created_at)) FILTER (
-                          WHERE deleted_at IS NULL AND chunk_count > 0
-                            AND acl_version < $2
-                        )::text AS oldest_stale
+                        extract(
+                          epoch FROM max(now() - claimed_at) FILTER (WHERE claimed_at IS NOT NULL)
+                        ) AS claim_age
                    FROM documents
                   WHERE org_id = $1
-                  GROUP BY status`, [org.id, Number(org.groups_version)])).rows, scope);
+                  GROUP BY status`, [org.id])).rows, scope);
                 let tombstoned = 0;
-                let oldest;
+                let claimAge = 0;
                 for (const row of rows) {
                     metrics.documents.set(Number(row.live), { org: org.slug, status: row.status });
                     tombstoned += Number(row.tombstoned);
-                    if (row.oldest_stale !== null) {
-                        const at = Date.parse(row.oldest_stale);
-                        if (!Number.isNaN(at) && (oldest === undefined || at < oldest))
-                            oldest = at;
-                    }
+                    if (row.claim_age !== null)
+                        claimAge = Math.max(claimAge, Number(row.claim_age));
                 }
                 metrics.tombstonesPending.set(tombstoned, { org: org.slug });
-                // Per organization, and set even when it is zero.
-                //
-                // A single worst-across-tenants number was the first shape of this and
-                // it was wrong twice over: one neglected tenant pins the gauge and hides
-                // every other tenant behind it, and when the alert does fire it does not
-                // say who. `max(nacre_acl_propagation_lag_seconds) > 60` is the same
-                // alert and is still independent of how many tenants exist, so the
-                // aggregate cost nothing and bought nothing.
-                //
-                // Zero is written rather than omitted: an absent series and a zero one
-                // mean "caught up" and "not being measured", and those must not look
-                // alike on the one metric that evidences I4.
-                const lag = oldest === undefined ? 0 : Math.max(0, (Date.now() - oldest) / 1000);
-                metrics.aclPropagationLag.set(lag, { org: org.slug });
+                // Only when something is in flight. A zero for every idle tenant would
+                // be true and useless; the alertable shape is a series that exists and
+                // climbs, so no claimed document means no series, the same way the
+                // reindex and retired-collection gauges stay silent about tenants with
+                // nothing to report.
+                if (claimAge > 0)
+                    metrics.processingAge.set(claimAge, { org: org.slug });
             }
         }
         finally {

package/dist/observability.js.map

@@ -1 +1 @@
-{"version":3,"file":"observability.js","sourceRoot":"","sources":["../observability.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAG7D;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAU,EAAE,OAAgB,EAAE,IAAa;IAC/E,MAAM,KAAK,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;IAEhD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACnC,IAAI,CAAC;YACH,uEAAuE;YACvE,wEAAwE;YACxE,yBAAyB;YACzB,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACvC,6EAA6E,CAC9E,CAAA;YAED,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;YACzB,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;YACjC,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;YACjC,OAAO,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;YAE/B,sEAAsE;YACtE,yEAAyE;YACzE,uEAAuE;YACvE,wEAAwE;YACxE,iBAAiB;YACjB,EAAE;YACF,yEAAyE;YACzE,uEAAuE;YACvE,yEAAyE;YACzE,iDAAiD;YACjD,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAK7C;;;;8DAIsD,CACvD,CAAA;YACD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;gBACtC,IAAI,KAAK,KAAK,SAAS;oBAAE,SAAQ;gBACjC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAA;YAC1F,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,yCAAyC;gBACzC,EAAE;gBACF,wEAAwE;gBACxE,wEAAwE;gBACxE,kEAAkE;gBAClE,sEAAsE;gBACtE,qEAAqE;gBACrE,uEAAuE;gBACvE,oDAAoD;gBACpD,EAAE;gBACF,mEAAmE;gBACnE,qEAAqE;gBACrE,uEAAuE;gBACvE,mEAAmE;gBACnE,0DAA0D;gBAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CACxB,IAAI,EACJ,GAAG,CAAC,EAAE,EACN,KAAK,EAAE,CAAC,EAAE,EAAE,CACV,CACE,MAAM,CAAC,CAAC,KAAK,CAMX;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kCA6BkB,EAClB,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CACrC,CACF,CAAC,IAAI,EACR,KAAK,CACN,CAAA;gBAED,IAAI,UAAU,GAAG,CAAC,CAAA;gBAClB,IAAI,MAA0B,CAAA;gBAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;oBAC9E,UAAU,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;oBACpC,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;wBAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;wBACvC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,GAAG,MAAM,CAAC;4BAAE,MAAM,GAAG,EAAE,CAAA;oBAC7E,CAAC;gBACH,CAAC;gBAED,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;gBAE5D,kDAAkD;gBAClD,EAAE;gBACF,uEAAuE;gBACvE,yEAAyE;gBACzE,yEAAyE;gBACzE,qEAAqE;gBACrE,mEAAmE;gBACnE,6CAA6C;gBAC7C,EAAE;gBACF,uEAAuE;gBACvE,qEAAqE;gBACrE,6CAA6C;gBAC7C,MAAM,GAAG,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAA;gBAChF,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YACvD,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,OAAO,EAAE,CAAA;QAClB,CAAC;IACH,CAAC,CAAA;AACH,CAAC"}
+{"version":3,"file":"observability.js","sourceRoot":"","sources":["../observability.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAG7D;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAU,EAAE,OAAgB,EAAE,IAAa;IAC/E,MAAM,KAAK,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;IAEhD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACnC,IAAI,CAAC;YACH,uEAAuE;YACvE,wEAAwE;YACxE,yBAAyB;YACzB,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CACvC,6DAA6D,CAC9D,CAAA;YAED,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;YACzB,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAA;YACjC,OAAO,CAAC,aAAa,CAAC,KAAK,EAAE,CAAA;YAC7B,OAAO,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;YAC/B,OAAO,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAA;YAElC,sEAAsE;YACtE,yEAAyE;YACzE,uEAAuE;YACvE,wEAAwE;YACxE,iBAAiB;YACjB,EAAE;YACF,yEAAyE;YACzE,uEAAuE;YACvE,yEAAyE;YACzE,iDAAiD;YACjD,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAK7C;;;;8DAIsD,CACvD,CAAA;YACD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;gBACtC,IAAI,KAAK,KAAK,SAAS;oBAAE,SAAQ;gBACjC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAA;YAC1F,CAAC;YAED,yEAAyE;YACzE,qEAAqE;YACrE,0EAA0E;YAC1E,uEAAuE;YACvE,0BAA0B;YAC1B,EAAE;YACF,0EAA0E;YAC1E,uEAAuE;YACvE,yEAAyE;YACzE,yEAAyE;YACzE,mCAAmC;YACnC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAC3C;;;0BAGkB,CACnB,CAAA;YACD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YAClE,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,yCAAyC;gBACzC,EAAE;gBACF,wEAAwE;gBACxE,wEAAwE;gBACxE,kEAAkE;gBAClE,sEAAsE;gBACtE,qEAAqE;gBACrE,uEAAuE;gBACvE,oDAAoD;gBACpD,EAAE;gBACF,mEAAmE;gBACnE,qEAAqE;gBACrE,uEAAuE;gBACvE,mEAAmE;gBACnE,0DAA0D;gBAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CACxB,IAAI,EACJ,GAAG,CAAC,EAAE,EACN,KAAK,EAAE,CAAC,EAAE,EAAE,CACV,CACE,MAAM,CAAC,CAAC,KAAK;gBAMX,gEAAgE;gBAChE,gEAAgE;gBAChE,+DAA+D;gBAC/D,8DAA8D;gBAC9D,wDAAwD;gBACxD,oCAAoC;gBACpC;;;;;;;;;;kCAUkB,EAClB,CAAC,GAAG,CAAC,EAAE,CAAC,CACT,CACF,CAAC,IAAI,EACR,KAAK,CACN,CAAA;gBAED,IAAI,UAAU,GAAG,CAAC,CAAA;gBAClB,IAAI,QAAQ,GAAG,CAAC,CAAA;gBAChB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;oBAC9E,UAAU,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;oBACpC,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;wBAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAA;gBAClF,CAAC;gBAED,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;gBAC5D,uEAAuE;gBACvE,uEAAuE;gBACvE,mEAAmE;gBACnE,uEAAuE;gBACvE,qBAAqB;gBACrB,IAAI,QAAQ,GAAG,CAAC;oBAAE,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YAC1E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,OAAO,EAAE,CAAA;QAClB,CAAC;IACH,CAAC,CAAA;AACH,CAAC"}

package/dist/reindex.d.ts

@@ -59,6 +59,40 @@ export interface ReindexState {
     readonly done: number;
     readonly failed: number;
     readonly error?: string;
+    /**
+     * What the recall check scored, once it has run.
+     *
+     * Absent until then, and absent forever for a layer with no reference query
+     * set — that layer has no gate, and `finishReindexIfDone` says so by asking
+     * whether the set exists rather than by looking for this key.
+     */
+    readonly check?: ReindexCheck;
+}
+/**
+ * The verdict of the gate between a finished reindex and its switch.
+ *
+ * `scores` is keyed by reference query id and not by text. The text is the
+ * operator's own, so echoing it would break no rule — but a status endpoint and
+ * a jsonb column are both read by more people than wrote the set, and the
+ * endpoint that lists it is one join away.
+ */
+export interface ReindexCheck {
+    /** The mean of `scores`, in [0, 1]. */
+    readonly recall: number;
+    readonly floor: number;
+    readonly passed: boolean;
+    readonly queries: number;
+    readonly scores: readonly {
+        readonly queryId: string;
+        readonly recall: number;
+    }[];
+    /**
+     * External ids in the set naming no live document. Any means `passed` is
+     * false whatever `recall` says: a stale reference set and a bad model are
+     * different problems, and scoring a missing document as a miss reports the
+     * first as the second.
+     */
+    readonly unresolved?: readonly string[];
 }
 /**
  * How far along, as a ratio, for `nacre_reindex_progress_ratio`.
@@ -75,6 +109,16 @@ export interface ReindexState {
 export declare function reindexProgress(state: ReindexState): number;
 /** The JSON shape stored in the column. Snake case, like every other jsonb here. */
 export declare function toStateJson(state: ReindexState): Record<string, unknown>;
+/**
+ * The verdict as it is stored, snake case like the rest of the column.
+ *
+ * Its own function because the worker writes this key on its own — one
+ * `jsonb_set` rather than a rewrite of the whole state — and a second
+ * hand-written shape there is how the two would drift. `passed` is read back
+ * out of this object by the SQL predicate that performs the switch, so its name
+ * and its type are load-bearing rather than cosmetic.
+ */
+export declare function toCheckJson(check: ReindexCheck): Record<string, unknown>;
 /**
  * Read it back, or `undefined` for a column that holds nothing usable.
  *

package/dist/reindex.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.d.ts","sourceRoot":"","sources":["../reindex.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,uFAAuF;AACvF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAA;IAClD;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,WAAW,CAAA;IACvC;;;;OAIG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,4CAA4C;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,kEAAkE;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,uEAAuE;IACvE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAQ3D;AAED,oFAAoF;AACpF,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAaxE;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,OAAO,GAAG,YAAY,GAAG,SAAS,CAsBpE"}
+{"version":3,"file":"reindex.d.ts","sourceRoot":"","sources":["../reindex.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,uFAAuF;AACvF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAA;IAClD;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,WAAW,CAAA;IACvC;;;;OAIG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,4CAA4C;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,kEAAkE;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,uEAAuE;IACvE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB;;;;;;OAMG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAA;CAC9B;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B,uCAAuC;IACvC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,SAAS;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;IACjF;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxC;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAQ3D;AAED,oFAAoF;AACpF,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAcxE;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CASxE;AAyBD;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,OAAO,GAAG,YAAY,GAAG,SAAS,CA0BpE"}

package/dist/reindex.js

@@ -60,6 +60,51 @@ export function toStateJson(state) {
         done: state.done,
         failed: state.failed,
         ...(state.error === undefined ? {} : { error: state.error }),
+        ...(state.check === undefined ? {} : { check: toCheckJson(state.check) }),
+    };
+}
+/**
+ * The verdict as it is stored, snake case like the rest of the column.
+ *
+ * Its own function because the worker writes this key on its own — one
+ * `jsonb_set` rather than a rewrite of the whole state — and a second
+ * hand-written shape there is how the two would drift. `passed` is read back
+ * out of this object by the SQL predicate that performs the switch, so its name
+ * and its type are load-bearing rather than cosmetic.
+ */
+export function toCheckJson(check) {
+    return {
+        recall: check.recall,
+        floor: check.floor,
+        passed: check.passed,
+        queries: check.queries,
+        scores: check.scores.map((s) => ({ query_id: s.queryId, recall: s.recall })),
+        ...(check.unresolved === undefined ? {} : { unresolved: [...check.unresolved] }),
+    };
+}
+function fromCheckJson(raw) {
+    if (raw === null || typeof raw !== 'object' || Array.isArray(raw))
+        return undefined;
+    const o = raw;
+    if (typeof o.recall !== 'number' || typeof o.passed !== 'boolean')
+        return undefined;
+    const scores = Array.isArray(o.scores) ? o.scores : [];
+    return {
+        recall: o.recall,
+        floor: typeof o.floor === 'number' ? o.floor : 0,
+        passed: o.passed,
+        queries: typeof o.queries === 'number' ? o.queries : scores.length,
+        scores: scores.flatMap((s) => {
+            if (s === null || typeof s !== 'object')
+                return [];
+            const e = s;
+            if (typeof e.query_id !== 'string' || typeof e.recall !== 'number')
+                return [];
+            return [{ queryId: e.query_id, recall: e.recall }];
+        }),
+        ...(Array.isArray(o.unresolved) && o.unresolved.length > 0
+            ? { unresolved: o.unresolved.filter((u) => typeof u === 'string') }
+            : {}),
     };
 }
 /**
@@ -92,6 +137,10 @@ export function fromStateJson(raw) {
         done: typeof o.done === 'number' ? o.done : 0,
         failed: typeof o.failed === 'number' ? o.failed : 0,
         ...(typeof o.error === 'string' ? { error: o.error } : {}),
+        ...(() => {
+            const check = fromCheckJson(o.check);
+            return check === undefined ? {} : { check };
+        })(),
     };
 }
 //# sourceMappingURL=reindex.js.map

package/dist/reindex.js.map

@@ -1 +1 @@
-{"version":3,"file":"reindex.js","sourceRoot":"","sources":["../reindex.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAwCH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,eAAe,CAAC,KAAmB;IACjD,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU;QAAE,OAAO,CAAC,CAAA;IACzC,0EAA0E;IAC1E,4EAA4E;IAC5E,+DAA+D;IAC/D,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IACvC,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAA;IAC9B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;AAC9C,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,WAAW,CAAC,KAAmB;IAC7C,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,aAAa,EAAE,KAAK,CAAC,YAAY;QACjC,WAAW,EAAE,KAAK,CAAC,UAAU;QAC7B,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;QAC5E,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;KAC7D,CAAA;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,GAAY;IACxC,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAC7D,MAAM,CAAC,GAAG,GAA8B,CAAA;IAExC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAA;IACvB,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,UAAU,IAAI,MAAM,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAC1F,IAAI,OAAO,CAAC,CAAC,aAAa,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAEzD,OAAO;QACL,MAAM;QACN,0EAA0E;QAC1E,uEAAuE;QACvE,KAAK,EAAE,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW;QACtD,YAAY,EAAE,CAAC,CAAC,aAAa;QAC7B,UAAU,EAAE,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE;QAClE,SAAS,EAAE,OAAO,CAAC,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAChD,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,EAAE,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACnD,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,CAAA;AACH,CAAC"}
+{"version":3,"file":"reindex.js","sourceRoot":"","sources":["../reindex.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAwEH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,eAAe,CAAC,KAAmB;IACjD,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU;QAAE,OAAO,CAAC,CAAA;IACzC,0EAA0E;IAC1E,4EAA4E;IAC5E,+DAA+D;IAC/D,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IACvC,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAA;IAC9B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;AAC9C,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,WAAW,CAAC,KAAmB;IAC7C,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,aAAa,EAAE,KAAK,CAAC,YAAY;QACjC,WAAW,EAAE,KAAK,CAAC,UAAU;QAC7B,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;QAC5E,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;QAC5D,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;KAC1E,CAAA;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,KAAmB;IAC7C,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5E,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;KACjF,CAAA;AACH,CAAC;AAED,SAAS,aAAa,CAAC,GAAY;IACjC,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAA;IACnF,MAAM,CAAC,GAAG,GAA8B,CAAA;IACxC,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IAEnF,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAA;IACtD,OAAO;QACL,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,OAAO,EAAE,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;QAClE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3B,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ;gBAAE,OAAO,EAAE,CAAA;YAClD,MAAM,CAAC,GAAG,CAA4B,CAAA;YACtC,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ;gBAAE,OAAO,EAAE,CAAA;YAC7E,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;QACpD,CAAC,CAAC;QACF,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;YACxD,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,EAAE;YAChF,CAAC,CAAC,EAAE,CAAC;KACR,CAAA;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,GAAY;IACxC,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAC7D,MAAM,CAAC,GAAG,GAA8B,CAAA;IAExC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAA;IACvB,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,UAAU,IAAI,MAAM,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAC1F,IAAI,OAAO,CAAC,CAAC,aAAa,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAEzD,OAAO;QACL,MAAM;QACN,0EAA0E;QAC1E,uEAAuE;QACvE,KAAK,EAAE,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW;QACtD,YAAY,EAAE,CAAC,CAAC,aAAa;QAC7B,UAAU,EAAE,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE;QAClE,SAAS,EAAE,OAAO,CAAC,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAChD,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,EAAE,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACnD,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,GAAG,CAAC,GAAG,EAAE;YACP,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;YACpC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAA;QAC7C,CAAC,CAAC,EAAE;KACL,CAAA;AACH,CAAC"}

package/dist/s3.d.ts

@@ -0,0 +1,111 @@
+/**
+ * Object storage, signed by hand.
+ *
+ * `NACRE_S3_*` has been in `docs/config.md` since before there was a server and
+ * read by nothing — not used, not validated, so a wrong endpoint or a missing
+ * credential was silent while the `full` profile started a MinIO nobody talked
+ * to. This is the client that ends that.
+ *
+ * ─── why not the AWS SDK ───
+ *
+ * Four operations are needed: put, get, delete, head. `@aws-sdk/client-s3` is
+ * tens of megabytes and hundreds of transitive packages, in a container whose
+ * job is to read documents other people send it — which is the argument
+ * `metrics.ts` already makes for not taking prom-client. SigV4 is a documented
+ * algorithm with a published set of test vectors, it fits in this file, and it
+ * has no release cadence to track.
+ *
+ * The rule that comes with that choice: **anything here is verified against a
+ * real MinIO before it is believed**, because a signing bug produces a 403 that
+ * says nothing about which of the six inputs was wrong.
+ *
+ * ─── what is deliberately not here ───
+ *
+ * No multipart upload, so an object is one PUT and `NACRE_MAX_DOCUMENT_BYTES`
+ * bounds it. No listing, because nothing needs to enumerate a bucket and a
+ * paginated list is where this file would stop being small. No retries: the
+ * callers are the ingest queue and the collector, both of which already retry
+ * whole units of work and would otherwise retry twice.
+ */
+export interface S3Options {
+    /** Including the scheme and port, e.g. `http://minio:9000`. No trailing slash. */
+    readonly endpoint: string;
+    readonly bucket: string;
+    readonly region: string;
+    readonly accessKey: string;
+    readonly secretKey: string;
+    /**
+     * `{endpoint}/{bucket}/{key}` rather than `{bucket}.{endpoint}/{key}`.
+     *
+     * True for MinIO and for anything reached by IP or through a name that is not
+     * a wildcard DNS entry, which is every self-hosted deployment. AWS itself
+     * wants it false.
+     */
+    readonly forcePathStyle: boolean;
+}
+export declare class S3Error extends Error {
+    readonly status: number;
+    readonly method: string;
+    readonly key: string;
+    constructor(status: number, method: string, key: string, body: string);
+}
+export declare class S3 {
+    #private;
+    constructor(options: S3Options);
+    get bucket(): string;
+    /** Store an object, overwriting whatever was there. */
+    put(key: string, body: Uint8Array, contentType?: string): Promise<void>;
+    /** Fetch an object. `undefined` when it is not there, which is not an error. */
+    get(key: string): Promise<Uint8Array | undefined>;
+    /**
+     * Remove an object. Absent is success.
+     *
+     * S3 answers `204` whether or not the key existed, and the caller — the
+     * collector, purging a deleted document — must be able to run twice.
+     */
+    remove(key: string): Promise<void>;
+    /**
+     * A URL that fetches one object, for a while, without a credential.
+     *
+     * Query-string SigV4 rather than the header form: the point is a link a
+     * client can follow, so everything that authenticates it has to be in the
+     * URL. `host` is the only signed header, and the payload is unsigned because
+     * a GET has none.
+     *
+     * **This is a bearer capability and it leaves the permission model behind.**
+     * Nacre checks `read` once, when the link is minted, and the object store
+     * checks nothing afterwards — a revocation during the window does not reach a
+     * URL already handed out, and neither does a delete. That is what presigning
+     * is, in every implementation of it, and it is why the lifetime is a
+     * configured number rather than a constant: `NACRE_PRESIGN_TTL` is how long a
+     * deployment is willing for that to be true.
+     *
+     * It is also why the caller of this decides, not this: minting one per search
+     * hit would issue ten capabilities to answer a question about relevance, most
+     * never followed. See the note on the search response.
+     */
+    presign(key: string, ttlSeconds: number): string;
+    /**
+     * Whether the bucket answers, for readiness.
+     *
+     * A HEAD on the bucket rather than on an object: it needs no key to exist and
+     * it fails for the three reasons that matter — unreachable, wrong bucket,
+     * wrong credential. A TCP connect would pass all three.
+     */
+    ready(): Promise<boolean>;
+}
+/**
+ * Where a document's bytes live.
+ *
+ * Derived from identity — organization, layer, external id — and never from the
+ * content. Content addressing would let two documents share an object, and then
+ * deleting one would either strand the other or need a reference count that
+ * nothing else in this system has. One document, one object, and a re-ingest
+ * overwrites in place.
+ *
+ * The external id is hashed rather than escaped: it is caller-chosen text that
+ * may contain anything, and a key derived by escaping is a key whose length and
+ * shape a caller controls.
+ */
+export declare function documentKey(orgId: string, layerId: string, externalId: string): string;
+//# sourceMappingURL=s3.d.ts.map

package/dist/s3.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../s3.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,WAAW,SAAS;IACxB,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B;;;;;;OAMG;IACH,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;CACjC;AA8BD,qBAAa,OAAQ,SAAQ,KAAK;IAE9B,QAAQ,CAAC,MAAM,EAAE,MAAM;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM;gBAFX,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM;CAQf;AAED,qBAAa,EAAE;;gBAGD,OAAO,EAAE,SAAS;IAI9B,IAAI,MAAM,IAAI,MAAM,CAEnB;IAuGD,uDAAuD;IACjD,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,SAA6B,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjG,gFAAgF;IAC1E,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAOvD;;;;;OAKG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOxC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM;IA2ChD;;;;;;OAMG;IACG,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC;CAiBhC;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAEtF"}

package/dist/s3.js

@@ -0,0 +1,242 @@
+import { createHash, createHmac } from 'node:crypto';
+const UNSIGNED = 'UNSIGNED-PAYLOAD';
+const ALGORITHM = 'AWS4-HMAC-SHA256';
+const SERVICE = 's3';
+const sha256 = (data) => createHash('sha256').update(data).digest('hex');
+const hmac = (key, data) => createHmac('sha256', key).update(data, 'utf8').digest();
+/**
+ * Percent-encode one path segment the way SigV4 wants it.
+ *
+ * `encodeURIComponent` leaves `!'()*` alone and AWS does not, and it encodes
+ * nothing else differently — so this is that function plus five characters. A
+ * key containing any of them signs one way and is requested another, which is a
+ * 403 with no clue in it.
+ */
+function encodeSegment(segment) {
+    return encodeURIComponent(segment).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
+}
+/** Keys are `a/b/c`; each segment is encoded and the separators are not. */
+const encodeKey = (key) => key.split('/').map(encodeSegment).join('/');
+export class S3Error extends Error {
+    status;
+    method;
+    key;
+    constructor(status, method, key, body) {
+        // The body, because S3 and MinIO both answer with an XML document naming
+        // the exact code — SignatureDoesNotMatch, NoSuchBucket, AccessDenied — and
+        // a status alone sends the reader to the wrong one of six inputs.
+        super(`s3 ${method} ${key} failed: ${status} ${body.slice(0, 512)}`);
+        this.status = status;
+        this.method = method;
+        this.key = key;
+        this.name = 'S3Error';
+    }
+}
+export class S3 {
+    #options;
+    constructor(options) {
+        this.#options = { ...options, endpoint: options.endpoint.replace(/\/+$/, '') };
+    }
+    get bucket() {
+        return this.#options.bucket;
+    }
+    #url(key) {
+        const { endpoint, bucket, forcePathStyle } = this.#options;
+        if (forcePathStyle)
+            return new URL(`${endpoint}/${bucket}/${encodeKey(key)}`);
+        const url = new URL(endpoint);
+        url.host = `${bucket}.${url.host}`;
+        url.pathname = `/${encodeKey(key)}`;
+        return url;
+    }
+    /**
+     * Sign a request, returning the headers to send with it.
+     *
+     * The payload hash goes in `x-amz-content-sha256` and into the canonical
+     * request, so a body that changes in flight fails the signature rather than
+     * being stored. `UNSIGNED-PAYLOAD` is used for GET and DELETE, which have
+     * none — hashing the empty string would also work and this is what every
+     * other implementation sends.
+     */
+    #sign(input) {
+        const { region, accessKey, secretKey } = this.#options;
+        const amzDate = input.now.toISOString().replace(/[-:]|\.\d{3}/g, '');
+        const dateStamp = amzDate.slice(0, 8);
+        const headers = {
+            ...input.headers,
+            host: input.url.host,
+            'x-amz-content-sha256': input.payloadHash,
+            'x-amz-date': amzDate,
+        };
+        // Lowercased, sorted, values trimmed — the canonical form. Getting the
+        // order wrong is a 403 that reads exactly like a wrong secret.
+        const canonicalHeaderNames = Object.keys(headers)
+            .map((h) => h.toLowerCase())
+            .sort();
+        const canonicalHeaders = canonicalHeaderNames
+            .map((name) => {
+            const key = Object.keys(headers).find((h) => h.toLowerCase() === name);
+            return `${name}:${String(headers[key]).trim()}\n`;
+        })
+            .join('');
+        const signedHeaders = canonicalHeaderNames.join(';');
+        const canonicalRequest = [
+            input.method,
+            input.url.pathname,
+            // No request here takes a query string. An empty one is still a line.
+            '',
+            canonicalHeaders,
+            signedHeaders,
+            input.payloadHash,
+        ].join('\n');
+        const scope = `${dateStamp}/${region}/${SERVICE}/aws4_request`;
+        const stringToSign = [ALGORITHM, amzDate, scope, sha256(canonicalRequest)].join('\n');
+        const signingKey = hmac(hmac(hmac(hmac(`AWS4${secretKey}`, dateStamp), region), SERVICE), 'aws4_request');
+        const signature = createHmac('sha256', signingKey).update(stringToSign, 'utf8').digest('hex');
+        return {
+            ...headers,
+            authorization: `${ALGORITHM} Credential=${accessKey}/${scope}, ` +
+                `SignedHeaders=${signedHeaders}, Signature=${signature}`,
+        };
+    }
+    async #send(method, key, body, extraHeaders = {}) {
+        const url = this.#url(key);
+        const headers = this.#sign({
+            method,
+            url,
+            payloadHash: body === undefined ? UNSIGNED : sha256(body),
+            headers: body === undefined
+                ? extraHeaders
+                : { ...extraHeaders, 'content-length': String(body.length) },
+            now: new Date(),
+        });
+        return fetch(url, {
+            method,
+            headers,
+            ...(body === undefined ? {} : { body }),
+        });
+    }
+    /** Store an object, overwriting whatever was there. */
+    async put(key, body, contentType = 'application/octet-stream') {
+        const response = await this.#send('PUT', key, body, { 'content-type': contentType });
+        if (!response.ok)
+            throw new S3Error(response.status, 'PUT', key, await response.text());
+    }
+    /** Fetch an object. `undefined` when it is not there, which is not an error. */
+    async get(key) {
+        const response = await this.#send('GET', key);
+        if (response.status === 404)
+            return undefined;
+        if (!response.ok)
+            throw new S3Error(response.status, 'GET', key, await response.text());
+        return new Uint8Array(await response.arrayBuffer());
+    }
+    /**
+     * Remove an object. Absent is success.
+     *
+     * S3 answers `204` whether or not the key existed, and the caller — the
+     * collector, purging a deleted document — must be able to run twice.
+     */
+    async remove(key) {
+        const response = await this.#send('DELETE', key);
+        if (!response.ok && response.status !== 404) {
+            throw new S3Error(response.status, 'DELETE', key, await response.text());
+        }
+    }
+    /**
+     * A URL that fetches one object, for a while, without a credential.
+     *
+     * Query-string SigV4 rather than the header form: the point is a link a
+     * client can follow, so everything that authenticates it has to be in the
+     * URL. `host` is the only signed header, and the payload is unsigned because
+     * a GET has none.
+     *
+     * **This is a bearer capability and it leaves the permission model behind.**
+     * Nacre checks `read` once, when the link is minted, and the object store
+     * checks nothing afterwards — a revocation during the window does not reach a
+     * URL already handed out, and neither does a delete. That is what presigning
+     * is, in every implementation of it, and it is why the lifetime is a
+     * configured number rather than a constant: `NACRE_PRESIGN_TTL` is how long a
+     * deployment is willing for that to be true.
+     *
+     * It is also why the caller of this decides, not this: minting one per search
+     * hit would issue ten capabilities to answer a question about relevance, most
+     * never followed. See the note on the search response.
+     */
+    presign(key, ttlSeconds) {
+        const { region, accessKey, secretKey } = this.#options;
+        const url = this.#url(key);
+        const now = new Date();
+        const amzDate = now.toISOString().replace(/[-:]|\.\d{3}/g, '');
+        const dateStamp = amzDate.slice(0, 8);
+        const scope = `${dateStamp}/${region}/${SERVICE}/aws4_request`;
+        // Sorted by key, and encoded the same way the canonical request will be —
+        // a mismatch between the query that is signed and the query that is sent is
+        // a 403 with nothing in it about which of the two was wrong.
+        const params = [
+            ['X-Amz-Algorithm', ALGORITHM],
+            ['X-Amz-Credential', `${accessKey}/${scope}`],
+            ['X-Amz-Date', amzDate],
+            ['X-Amz-Expires', String(Math.max(1, Math.floor(ttlSeconds)))],
+            ['X-Amz-SignedHeaders', 'host'],
+        ];
+        const canonicalQuery = params
+            .map(([k, v]) => [encodeSegment(k), encodeSegment(v)])
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([k, v]) => `${k}=${v}`)
+            .join('&');
+        const canonicalRequest = [
+            'GET',
+            url.pathname,
+            canonicalQuery,
+            `host:${url.host}\n`,
+            'host',
+            UNSIGNED,
+        ].join('\n');
+        const stringToSign = [ALGORITHM, amzDate, scope, sha256(canonicalRequest)].join('\n');
+        const signingKey = hmac(hmac(hmac(hmac(`AWS4${secretKey}`, dateStamp), region), SERVICE), 'aws4_request');
+        const signature = createHmac('sha256', signingKey).update(stringToSign, 'utf8').digest('hex');
+        return `${url.origin}${url.pathname}?${canonicalQuery}&X-Amz-Signature=${signature}`;
+    }
+    /**
+     * Whether the bucket answers, for readiness.
+     *
+     * A HEAD on the bucket rather than on an object: it needs no key to exist and
+     * it fails for the three reasons that matter — unreachable, wrong bucket,
+     * wrong credential. A TCP connect would pass all three.
+     */
+    async ready() {
+        const url = this.#options.forcePathStyle
+            ? new URL(`${this.#options.endpoint}/${this.#options.bucket}`)
+            : this.#url('');
+        const headers = this.#sign({
+            method: 'HEAD',
+            url,
+            payloadHash: UNSIGNED,
+            headers: {},
+            now: new Date(),
+        });
+        const response = await fetch(url, { method: 'HEAD', headers });
+        if (!response.ok) {
+            throw new S3Error(response.status, 'HEAD', this.#options.bucket, response.statusText);
+        }
+        return true;
+    }
+}
+/**
+ * Where a document's bytes live.
+ *
+ * Derived from identity — organization, layer, external id — and never from the
+ * content. Content addressing would let two documents share an object, and then
+ * deleting one would either strand the other or need a reference count that
+ * nothing else in this system has. One document, one object, and a re-ingest
+ * overwrites in place.
+ *
+ * The external id is hashed rather than escaped: it is caller-chosen text that
+ * may contain anything, and a key derived by escaping is a key whose length and
+ * shape a caller controls.
+ */
+export function documentKey(orgId, layerId, externalId) {
+    return `org/${orgId}/layer/${layerId}/${sha256(externalId)}`;
+}
+//# sourceMappingURL=s3.js.map