@nacre.work/core 0.0.0 → 0.2.0

package/dist/reindex.js

@@ -0,0 +1,97 @@
+/**
+ * The state of a layer's move to a different embedding model.
+ *
+ * Shared between the API, which starts one and reports on it, and the worker,
+ * which does the work — so the shape is defined once rather than agreed twice.
+ *
+ * ## What is built, and what is not
+ *
+ * `docs/architecture.md` specifies six steps. Four are here: add the new named
+ * vector, reindex existing documents in bounded batches, switch `vector_name`
+ * atomically, keep search available throughout.
+ *
+ * Two are not, and are named rather than quietly skipped:
+ *
+ * - **The Recall@10 gate.** Step 4 checks the new index against "the layer's
+ *   reference query set" before switching. There is no reference query set
+ *   anywhere in this product — not a table, not an endpoint, not a
+ *   configuration file — so building the gate would mean inventing the surface
+ *   it reads from, and a gate that checks against an empty set is a gate that
+ *   passes. Until it exists, a switch is a decision the operator makes by
+ *   starting the reindex, and `docs/architecture.md` says so.
+ * - **Dropping the old vector after a rollback window.** Step 6. The old vector
+ *   stays. It costs storage and it is what makes a rollback a one-statement
+ *   change, which is worth more than the space while this path is new.
+ */
+/**
+ * How far along, as a ratio, for `nacre_reindex_progress_ratio`.
+ *
+ * Clamped at 1 rather than allowed past it. `total` is counted once when the
+ * reindex starts and documents ingested afterwards are picked up by the same
+ * pass, so `done` legitimately overshoots — and a progress gauge reading 1.4
+ * makes an operator think the number is broken rather than that more work
+ * arrived. A reindex that is complete reads 1 whatever the counts say.
+ *
+ * Zero documents is 1, not 0. An empty layer is finished, and a gauge that sits
+ * at zero forever for a layer with nothing in it is the stuck-alert shape.
+ */
+export function reindexProgress(state) {
+    if (state.status === 'complete')
+        return 1;
+    // The copy has no per-document measure — it moves points, and how many an
+    // organization has is not the layer's `total`. Reporting zero through it is
+    // honest: nothing this layer is waiting for has been done yet.
+    if (state.phase === 'copying')
+        return 0;
+    if (state.total <= 0)
+        return 1;
+    return Math.min(1, state.done / state.total);
+}
+/** The JSON shape stored in the column. Snake case, like every other jsonb here. */
+export function toStateJson(state) {
+    return {
+        status: state.status,
+        phase: state.phase,
+        shadow_vector: state.shadowVector,
+        provider_id: state.providerId,
+        started_at: state.startedAt,
+        ...(state.finishedAt === undefined ? {} : { finished_at: state.finishedAt }),
+        total: state.total,
+        done: state.done,
+        failed: state.failed,
+        ...(state.error === undefined ? {} : { error: state.error }),
+    };
+}
+/**
+ * Read it back, or `undefined` for a column that holds nothing usable.
+ *
+ * Undefined rather than a thrown error or a default: a layer with no reindex
+ * and a layer whose state cannot be parsed both mean "there is nothing to
+ * report", and the caller has no different action to take. The check constraint
+ * added in `0013` is what makes the second case not happen.
+ */
+export function fromStateJson(raw) {
+    if (raw === null || typeof raw !== 'object')
+        return undefined;
+    const o = raw;
+    const status = o.status;
+    if (status !== 'running' && status !== 'complete' && status !== 'failed')
+        return undefined;
+    if (typeof o.shadow_vector !== 'string')
+        return undefined;
+    return {
+        status,
+        // Defaulted rather than refused: a state written before the phase existed
+        // is an embedding-phase state, because the copy is what introduced it.
+        phase: o.phase === 'copying' ? 'copying' : 'embedding',
+        shadowVector: o.shadow_vector,
+        providerId: typeof o.provider_id === 'string' ? o.provider_id : '',
+        startedAt: typeof o.started_at === 'string' ? o.started_at : '',
+        ...(typeof o.finished_at === 'string' ? { finishedAt: o.finished_at } : {}),
+        total: typeof o.total === 'number' ? o.total : 0,
+        done: typeof o.done === 'number' ? o.done : 0,
+        failed: typeof o.failed === 'number' ? o.failed : 0,
+        ...(typeof o.error === 'string' ? { error: o.error } : {}),
+    };
+}
+//# sourceMappingURL=reindex.js.map

package/dist/reindex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reindex.js","sourceRoot":"","sources":["../reindex.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAwCH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,eAAe,CAAC,KAAmB;IACjD,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU;QAAE,OAAO,CAAC,CAAA;IACzC,0EAA0E;IAC1E,4EAA4E;IAC5E,+DAA+D;IAC/D,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IACvC,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAA;IAC9B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;AAC9C,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,WAAW,CAAC,KAAmB;IAC7C,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,aAAa,EAAE,KAAK,CAAC,YAAY;QACjC,WAAW,EAAE,KAAK,CAAC,UAAU;QAC7B,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;QAC5E,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;KAC7D,CAAA;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,GAAY;IACxC,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAC7D,MAAM,CAAC,GAAG,GAA8B,CAAA;IAExC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAA;IACvB,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,UAAU,IAAI,MAAM,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAC1F,IAAI,OAAO,CAAC,CAAC,aAAa,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAEzD,OAAO;QACL,MAAM;QACN,0EAA0E;QAC1E,uEAAuE;QACvE,KAAK,EAAE,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW;QACtD,YAAY,EAAE,CAAC,CAAC,aAAa;QAC7B,UAAU,EAAE,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE;QAClE,SAAS,EAAE,OAAO,CAAC,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAChD,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,EAAE,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACnD,GAAG,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,CAAA;AACH,CAAC"}

package/dist/s3.d.ts

@@ -0,0 +1,111 @@
+/**
+ * Object storage, signed by hand.
+ *
+ * `NACRE_S3_*` has been in `docs/config.md` since before there was a server and
+ * read by nothing — not used, not validated, so a wrong endpoint or a missing
+ * credential was silent while the `full` profile started a MinIO nobody talked
+ * to. This is the client that ends that.
+ *
+ * ─── why not the AWS SDK ───
+ *
+ * Four operations are needed: put, get, delete, head. `@aws-sdk/client-s3` is
+ * tens of megabytes and hundreds of transitive packages, in a container whose
+ * job is to read documents other people send it — which is the argument
+ * `metrics.ts` already makes for not taking prom-client. SigV4 is a documented
+ * algorithm with a published set of test vectors, it fits in this file, and it
+ * has no release cadence to track.
+ *
+ * The rule that comes with that choice: **anything here is verified against a
+ * real MinIO before it is believed**, because a signing bug produces a 403 that
+ * says nothing about which of the six inputs was wrong.
+ *
+ * ─── what is deliberately not here ───
+ *
+ * No multipart upload, so an object is one PUT and `NACRE_MAX_DOCUMENT_BYTES`
+ * bounds it. No listing, because nothing needs to enumerate a bucket and a
+ * paginated list is where this file would stop being small. No retries: the
+ * callers are the ingest queue and the collector, both of which already retry
+ * whole units of work and would otherwise retry twice.
+ */
+export interface S3Options {
+    /** Including the scheme and port, e.g. `http://minio:9000`. No trailing slash. */
+    readonly endpoint: string;
+    readonly bucket: string;
+    readonly region: string;
+    readonly accessKey: string;
+    readonly secretKey: string;
+    /**
+     * `{endpoint}/{bucket}/{key}` rather than `{bucket}.{endpoint}/{key}`.
+     *
+     * True for MinIO and for anything reached by IP or through a name that is not
+     * a wildcard DNS entry, which is every self-hosted deployment. AWS itself
+     * wants it false.
+     */
+    readonly forcePathStyle: boolean;
+}
+export declare class S3Error extends Error {
+    readonly status: number;
+    readonly method: string;
+    readonly key: string;
+    constructor(status: number, method: string, key: string, body: string);
+}
+export declare class S3 {
+    #private;
+    constructor(options: S3Options);
+    get bucket(): string;
+    /** Store an object, overwriting whatever was there. */
+    put(key: string, body: Uint8Array, contentType?: string): Promise<void>;
+    /** Fetch an object. `undefined` when it is not there, which is not an error. */
+    get(key: string): Promise<Uint8Array | undefined>;
+    /**
+     * Remove an object. Absent is success.
+     *
+     * S3 answers `204` whether or not the key existed, and the caller — the
+     * collector, purging a deleted document — must be able to run twice.
+     */
+    remove(key: string): Promise<void>;
+    /**
+     * A URL that fetches one object, for a while, without a credential.
+     *
+     * Query-string SigV4 rather than the header form: the point is a link a
+     * client can follow, so everything that authenticates it has to be in the
+     * URL. `host` is the only signed header, and the payload is unsigned because
+     * a GET has none.
+     *
+     * **This is a bearer capability and it leaves the permission model behind.**
+     * Nacre checks `read` once, when the link is minted, and the object store
+     * checks nothing afterwards — a revocation during the window does not reach a
+     * URL already handed out, and neither does a delete. That is what presigning
+     * is, in every implementation of it, and it is why the lifetime is a
+     * configured number rather than a constant: `NACRE_PRESIGN_TTL` is how long a
+     * deployment is willing for that to be true.
+     *
+     * It is also why the caller of this decides, not this: minting one per search
+     * hit would issue ten capabilities to answer a question about relevance, most
+     * never followed. See the note on the search response.
+     */
+    presign(key: string, ttlSeconds: number): string;
+    /**
+     * Whether the bucket answers, for readiness.
+     *
+     * A HEAD on the bucket rather than on an object: it needs no key to exist and
+     * it fails for the three reasons that matter — unreachable, wrong bucket,
+     * wrong credential. A TCP connect would pass all three.
+     */
+    ready(): Promise<boolean>;
+}
+/**
+ * Where a document's bytes live.
+ *
+ * Derived from identity — organization, layer, external id — and never from the
+ * content. Content addressing would let two documents share an object, and then
+ * deleting one would either strand the other or need a reference count that
+ * nothing else in this system has. One document, one object, and a re-ingest
+ * overwrites in place.
+ *
+ * The external id is hashed rather than escaped: it is caller-chosen text that
+ * may contain anything, and a key derived by escaping is a key whose length and
+ * shape a caller controls.
+ */
+export declare function documentKey(orgId: string, layerId: string, externalId: string): string;
+//# sourceMappingURL=s3.d.ts.map

package/dist/s3.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../s3.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,WAAW,SAAS;IACxB,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B;;;;;;OAMG;IACH,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;CACjC;AA8BD,qBAAa,OAAQ,SAAQ,KAAK;IAE9B,QAAQ,CAAC,MAAM,EAAE,MAAM;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM;gBAFX,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM;CAQf;AAED,qBAAa,EAAE;;gBAGD,OAAO,EAAE,SAAS;IAI9B,IAAI,MAAM,IAAI,MAAM,CAEnB;IAuGD,uDAAuD;IACjD,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,SAA6B,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjG,gFAAgF;IAC1E,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAOvD;;;;;OAKG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOxC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM;IA2ChD;;;;;;OAMG;IACG,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC;CAiBhC;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAEtF"}

package/dist/s3.js

@@ -0,0 +1,242 @@
+import { createHash, createHmac } from 'node:crypto';
+const UNSIGNED = 'UNSIGNED-PAYLOAD';
+const ALGORITHM = 'AWS4-HMAC-SHA256';
+const SERVICE = 's3';
+const sha256 = (data) => createHash('sha256').update(data).digest('hex');
+const hmac = (key, data) => createHmac('sha256', key).update(data, 'utf8').digest();
+/**
+ * Percent-encode one path segment the way SigV4 wants it.
+ *
+ * `encodeURIComponent` leaves `!'()*` alone and AWS does not, and it encodes
+ * nothing else differently — so this is that function plus five characters. A
+ * key containing any of them signs one way and is requested another, which is a
+ * 403 with no clue in it.
+ */
+function encodeSegment(segment) {
+    return encodeURIComponent(segment).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
+}
+/** Keys are `a/b/c`; each segment is encoded and the separators are not. */
+const encodeKey = (key) => key.split('/').map(encodeSegment).join('/');
+export class S3Error extends Error {
+    status;
+    method;
+    key;
+    constructor(status, method, key, body) {
+        // The body, because S3 and MinIO both answer with an XML document naming
+        // the exact code — SignatureDoesNotMatch, NoSuchBucket, AccessDenied — and
+        // a status alone sends the reader to the wrong one of six inputs.
+        super(`s3 ${method} ${key} failed: ${status} ${body.slice(0, 512)}`);
+        this.status = status;
+        this.method = method;
+        this.key = key;
+        this.name = 'S3Error';
+    }
+}
+export class S3 {
+    #options;
+    constructor(options) {
+        this.#options = { ...options, endpoint: options.endpoint.replace(/\/+$/, '') };
+    }
+    get bucket() {
+        return this.#options.bucket;
+    }
+    #url(key) {
+        const { endpoint, bucket, forcePathStyle } = this.#options;
+        if (forcePathStyle)
+            return new URL(`${endpoint}/${bucket}/${encodeKey(key)}`);
+        const url = new URL(endpoint);
+        url.host = `${bucket}.${url.host}`;
+        url.pathname = `/${encodeKey(key)}`;
+        return url;
+    }
+    /**
+     * Sign a request, returning the headers to send with it.
+     *
+     * The payload hash goes in `x-amz-content-sha256` and into the canonical
+     * request, so a body that changes in flight fails the signature rather than
+     * being stored. `UNSIGNED-PAYLOAD` is used for GET and DELETE, which have
+     * none — hashing the empty string would also work and this is what every
+     * other implementation sends.
+     */
+    #sign(input) {
+        const { region, accessKey, secretKey } = this.#options;
+        const amzDate = input.now.toISOString().replace(/[-:]|\.\d{3}/g, '');
+        const dateStamp = amzDate.slice(0, 8);
+        const headers = {
+            ...input.headers,
+            host: input.url.host,
+            'x-amz-content-sha256': input.payloadHash,
+            'x-amz-date': amzDate,
+        };
+        // Lowercased, sorted, values trimmed — the canonical form. Getting the
+        // order wrong is a 403 that reads exactly like a wrong secret.
+        const canonicalHeaderNames = Object.keys(headers)
+            .map((h) => h.toLowerCase())
+            .sort();
+        const canonicalHeaders = canonicalHeaderNames
+            .map((name) => {
+            const key = Object.keys(headers).find((h) => h.toLowerCase() === name);
+            return `${name}:${String(headers[key]).trim()}\n`;
+        })
+            .join('');
+        const signedHeaders = canonicalHeaderNames.join(';');
+        const canonicalRequest = [
+            input.method,
+            input.url.pathname,
+            // No request here takes a query string. An empty one is still a line.
+            '',
+            canonicalHeaders,
+            signedHeaders,
+            input.payloadHash,
+        ].join('\n');
+        const scope = `${dateStamp}/${region}/${SERVICE}/aws4_request`;
+        const stringToSign = [ALGORITHM, amzDate, scope, sha256(canonicalRequest)].join('\n');
+        const signingKey = hmac(hmac(hmac(hmac(`AWS4${secretKey}`, dateStamp), region), SERVICE), 'aws4_request');
+        const signature = createHmac('sha256', signingKey).update(stringToSign, 'utf8').digest('hex');
+        return {
+            ...headers,
+            authorization: `${ALGORITHM} Credential=${accessKey}/${scope}, ` +
+                `SignedHeaders=${signedHeaders}, Signature=${signature}`,
+        };
+    }
+    async #send(method, key, body, extraHeaders = {}) {
+        const url = this.#url(key);
+        const headers = this.#sign({
+            method,
+            url,
+            payloadHash: body === undefined ? UNSIGNED : sha256(body),
+            headers: body === undefined
+                ? extraHeaders
+                : { ...extraHeaders, 'content-length': String(body.length) },
+            now: new Date(),
+        });
+        return fetch(url, {
+            method,
+            headers,
+            ...(body === undefined ? {} : { body }),
+        });
+    }
+    /** Store an object, overwriting whatever was there. */
+    async put(key, body, contentType = 'application/octet-stream') {
+        const response = await this.#send('PUT', key, body, { 'content-type': contentType });
+        if (!response.ok)
+            throw new S3Error(response.status, 'PUT', key, await response.text());
+    }
+    /** Fetch an object. `undefined` when it is not there, which is not an error. */
+    async get(key) {
+        const response = await this.#send('GET', key);
+        if (response.status === 404)
+            return undefined;
+        if (!response.ok)
+            throw new S3Error(response.status, 'GET', key, await response.text());
+        return new Uint8Array(await response.arrayBuffer());
+    }
+    /**
+     * Remove an object. Absent is success.
+     *
+     * S3 answers `204` whether or not the key existed, and the caller — the
+     * collector, purging a deleted document — must be able to run twice.
+     */
+    async remove(key) {
+        const response = await this.#send('DELETE', key);
+        if (!response.ok && response.status !== 404) {
+            throw new S3Error(response.status, 'DELETE', key, await response.text());
+        }
+    }
+    /**
+     * A URL that fetches one object, for a while, without a credential.
+     *
+     * Query-string SigV4 rather than the header form: the point is a link a
+     * client can follow, so everything that authenticates it has to be in the
+     * URL. `host` is the only signed header, and the payload is unsigned because
+     * a GET has none.
+     *
+     * **This is a bearer capability and it leaves the permission model behind.**
+     * Nacre checks `read` once, when the link is minted, and the object store
+     * checks nothing afterwards — a revocation during the window does not reach a
+     * URL already handed out, and neither does a delete. That is what presigning
+     * is, in every implementation of it, and it is why the lifetime is a
+     * configured number rather than a constant: `NACRE_PRESIGN_TTL` is how long a
+     * deployment is willing for that to be true.
+     *
+     * It is also why the caller of this decides, not this: minting one per search
+     * hit would issue ten capabilities to answer a question about relevance, most
+     * never followed. See the note on the search response.
+     */
+    presign(key, ttlSeconds) {
+        const { region, accessKey, secretKey } = this.#options;
+        const url = this.#url(key);
+        const now = new Date();
+        const amzDate = now.toISOString().replace(/[-:]|\.\d{3}/g, '');
+        const dateStamp = amzDate.slice(0, 8);
+        const scope = `${dateStamp}/${region}/${SERVICE}/aws4_request`;
+        // Sorted by key, and encoded the same way the canonical request will be —
+        // a mismatch between the query that is signed and the query that is sent is
+        // a 403 with nothing in it about which of the two was wrong.
+        const params = [
+            ['X-Amz-Algorithm', ALGORITHM],
+            ['X-Amz-Credential', `${accessKey}/${scope}`],
+            ['X-Amz-Date', amzDate],
+            ['X-Amz-Expires', String(Math.max(1, Math.floor(ttlSeconds)))],
+            ['X-Amz-SignedHeaders', 'host'],
+        ];
+        const canonicalQuery = params
+            .map(([k, v]) => [encodeSegment(k), encodeSegment(v)])
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([k, v]) => `${k}=${v}`)
+            .join('&');
+        const canonicalRequest = [
+            'GET',
+            url.pathname,
+            canonicalQuery,
+            `host:${url.host}\n`,
+            'host',
+            UNSIGNED,
+        ].join('\n');
+        const stringToSign = [ALGORITHM, amzDate, scope, sha256(canonicalRequest)].join('\n');
+        const signingKey = hmac(hmac(hmac(hmac(`AWS4${secretKey}`, dateStamp), region), SERVICE), 'aws4_request');
+        const signature = createHmac('sha256', signingKey).update(stringToSign, 'utf8').digest('hex');
+        return `${url.origin}${url.pathname}?${canonicalQuery}&X-Amz-Signature=${signature}`;
+    }
+    /**
+     * Whether the bucket answers, for readiness.
+     *
+     * A HEAD on the bucket rather than on an object: it needs no key to exist and
+     * it fails for the three reasons that matter — unreachable, wrong bucket,
+     * wrong credential. A TCP connect would pass all three.
+     */
+    async ready() {
+        const url = this.#options.forcePathStyle
+            ? new URL(`${this.#options.endpoint}/${this.#options.bucket}`)
+            : this.#url('');
+        const headers = this.#sign({
+            method: 'HEAD',
+            url,
+            payloadHash: UNSIGNED,
+            headers: {},
+            now: new Date(),
+        });
+        const response = await fetch(url, { method: 'HEAD', headers });
+        if (!response.ok) {
+            throw new S3Error(response.status, 'HEAD', this.#options.bucket, response.statusText);
+        }
+        return true;
+    }
+}
+/**
+ * Where a document's bytes live.
+ *
+ * Derived from identity — organization, layer, external id — and never from the
+ * content. Content addressing would let two documents share an object, and then
+ * deleting one would either strand the other or need a reference count that
+ * nothing else in this system has. One document, one object, and a re-ingest
+ * overwrites in place.
+ *
+ * The external id is hashed rather than escaped: it is caller-chosen text that
+ * may contain anything, and a key derived by escaping is a key whose length and
+ * shape a caller controls.
+ */
+export function documentKey(orgId, layerId, externalId) {
+    return `org/${orgId}/layer/${layerId}/${sha256(externalId)}`;
+}
+//# sourceMappingURL=s3.js.map

package/dist/s3.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3.js","sourceRoot":"","sources":["../s3.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAiDpD,MAAM,QAAQ,GAAG,kBAAkB,CAAA;AACnC,MAAM,SAAS,GAAG,kBAAkB,CAAA;AACpC,MAAM,OAAO,GAAG,IAAI,CAAA;AAEpB,MAAM,MAAM,GAAG,CAAC,IAAyB,EAAU,EAAE,CACnD,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAEjD,MAAM,IAAI,GAAG,CAAC,GAAwB,EAAE,IAAY,EAAU,EAAE,CAC9D,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,MAAM,EAAE,CAAA;AAEzD;;;;;;;GAOG;AACH,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAC,OAAO,CACxC,UAAU,EACV,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,CACxD,CAAA;AACH,CAAC;AAED,4EAA4E;AAC5E,MAAM,SAAS,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAEtF,MAAM,OAAO,OAAQ,SAAQ,KAAK;IAErB;IACA;IACA;IAHX,YACW,MAAc,EACd,MAAc,EACd,GAAW,EACpB,IAAY;QAEZ,yEAAyE;QACzE,2EAA2E;QAC3E,kEAAkE;QAClE,KAAK,CAAC,MAAM,MAAM,IAAI,GAAG,YAAY,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;QAR3D,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAAQ;QACd,QAAG,GAAH,GAAG,CAAQ;QAOpB,IAAI,CAAC,IAAI,GAAG,SAAS,CAAA;IACvB,CAAC;CACF;AAED,MAAM,OAAO,EAAE;IACJ,QAAQ,CAAW;IAE5B,YAAY,OAAkB;QAC5B,IAAI,CAAC,QAAQ,GAAG,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAA;IAChF,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAA;IAC7B,CAAC;IAED,IAAI,CAAC,GAAW;QACd,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAA;QAC1D,IAAI,cAAc;YAAE,OAAO,IAAI,GAAG,CAAC,GAAG,QAAQ,IAAI,MAAM,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC7E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC7B,GAAG,CAAC,IAAI,GAAG,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAA;QAClC,GAAG,CAAC,QAAQ,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAA;QACnC,OAAO,GAAG,CAAA;IACZ,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,KAML;QACC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAA;QACtD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAA;QACpE,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAErC,MAAM,OAAO,GAA2B;YACtC,GAAG,KAAK,CAAC,OAAO;YAChB,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI;YACpB,sBAAsB,EAAE,KAAK,CAAC,WAAW;YACzC,YAAY,EAAE,OAAO;SACtB,CAAA;QAED,uEAAuE;QACvE,+DAA+D;QAC/D,MAAM,oBAAoB,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;aAC9C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;aAC3B,IAAI,EAAE,CAAA;QACT,MAAM,gBAAgB,GAAG,oBAAoB;aAC1C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,IAAI,CAAW,CAAA;YAChF,OAAO,GAAG,IAAI,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAA;QACnD,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAA;QACX,MAAM,aAAa,GAAG,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAEpD,MAAM,gBAAgB,GAAG;YACvB,KAAK,CAAC,MAAM;YACZ,KAAK,CAAC,GAAG,CAAC,QAAQ;YAClB,sEAAsE;YACtE,EAAE;YACF,gBAAgB;YAChB,aAAa;YACb,KAAK,CAAC,WAAW;SAClB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEZ,MAAM,KAAK,GAAG,GAAG,SAAS,IAAI,MAAM,IAAI,OAAO,eAAe,CAAA;QAC9D,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAErF,MAAM,UAAU,GAAG,IAAI,CACrB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,SAAS,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,EAChE,cAAc,CACf,CAAA;QACD,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE7F,OAAO;YACL,GAAG,OAAO;YACV,aAAa,EACX,GAAG,SAAS,eAAe,SAAS,IAAI,KAAK,IAAI;gBACjD,iBAAiB,aAAa,eAAe,SAAS,EAAE;SAC3D,CAAA;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CACT,MAAc,EACd,GAAW,EACX,IAAiB,EACjB,eAAuC,EAAE;QAEzC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;YACzB,MAAM;YACN,GAAG;YACH,WAAW,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;YACzD,OAAO,EACL,IAAI,KAAK,SAAS;gBAChB,CAAC,CAAC,YAAY;gBACd,CAAC,CAAC,EAAE,GAAG,YAAY,EAAE,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YAChE,GAAG,EAAE,IAAI,IAAI,EAAE;SAChB,CAAC,CAAA;QAEF,OAAO,KAAK,CAAC,GAAG,EAAE;YAChB,MAAM;YACN,OAAO;YACP,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;SACxC,CAAC,CAAA;IACJ,CAAC;IAED,uDAAuD;IACvD,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,IAAgB,EAAE,WAAW,GAAG,0BAA0B;QAC/E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC,CAAA;QACpF,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;IACzF,CAAC;IAED,gFAAgF;IAChF,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,SAAS,CAAA;QAC7C,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QACvF,OAAO,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAA;IACrD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QAChD,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5C,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAC1E,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,GAAW,EAAE,UAAkB;QACrC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC1B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAA;QACtB,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAA;QAC9D,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QACrC,MAAM,KAAK,GAAG,GAAG,SAAS,IAAI,MAAM,IAAI,OAAO,eAAe,CAAA;QAE9D,0EAA0E;QAC1E,4EAA4E;QAC5E,6DAA6D;QAC7D,MAAM,MAAM,GAAuB;YACjC,CAAC,iBAAiB,EAAE,SAAS,CAAC;YAC9B,CAAC,kBAAkB,EAAE,GAAG,SAAS,IAAI,KAAK,EAAE,CAAC;YAC7C,CAAC,YAAY,EAAE,OAAO,CAAC;YACvB,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC9D,CAAC,qBAAqB,EAAE,MAAM,CAAC;SAChC,CAAA;QACD,MAAM,cAAc,GAAG,MAAM;aAC1B,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAU,CAAC;aAC9D,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAChD,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;aAC5B,IAAI,CAAC,GAAG,CAAC,CAAA;QAEZ,MAAM,gBAAgB,GAAG;YACvB,KAAK;YACL,GAAG,CAAC,QAAQ;YACZ,cAAc;YACd,QAAQ,GAAG,CAAC,IAAI,IAAI;YACpB,MAAM;YACN,QAAQ;SACT,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEZ,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACrF,MAAM,UAAU,GAAG,IAAI,CACrB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,SAAS,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,EAChE,cAAc,CACf,CAAA;QACD,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE7F,OAAO,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,IAAI,cAAc,oBAAoB,SAAS,EAAE,CAAA;IACtF,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,cAAc;YACtC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9D,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACjB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;YACzB,MAAM,EAAE,MAAM;YACd,GAAG;YACH,WAAW,EAAE,QAAQ;YACrB,OAAO,EAAE,EAAE;YACX,GAAG,EAAE,IAAI,IAAI,EAAE;SAChB,CAAC,CAAA;QACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;QAC9D,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAA;QACvF,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;CACF;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,WAAW,CAAC,KAAa,EAAE,OAAe,EAAE,UAAkB;IAC5E,OAAO,OAAO,KAAK,UAAU,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,EAAE,CAAA;AAC9D,CAAC"}