@agent-custody/receipts 0.3.0 → 0.5.0

package/dist/log-sink.js

@@ -6,12 +6,22 @@
 import { timingSafeEqual } from "node:crypto";
 import { createServer } from "node:http";
 import { dsseSign } from "./crypto.js";
-import { MerkleLog } from "./log.js";
+import { createHash } from "node:crypto";
+import { leafHash, MerkleLog } from "./log.js";
+import { fileBackend, RateLimiter } from "./log-store.js";
+import { localSigner } from "./signer.js";
 import { TREEHEAD_TYPE } from "./receipt.js";
-function appendSigned(log, key, leaf) {
-    const e = log.append(leaf);
-    const head = { treeSize: e.treeSize, rootHash: e.rootHash, timestamp: new Date().toISOString() };
-    return { inclusion: { leafIndex: e.leafIndex, treeSize: e.treeSize, hashes: e.hashes }, treeHead: dsseSign(TREEHEAD_TYPE, head, key) };
+function signedHead(e, key, logId) {
+    const head = { treeSize: e.treeSize, rootHash: e.rootHash, timestamp: new Date().toISOString(), ...(logId ? { log: logId } : {}) };
+    return dsseSign(TREEHEAD_TYPE, head, key);
+}
+async function signHead(e, signer, logId) {
+    const head = { treeSize: e.treeSize, rootHash: e.rootHash, timestamp: new Date().toISOString(), ...(logId ? { log: logId } : {}) };
+    return signer.sign(TREEHEAD_TYPE, head);
+}
+async function appendSigned(log, signer, leaf, logId) {
+    const e = "leaf" in leaf ? await log.append(leaf.leaf) : await log.appendHash(leaf.leafHash);
+    return { inclusion: { leafIndex: e.leafIndex, treeSize: e.treeSize, hashes: e.hashes }, treeHead: await signHead(e, signer, logId) };
 }
 /** A local JSONL Merkle log. Tree heads are signed with the given key, normally the issuer's own. */
 export function fileLog(file, key) {
@@ -20,29 +30,54 @@ export function fileLog(file, key) {
         kind: "file",
         where: file,
         async append(leaf) {
-            return appendSigned(log, key, leaf);
+            const e = log.append(leaf);
+            return { inclusion: { leafIndex: e.leafIndex, treeSize: e.treeSize, hashes: e.hashes }, treeHead: signedHead(e, key) };
         },
     };
 }
-/** A log reached over HTTP: POST <url>/append with {leaf}, expecting a LogAppend back. */
+/**
+ * A log reached over HTTP: POST <url>/append with {leaf} or {leafHash}, expecting a LogAppend back. A 429 or a 5xx
+ * is retried a few times with backoff, honouring Retry-After; anything else, or the last failure, is the caller's.
+ */
 export function httpLog(url, opts = {}) {
     const f = opts.fetch ?? fetch;
     const base = url.endsWith("/") ? url : `${url}/`;
+    const attempts = opts.retries ?? 3;
     return {
         kind: "http",
         where: url,
         async append(leaf) {
-            const res = await f(new URL("append", base), {
-                method: "POST",
-                headers: { "content-type": "application/json", ...(opts.token ? { authorization: `Bearer ${opts.token}` } : {}) },
-                body: JSON.stringify({ leaf }),
-            });
-            if (!res.ok)
-                throw new Error(`log ${url} refused the append: ${res.status} ${(await res.text()).slice(0, 200)}`);
-            const body = (await res.json());
-            if (!body.inclusion || !body.treeHead)
-                throw new Error(`log ${url} returned a malformed append result`);
-            return body;
+            let last = "";
+            for (let i = 0; i < attempts; i++) {
+                let res;
+                try {
+                    res = await f(new URL("append", base), {
+                        method: "POST",
+                        headers: { "content-type": "application/json", ...(opts.token ? { authorization: `Bearer ${opts.token}` } : {}) },
+                        body: JSON.stringify(opts.hashOnly ? { leafHash: leafHash(leaf).toString("hex") } : { leaf }),
+                    });
+                }
+                catch (e) {
+                    last = `log ${url} unreachable: ${e instanceof Error ? e.message : String(e)}`;
+                    if (i + 1 < attempts)
+                        await new Promise((r) => setTimeout(r, 200 * 2 ** i));
+                    continue;
+                }
+                if (res.ok) {
+                    const body = (await res.json());
+                    if (!body.inclusion || !body.treeHead)
+                        throw new Error(`log ${url} returned a malformed append result`);
+                    return body;
+                }
+                last = `log ${url} refused the append: ${res.status} ${(await res.text()).slice(0, 200)}`;
+                if (res.status !== 429 && res.status < 500)
+                    break;
+                if (i + 1 < attempts) {
+                    const after = Number(res.headers.get("retry-after"));
+                    await new Promise((r) => setTimeout(r, Number.isFinite(after) && after > 0 ? Math.min(after, 5) * 1000 : 200 * 2 ** i));
+                }
+            }
+            throw new Error(last);
         },
     };
 }
@@ -52,12 +87,120 @@ export function openLog(cfg, key) {
         const token = cfg.log.tokenEnv ? process.env[cfg.log.tokenEnv] : undefined;
         if (cfg.log.tokenEnv && !token)
             throw new Error(`log token: environment variable ${cfg.log.tokenEnv} is not set`);
-        return httpLog(cfg.log.url, token === undefined ? {} : { token });
+        return httpLog(cfg.log.url, { ...(token === undefined ? {} : { token }), ...(cfg.log.hashOnly ? { hashOnly: true } : {}) });
     }
     if (!cfg.logFile)
         throw new Error("config needs logFile or log.url");
     return fileLog(cfg.logFile, key);
 }
+const tokenMatches = (tokens, token) => {
+    if (tokens.length === 0)
+        return true;
+    if (!token)
+        return false;
+    const given = Buffer.from(token);
+    return tokens.some((t) => {
+        const want = Buffer.from(t);
+        return want.length === given.length && timingSafeEqual(want, given);
+    });
+};
+/** The reference server's logs: one file for the root paths and, optionally, a file per tenant from the options. */
+export function fileResolver(file, opts = {}) {
+    const root = { backend: fileBackend(file), logId: opts.logId, authorize: async (t) => tokenMatches(opts.tokens ?? [], t) };
+    const tenants = new Map();
+    for (const [name, t] of Object.entries(opts.tenants ?? {}))
+        tenants.set(name, { backend: fileBackend(t.file), logId: t.logId ?? name, authorize: async (tok) => tokenMatches(t.tokens ?? [], tok) });
+    return {
+        async resolve(tenant) {
+            return tenant === null ? root : (tenants.get(tenant) ?? null);
+        },
+        async tenants() {
+            return [null, ...tenants.keys()];
+        },
+    };
+}
+/**
+ * Logs in Postgres: every tenant from the tenants table, each with its own log and tokens; the root paths serve the
+ * tenant named `defaultTenant`, which also accepts `staticTokens` so a server can keep its environment token.
+ */
+export function postgresResolver(tenancy, opts = {}) {
+    const def = opts.defaultTenant ?? "default";
+    return {
+        async resolve(tenant) {
+            const id = tenant ?? def;
+            const t = await tenancy.tenant(id);
+            if (!t || t.disabledAt)
+                return null;
+            const backend = await tenancy.log(id);
+            return {
+                backend,
+                logId: t.logId,
+                authorize: async (tok) => (tenant === null && (opts.staticTokens?.length ?? 0) > 0 && tokenMatches(opts.staticTokens, tok)) || (await tenancy.authorize(id, tok)),
+            };
+        },
+        async tenants() {
+            return (await tenancy.listTenants()).filter((t) => !t.disabledAt).map((t) => (t.id === def ? null : t.id));
+        },
+    };
+}
+/**
+ * Publishes one checkpoint per log whose tree has grown since the last one: the current head, signed, into the
+ * checkpoint store. Call publishOnce on a timer, or start() to run it every `everyMs`.
+ */
+export class CheckpointPublisher {
+    timer = null;
+    resolver;
+    signer;
+    store;
+    everyMs;
+    warn;
+    constructor(resolver, signer, store, everyMs = 300_000, warn = (m) => console.error(m)) {
+        this.resolver = resolver;
+        this.signer = signer;
+        this.store = store;
+        this.everyMs = everyMs;
+        this.warn = warn;
+    }
+    /** Publishes for every log that has grown; returns the checkpoints written. */
+    async publishOnce() {
+        const out = [];
+        for (const tenant of await this.resolver.tenants()) {
+            try {
+                const r = await this.resolver.resolve(tenant);
+                if (!r)
+                    continue;
+                const name = tenant ?? "default";
+                const size = await r.backend.size();
+                if (size === 0)
+                    continue; // an empty tree is not a checkpoint worth publishing
+                const last = await this.store.latest(name);
+                if (last && last.treeSize >= size)
+                    continue;
+                const rootHash = await r.backend.root(size);
+                const envelope = await signHead({ treeSize: size, rootHash }, this.signer, r.logId);
+                const c = { tenant: name, logId: r.logId, treeSize: size, rootHash, signedAt: new Date().toISOString(), envelope };
+                await this.store.save(c);
+                out.push(c);
+            }
+            catch (e) {
+                this.warn(`agent-custody log: checkpoint for ${tenant ?? "default"} failed: ${e instanceof Error ? e.message : String(e)}`);
+            }
+        }
+        return out;
+    }
+    start() {
+        if (this.timer)
+            return;
+        this.timer = setInterval(() => void this.publishOnce(), this.everyMs);
+        this.timer.unref?.();
+        void this.publishOnce();
+    }
+    stop() {
+        if (this.timer)
+            clearInterval(this.timer);
+        this.timer = null;
+    }
+}
 /**
  * The reference log server as a node:http request handler.
  *   POST /append  {leaf}   -> LogAppend, tree head signed with the log's key
@@ -65,67 +208,110 @@ export function openLog(cfg, key) {
  *   GET  /consistency?old=M&new=N -> {oldSize, newSize, hashes}, proof that the log at N extends the log at M
  *   GET  /head             -> {treeHead}, the current tree head signed with the log's key
  */
-export function logHandler(file, key, opts = {}) {
-    const log = new MerkleLog(file);
-    const tokens = opts.tokens ?? [];
-    const authorized = (req) => {
-        if (tokens.length === 0)
-            return true;
+export function logHandler(source, keyOrSigner, opts = {}) {
+    const resolver = typeof source === "string" ? fileResolver(source, opts) : source;
+    const signer = "privateKey" in keyOrSigner ? localSigner(keyOrSigner) : keyOrSigner;
+    const limiter = new RateLimiter(opts.rateLimit);
+    const maxBody = opts.maxBodyBytes ?? 65_536;
+    const bearer = (req) => {
         const h = req.headers.authorization ?? "";
-        const given = Buffer.from(h.startsWith("Bearer ") ? h.slice(7) : "");
-        return tokens.some((t) => {
-            const want = Buffer.from(t);
-            return want.length === given.length && timingSafeEqual(want, given);
-        });
+        return h.startsWith("Bearer ") && h.length > 7 ? h.slice(7) : null;
     };
     return async (req, res) => {
-        const json = (status, body) => {
-            res.writeHead(status, { "content-type": "application/json" });
+        const json = (status, body, headers = {}) => {
+            res.writeHead(status, { "content-type": "application/json", ...headers });
             res.end(JSON.stringify(body));
         };
         const url = new URL(req.url ?? "/", "http://localhost");
-        if (req.method === "POST" && url.pathname.endsWith("/append")) {
-            if (!authorized(req))
-                return json(401, { error: "unauthorized" });
-            let body = "";
-            for await (const chunk of req)
-                body += chunk;
-            let leaf;
+        if (req.method === "GET" && url.pathname === "/.well-known/agent-custody-log.json") {
             try {
-                leaf = JSON.parse(body).leaf;
+                const doc = await signer.keys();
+                const root = await resolver.resolve(null);
+                return json(200, { ...(root?.logId ? { log: root.logId } : {}), ...doc }, { "cache-control": "public, max-age=300" });
             }
-            catch {
-                return json(400, { error: "body must be JSON {leaf}" });
+            catch (e) {
+                return json(503, { error: `keys unavailable: ${e instanceof Error ? e.message : String(e)}` });
             }
-            if (typeof leaf !== "string" || leaf.length === 0)
-                return json(400, { error: "leaf must be a non-empty string" });
-            return json(200, appendSigned(log, key, leaf));
         }
-        if (req.method === "GET" && url.pathname.endsWith("/root")) {
-            const size = url.searchParams.has("size") ? Number(url.searchParams.get("size")) : log.size;
-            if (!Number.isInteger(size) || size < 0 || size > log.size)
-                return json(400, { error: `size must be an integer in 0..${log.size}` });
-            return json(200, { treeSize: size, rootHash: log.root(size) });
+        // /t/<tenant>/<op> reaches that tenant's log; anything else is the default log.
+        const m = /^\/t\/([A-Za-z0-9_.-]+)\/(append|root|consistency|head|checkpoints)$/.exec(url.pathname);
+        let which;
+        try {
+            which = await resolver.resolve(m ? m[1] : null);
         }
-        if (req.method === "GET" && url.pathname.endsWith("/consistency")) {
-            const oldSize = Number(url.searchParams.get("old"));
-            const newSize = url.searchParams.has("new") ? Number(url.searchParams.get("new")) : log.size;
-            if (![oldSize, newSize].every(Number.isInteger) || oldSize < 0 || oldSize > newSize || newSize > log.size)
-                return json(400, { error: `old and new must be integers with 0 <= old <= new <= ${log.size}` });
-            return json(200, { oldSize, newSize, hashes: log.consistencyProof(oldSize, newSize) });
+        catch (e) {
+            return json(503, { error: `log unavailable: ${e instanceof Error ? e.message : String(e)}` });
+        }
+        if (!which)
+            return json(404, { error: "unknown log" });
+        const { backend: log, logId } = which;
+        try {
+            if (req.method === "POST" && url.pathname.endsWith("/append")) {
+                const token = bearer(req);
+                if (!(await which.authorize(token)))
+                    return json(401, { error: "unauthorized" });
+                const limitKey = token ? createHash("sha256").update(token).digest("hex").slice(0, 16) : `addr:${req.socket.remoteAddress ?? "?"}`;
+                if (!limiter.take(limitKey))
+                    return json(429, { error: "too many appends; retry shortly" }, { "retry-after": "1" });
+                let body = "";
+                for await (const chunk of req) {
+                    body += chunk;
+                    if (body.length > maxBody)
+                        return json(413, { error: `append body larger than ${maxBody} bytes` });
+                }
+                let parsed;
+                try {
+                    parsed = JSON.parse(body);
+                }
+                catch {
+                    return json(400, { error: "body must be JSON {leaf} or {leafHash}" });
+                }
+                if (typeof parsed.leafHash === "string") {
+                    if (!/^[0-9a-f]{64}$/.test(parsed.leafHash))
+                        return json(400, { error: "leafHash must be 64 lowercase hex characters" });
+                    return json(200, await appendSigned(log, signer, { leafHash: parsed.leafHash }, logId));
+                }
+                if (typeof parsed.leaf !== "string" || parsed.leaf.length === 0)
+                    return json(400, { error: "leaf must be a non-empty string, or send leafHash" });
+                return json(200, await appendSigned(log, signer, { leaf: parsed.leaf }, logId));
+            }
+            const current = await log.size();
+            if (req.method === "GET" && url.pathname.endsWith("/root")) {
+                const size = url.searchParams.has("size") ? Number(url.searchParams.get("size")) : current;
+                if (!Number.isInteger(size) || size < 0 || size > current)
+                    return json(400, { error: `size must be an integer in 0..${current}` });
+                return json(200, { treeSize: size, rootHash: await log.root(size) });
+            }
+            if (req.method === "GET" && url.pathname.endsWith("/consistency")) {
+                const oldSize = Number(url.searchParams.get("old"));
+                const newSize = url.searchParams.has("new") ? Number(url.searchParams.get("new")) : current;
+                if (![oldSize, newSize].every(Number.isInteger) || oldSize < 0 || oldSize > newSize || newSize > current)
+                    return json(400, { error: `old and new must be integers with 0 <= old <= new <= ${current}` });
+                return json(200, { oldSize, newSize, hashes: await log.consistencyProof(oldSize, newSize) });
+            }
+            if (req.method === "GET" && url.pathname.endsWith("/head")) {
+                return json(200, { treeHead: await signHead({ treeSize: current, rootHash: await log.root(current) }, signer, logId) });
+            }
+            if (req.method === "GET" && url.pathname.endsWith("/checkpoints")) {
+                const since = url.searchParams.has("since") ? Number(url.searchParams.get("since")) : -1;
+                if (!Number.isInteger(since))
+                    return json(400, { error: "since must be an integer tree size" });
+                const list = opts.checkpoints ? await opts.checkpoints.list(m ? m[1] : "default", since) : [];
+                return json(200, { checkpoints: list.map((c) => ({ treeSize: c.treeSize, rootHash: c.rootHash, signedAt: c.signedAt, treeHead: c.envelope })) });
+            }
+            return json(404, { error: "not found" });
         }
-        if (req.method === "GET" && url.pathname.endsWith("/head")) {
-            const head = { treeSize: log.size, rootHash: log.root(), timestamp: new Date().toISOString() };
-            return json(200, { treeHead: dsseSign(TREEHEAD_TYPE, head, key) });
+        catch (e) {
+            return json(500, { error: e instanceof Error ? e.message : String(e) });
         }
-        return json(404, { error: "not found" });
     };
 }
 /** Starts the reference log server. Port 0 picks a free port. */
-export function serveLog(file, key, opts) {
+export function serveLog(source, keyOrSigner, opts) {
     const host = opts.host ?? "127.0.0.1";
+    const handler = logHandler(source, keyOrSigner, opts);
     const server = createServer((req, res) => {
-        void logHandler(file, key, opts)(req, res);
+        void handler(req, res);
     });
     return new Promise((resolve) => {
         server.listen(opts.port, host, () => {

package/dist/log-store.d.ts

@@ -0,0 +1,122 @@
+import { type InclusionProof } from "./log.ts";
+export interface AppendResult extends InclusionProof {
+    rootHash: string;
+}
+/** One log, as the server sees it: everything a Merkle log answers, asynchronously, so a database can stand behind it. */
+export interface LogBackend {
+    size(): Promise<number>;
+    append(leaf: string): Promise<AppendResult>;
+    appendHash(leafHashHex: string): Promise<AppendResult>;
+    root(size?: number): Promise<string>;
+    consistencyProof(oldSize: number, newSize?: number): Promise<string[]>;
+}
+/** The JSONL file log behind the asynchronous interface. */
+export declare function fileBackend(file: string): LogBackend;
+/**
+ * What the Postgres backend needs from a client: the query method of a `pg` Pool and of PGlite, and, for a pool,
+ * `connect` so a transaction runs on one connection. A store never closes a client it was given.
+ */
+export interface PostgresLike {
+    query(text: string, values?: unknown[]): Promise<{
+        rows: unknown[];
+    }>;
+    connect?(): Promise<{
+        query(text: string, values?: unknown[]): Promise<{
+            rows: unknown[];
+        }>;
+        release(): void;
+    }>;
+}
+export interface PostgresLogOptions {
+    /** table name prefix; default "log_", giving log_leaves, log_tenants, log_tokens */
+    prefix?: string;
+}
+/** One tenant's log in Postgres. Leaves are hashes only; the table never holds a receipt. */
+export declare class PostgresLog implements LogBackend {
+    readonly tenant: string;
+    private readonly client;
+    private readonly leaves;
+    private hashes;
+    private cache;
+    private ready;
+    private chain;
+    constructor(client: PostgresLike, tenant: string, opts?: PostgresLogOptions);
+    static ensureSchema(client: PostgresLike, prefix?: string): Promise<void>;
+    private init;
+    /** Replaces the in-memory tree with what the table holds. Called when the table has moved on without this instance. */
+    private reload;
+    private count;
+    private sync;
+    size(): Promise<number>;
+    private commit;
+    append(leaf: string): Promise<AppendResult>;
+    appendHash(leafHashHex: string): Promise<AppendResult>;
+    root(size?: number): Promise<string>;
+    consistencyProof(oldSize: number, newSize?: number): Promise<string[]>;
+}
+export interface Tenant {
+    id: string;
+    logId: string;
+    createdAt: string;
+    disabledAt: string | null;
+}
+export interface TokenRecord {
+    tenantId: string;
+    label: string;
+    /** hex sha256 of the token; the token itself is shown once, at creation */
+    tokenHash: string;
+    createdAt: string;
+    revokedAt: string | null;
+}
+/** Tenants and their tokens, in Postgres. Tokens are stored hashed; a lookup hashes what the caller presented. */
+export declare class PostgresTenancy {
+    private readonly client;
+    private readonly prefix;
+    private readonly logs;
+    private readonly tenantCache;
+    private readonly tokenCache;
+    private ready;
+    constructor(client: PostgresLike, opts?: PostgresLogOptions);
+    private init;
+    private row;
+    /** The tenant, or null. Answers from a ten-second cache, so a disabled tenant is refused within that. */
+    tenant(id: string): Promise<Tenant | null>;
+    /** Whether the presented token is a live token of this tenant. Positive answers are cached for thirty seconds. */
+    authorize(tenantId: string, token: string | null): Promise<boolean>;
+    /** The tenant's log, one instance per tenant per process. */
+    log(tenantId: string): Promise<PostgresLog>;
+    addTenant(id: string, logId?: string): Promise<Tenant>;
+    disableTenant(id: string): Promise<void>;
+    listTenants(): Promise<Tenant[]>;
+    /** Mints a token for a tenant. The token is returned once and stored only as its hash. */
+    addToken(tenantId: string, label: string): Promise<{
+        token: string;
+        tokenHash: string;
+    }>;
+    /** Revokes the tokens of a tenant whose hash starts with the prefix; returns how many. */
+    revokeToken(tenantId: string, hashPrefix: string): Promise<number>;
+    listTokens(tenantId: string): Promise<TokenRecord[]>;
+}
+/**
+ * Copies a JSONL log file into a backend as hashes: leaf strings are hashed, {pruned} and {hash} lines are taken as
+ * they are. Skips the leaves the backend already has, so it can be re-run. Returns how many were added.
+ */
+export declare function importLogFile(file: string, into: LogBackend): Promise<{
+    added: number;
+    total: number;
+}>;
+export interface RateLimitOptions {
+    /** sustained appends per second per key; default 50 */
+    perSecond?: number;
+    /** how many may arrive at once before the limit bites; default 100 */
+    burst?: number;
+}
+/** A token bucket per key, in memory: enough to keep one tenant from crowding out the rest on one instance. */
+export declare class RateLimiter {
+    private readonly perSecond;
+    private readonly burst;
+    private readonly buckets;
+    constructor(opts?: RateLimitOptions);
+    /** Takes one unit for the key; false when the key must wait. */
+    take(key: string, now?: number): boolean;
+}