@agnt-sdk/simon-server 0.0.0-stage → 0.1.1

package/dist/notice.d.ts

@@ -0,0 +1,5 @@
+import { type Context } from './context.ts';
+/** The local part this server writes notices as, on every domain it serves; no agent may register it. */
+export declare const POSTMASTER = "postmaster";
+/** Put a notice in the inbox. True confirms residence; false means the host identity is unavailable. Store failures throw. */
+export declare function notify(ctx: Context, address: string, text: string): Promise<boolean>;

package/dist/notice.js

@@ -0,0 +1,61 @@
+/**
+ * Notices: a message from this server itself to an agent here (CLIENT.md register, recovery). It is an
+ * ordinary signed message, from `postmaster@<the agent's domain>`, a solo agent this server keeps for
+ * itself with a key held in its store, so a reader verifies it as it verifies any other. It goes straight
+ * into the inbox: no policy stands between a server and its own agent.
+ */
+import crypto from 'node:crypto';
+import { b64url, domainOf, fromB64url, keyToken, messageId, privateKeyFromSeed, signMessage, timestamp, } from '@agnt-sdk/simon-protocol';
+import { iso, later } from "./context.js";
+import { takeIntoInbox } from "./fanout.js";
+/** The local part this server writes notices as, on every domain it serves; no agent may register it. */
+export const POSTMASTER = 'postmaster';
+const DAY_MS = 86_400_000;
+/** Put a notice in the inbox. True confirms residence; false means the host identity is unavailable. Store failures throw. */
+export async function notify(ctx, address, text) {
+    const { store, now: clock } = ctx;
+    const now = clock();
+    const from = `${POSTMASTER}@${domainOf(address)}`;
+    const seed = await store.valueOnce('postmaster-key', from, b64url(crypto.randomBytes(32)), null, iso(now));
+    const key = privateKeyFromSeed('ed25519', fromB64url(seed));
+    const token = keyToken(key);
+    await store.createAgent({
+        address: from,
+        kind: 'solo',
+        principal: null,
+        keys: [{ key: token, added: iso(now), revoked: null }],
+        mailbox: false,
+        createdAt: iso(now),
+        retiredAt: null,
+    });
+    const author = await store.agentByAddress(from);
+    if (!author ||
+        author.kind !== 'solo' ||
+        author.principal !== null ||
+        author.retiredAt ||
+        (author.authorizedKeys && !author.authorizedKeys.includes(token)) ||
+        !author.keys.some(k => k.key === token && k.revoked === null)) {
+        ctx.log('warn', 'no notice: the postmaster identity is unavailable', { address, from });
+        return false;
+    }
+    const message = signMessage({
+        v: 1,
+        from,
+        to: [address],
+        inReplyTo: null,
+        parts: [{ type: 'text/plain', data: text }],
+        headers: {},
+        sentAt: timestamp(now),
+    }, key);
+    const record = await store.putMessage({
+        id: messageId(message),
+        message,
+        carrier: ctx.config.origin,
+        claims: [],
+        signedClaims: null,
+        receivedAt: iso(now),
+        expiresAt: later(now, ctx.config.retentionDays * DAY_MS),
+    });
+    const added = await takeIntoInbox(ctx, address, record);
+    return added || (await store.inInbox(address, record.id));
+}

package/dist/outbox.d.ts

@@ -0,0 +1,14 @@
+import { type Context } from './context.ts';
+export declare const PER_DESTINATION = 4;
+export declare const CONCURRENCY = 32;
+/** How long the answers a domain's move can cause are retried: the longest a resolver caches (spec section 3). */
+export declare const MOVE_WINDOW_MS = 3600000;
+export declare const SERVING_SINCE = "serving-since";
+export declare class Outbox {
+    #private;
+    constructor(ctx: Context);
+    /** Claim due rows as far as capacity allows, and start attempting them. Returns how many messages it started. */
+    tick(): Promise<number>;
+    /** Wait until every attempt underway has ended. */
+    idle(): Promise<void>;
+}

package/dist/outbox.js

@@ -0,0 +1,356 @@
+/**
+ * The outbox: delivering stored messages to recipient servers (spec section 6), modelled on Postfix's
+ * queue manager.
+ *
+ *   - **Claims.** Due rows are claimed from the store atomically, each leased to this attempt's token, so
+ *     two workers never take the same row, and every outcome is written only under that lease: an attempt
+ *     that outlived its lease writes nothing, and a delivered row is never written again.
+ *   - **Bounded concurrency:** at most CONCURRENCY rows in flight, so at most that many POSTs, and at most
+ *     PER_DESTINATION POSTs to one server at once. Rows for a busy server wait a moment without spending
+ *     an attempt.
+ *   - **Dead and throttled destinations.** A server that refuses connections, cannot be connected to, or
+ *     holds one past the deadline (a tarpit) is skipped for DEAD_MS, and so is its address, so a thousand
+ *     names for one tarpit cost one timeout. A `429` throttles its server until its Retry-After.
+ *   - **Timing.** No attempt before `sentAt`, none at or after the deadline (the earlier of five days after
+ *     acceptance and 143 hours after `sentAt`), when the recipients fail as `expired`. Retries back off
+ *     exponentially from a minute to four hours, honour Retry-After up to an hour, and are capped to fall
+ *     before the deadline. `429` and `5xx` answers, and failures that may pass, are retried; other answers
+ *     are final (spec section 6.3), except two a domain's move can cause while caches still name the old
+ *     server: a `400 invalid` on `recipients` is retried for an hour from the first such answer, the
+ *     recipients' domain resolved again without the cache before each attempt; a `403 bad_signature` is
+ *     retried while this server began serving the author's domain less than an hour ago ("serving since",
+ *     kept in the store by `send`).
+ *   - **One envelope per server,** naming that server and the recipients it serves, with only their hints:
+ *     a message's recipient domains are resolved together, and those naming the same server share one. A
+ *     server document must be readable before anything is posted.
+ *   - Recipients on a domain naming this server become arrivals here, offered through the admit chain.
+ *
+ * Caps, dead and throttled destinations are this process's; several workers on one store each keep their own.
+ */
+import crypto from 'node:crypto';
+import { canonicalBytes, domainOf, envelopeFor, parseProblem, signEnvelope, } from '@agnt-sdk/simon-protocol';
+import { iso, later } from "./context.js";
+import { arrivalsFor } from "./fanout.js";
+import { Unreachable } from "./net/fetch.js";
+export const PER_DESTINATION = 4;
+export const CONCURRENCY = 32;
+const LEASE_MS = 2 * 60_000;
+const MIN_BACKOFF_MS = 60_000;
+const MAX_BACKOFF_MS = 4 * 3600_000;
+const MAX_RETRY_AFTER_S = 3600;
+const DEAD_MS = 5 * 60_000;
+const CAPACITY_WAIT_MS = 5_000;
+const POST_TIMEOUT_MS = 30_000;
+const ANSWER_BYTES = 65_536;
+/** The last retry is scheduled this long before the deadline, at the latest. */
+const LAST_CHANCE_MS = 1_000;
+/** Outcomes of resolution that mean a domain has no server to deliver to. */
+const NO_SERVER = ['invalid_domain', 'invalid_record', 'conflict', 'not_found'];
+/** How long the answers a domain's move can cause are retried: the longest a resolver caches (spec section 3). */
+export const MOVE_WINDOW_MS = 3600_000;
+/** Store scopes: when a row was first answered 400 on its recipients; when this server began serving a domain. */
+const RECIPIENTS_REFUSED = 'recipients-refused';
+/** Per-row absolute migration cutoffs, retained through the normal delivery deadline. */
+const MOVE_CUTOFF = 'delivery-move-cutoff';
+export const SERVING_SINCE = 'serving-since';
+export class Outbox {
+    #ctx;
+    #running = new Set();
+    /** POSTs in flight, per server. */
+    #posting = new Map();
+    /** Servers skipped until a time: dead, or throttled by a 429. */
+    #blocked = new Map();
+    /** Addresses that refused or held a connection, skipped until a time. */
+    #deadAddresses = new Map();
+    /** Rows claimed and not yet settled. */
+    #active = 0;
+    constructor(ctx) {
+        this.#ctx = ctx;
+    }
+    /** Claim due rows as far as capacity allows, and start attempting them. Returns how many messages it started. */
+    async tick() {
+        const room = CONCURRENCY - this.#active;
+        if (room <= 0)
+            return 0;
+        const now = this.#ctx.now();
+        const busy = [...new Set([...this.#posting.keys(), ...this.#blocked.keys()])].filter(s => !this.#open(s, now));
+        const token = crypto.randomUUID();
+        const rows = await this.#ctx.store.claimDeliveries(iso(now), room, busy, { token, until: later(now, LEASE_MS) });
+        this.#active += rows.length;
+        const byMessage = Map.groupBy(rows, r => r.id);
+        for (const [id, group] of byMessage) {
+            const attempt = this.#attempt(id, group, token)
+                .catch(e => this.#ctx.log('error', 'delivery attempt failed', { id, error: String(e) }))
+                .finally(() => {
+                this.#active -= group.length;
+                this.#running.delete(attempt);
+            });
+            this.#running.add(attempt);
+        }
+        return byMessage.size;
+    }
+    /** Wait until every attempt underway has ended. */
+    async idle() {
+        while (this.#running.size)
+            await Promise.all(this.#running);
+    }
+    /** Whether a server may take another POST now: not blocked, and under its cap. */
+    #open(server, now) {
+        return !blockedUntil(this.#blocked, server, now) && (this.#posting.get(server) ?? 0) < PER_DESTINATION;
+    }
+    async #attempt(id, claimed, token) {
+        const ctx = this.#ctx;
+        const now = ctx.now();
+        const record = await ctx.store.message(id);
+        const rows = record ? claimed.filter(r => now < new Date(r.deadline)) : [];
+        const expired = claimed.filter(r => !rows.includes(r));
+        if (expired.length)
+            await this.#fail(expired, 'expired', token);
+        if (!record)
+            return;
+        const byServer = new Map();
+        for (const row of rows) {
+            const route = await this.#route(row, record.message.from);
+            const r = { ...row, server: 'server' in route ? route.server : row.server };
+            if ('retry' in route)
+                await this.#retry([r], route.retry, token);
+            else if ('fail' in route)
+                await this.#fail([r], route.fail, token);
+            else
+                byServer.set(route.server, [...(byServer.get(route.server) ?? []), r]);
+        }
+        await Promise.all([...byServer].map(([server, group]) => this.#deliverTo(server, record, group, token)));
+    }
+    /**
+     * Where a row's domain delivers to now, or why not; resolved afresh while its recipients were refused. The
+     * lookups are its author's, in the author's share of the server's own pool, so no agent's slow recipient
+     * domains hold up another's.
+     */
+    async #route(row, author) {
+        try {
+            const { store, now } = this.#ctx;
+            const at = now();
+            const move = await store.value(MOVE_CUTOFF, rowKey(row), iso(at));
+            if (move && at.getTime() >= Date.parse(move.until))
+                return { fail: move.reason };
+            const first = await store.value(RECIPIENTS_REFUSED, rowKey(row), iso(at));
+            if (first && at.getTime() >= Date.parse(first) + MOVE_WINDOW_MS)
+                return { fail: 'invalid' };
+            const fresh = move?.fresh || first !== null;
+            const peer = await this.#ctx.resolver.peer(row.domain, { agent: author }, { fresh });
+            if (peer.outcome !== 'server') {
+                return NO_SERVER.includes(peer.outcome) ? { fail: peer.outcome } : { retry: peer.outcome };
+            }
+            if (!peer.document)
+                return { retry: 'no_server_document' };
+            return { server: peer.server };
+        }
+        catch {
+            return { retry: 'temporary' };
+        }
+    }
+    async #deliverTo(server, record, rows, token) {
+        const ctx = this.#ctx;
+        const now = ctx.now();
+        const recipients = rows.flatMap(r => r.recipients);
+        const hints = Object.assign({}, ...rows.map(r => r.hints));
+        if (server === ctx.config.origin) {
+            await ctx.store.putMessage(record, { arrivals: arrivalsFor(record.id, recipients, hints, now) });
+            ctx.wake();
+            return this.#delivered(rows, server, token);
+        }
+        if (!this.#open(server, now)) {
+            const until = Math.max(blockedUntil(this.#blocked, server, now), now.getTime() + CAPACITY_WAIT_MS);
+            for (const r of rows)
+                await ctx.store.settleDelivery({ ...r, nextAttemptAt: iso(new Date(until)) }, token);
+            return;
+        }
+        this.#posting.set(server, (this.#posting.get(server) ?? 0) + 1);
+        try {
+            const outcome = await this.#post(server, record, recipients, hints);
+            if ('delivered' in outcome)
+                await this.#delivered(rows, server, token);
+            else if ('fail' in outcome) {
+                // Rows sharing one destination can have different migration histories.
+                for (const row of rows) {
+                    if (await this.#moved(record, row, outcome))
+                        await this.#retry([row], outcome.fail, token);
+                    else
+                        await this.#fail([row], outcome.fail, token);
+                }
+            }
+            else
+                await this.#retry(rows, outcome.retry, token, outcome.afterS);
+        }
+        finally {
+            const n = this.#posting.get(server) - 1;
+            if (n)
+                this.#posting.set(server, n);
+            else
+                this.#posting.delete(server);
+        }
+    }
+    async #post(server, record, recipients, hints) {
+        const ctx = this.#ctx;
+        // The claims as they were signed for the message at send, without this server's judgement of them.
+        const claims = record.signedClaims?.claims;
+        const envelope = signEnvelope(envelopeFor(record.message, { server, recipients }, { ...(claims ? { claims } : {}), hints }), ctx.config.serverKey);
+        try {
+            const res = await ctx.http.request({
+                method: 'POST',
+                url: `${server}/simon/deliver`,
+                headers: { 'content-type': 'application/json' },
+                body: canonicalBytes(envelope),
+                maxBytes: ANSWER_BYTES,
+                timeoutMs: POST_TIMEOUT_MS,
+                avoid: address => blockedUntil(this.#deadAddresses, address, ctx.now()) > 0,
+            });
+            if (res.status >= 200 && res.status < 300)
+                return { delivered: true };
+            const problem = parseProblem(res.status, res.body);
+            const code = problem?.code ?? `http_${res.status}`;
+            if (res.status !== 429 && res.status < 500)
+                return { fail: code, field: problem?.details?.field, status: res.status };
+            const afterS = retryAfter(res.headers['retry-after'], ctx.now());
+            if (res.status === 429)
+                this.#blocked.set(server, ctx.now().getTime() + (afterS ?? 60) * 1000);
+            return { retry: code, afterS };
+        }
+        catch (e) {
+            const kind = e instanceof Unreachable ? e.kind : 'failed';
+            if (kind === 'refused' || kind === 'connect' || kind === 'timeout') {
+                const until = ctx.now().getTime() + DEAD_MS;
+                this.#blocked.set(server, until);
+                if (e instanceof Unreachable && e.address)
+                    this.#deadAddresses.set(e.address, until);
+            }
+            return { retry: kind };
+        }
+    }
+    /**
+     * Whether a refusal is one a domain's move can cause, still within its hour (spec section 6.3): a `400
+     * invalid` on `recipients`, an hour from the first such answer for the row, which is then resolved afresh
+     * before each attempt; a `403 bad_signature` while this server began serving the author's domain less than
+     * an hour ago.
+     */
+    async #moved(record, row, outcome) {
+        const { store, now } = this.#ctx;
+        const at = now();
+        if (!((outcome.status === 403 && outcome.fail === 'bad_signature') ||
+            (outcome.status === 400 && outcome.fail === 'invalid' && outcome.field === 'recipients')))
+            return false;
+        const existing = await store.value(MOVE_CUTOFF, rowKey(row), iso(at));
+        if (existing) {
+            if (outcome.status === 400 && outcome.field === 'recipients') {
+                // A later routing mismatch requires fresh resolution without extending the existing cutoff.
+                await store.valueOnce(RECIPIENTS_REFUSED, rowKey(row), iso(at), row.deadline, iso(at));
+            }
+            return at.getTime() < Date.parse(existing.until);
+        }
+        let cutoff = null;
+        if (outcome.fail === 'invalid' && outcome.field === 'recipients') {
+            const first = await store.valueOnce(RECIPIENTS_REFUSED, rowKey(row), iso(at), row.deadline, iso(at));
+            cutoff = { until: later(new Date(first), MOVE_WINDOW_MS), reason: 'invalid', fresh: true };
+        }
+        else if (outcome.fail === 'bad_signature') {
+            const noted = await store.value(SERVING_SINCE, domainOf(record.message.from), iso(at));
+            if (typeof noted === 'object' && noted !== null) {
+                const since = Date.parse(noted.since);
+                if (since <= at.getTime() && at.getTime() < since + MOVE_WINDOW_MS) {
+                    cutoff = { until: later(new Date(since), MOVE_WINDOW_MS), reason: 'bad_signature', fresh: false };
+                }
+            }
+        }
+        if (!cutoff)
+            return false;
+        cutoff.until = iso(new Date(Math.min(Date.parse(cutoff.until), Date.parse(row.deadline))));
+        const fixed = await store.valueOnce(MOVE_CUTOFF, rowKey(row), cutoff, row.deadline, iso(at));
+        return at.getTime() < Date.parse(fixed.until);
+    }
+    async #delivered(rows, server, token) {
+        const settled = [];
+        for (const r of rows) {
+            const row = { ...r, server, status: 'delivered', reason: null, attempts: r.attempts + 1 };
+            if (await this.#ctx.store.settleDelivery(row, token))
+                settled.push(r);
+        }
+        if (settled.length) {
+            await this.#ctx.hooks.emit('delivered', {
+                id: rows[0].id,
+                server,
+                recipients: settled.flatMap(r => r.recipients),
+            });
+        }
+    }
+    async #fail(rows, reason, token) {
+        const { store, hooks } = this.#ctx;
+        const failed = [];
+        for (const r of rows) {
+            const attempts = r.attempts + 1;
+            if (reason === 'not_found' && (await this.#bridged(r))) {
+                await store.settleDelivery({ ...r, status: 'delivered', reason: null, attempts }, token);
+            }
+            else if (await store.settleDelivery({ ...r, status: 'failed', reason, attempts }, token)) {
+                failed.push(r);
+            }
+        }
+        if (!failed.length)
+            return;
+        await hooks.emit('deliveryFailed', {
+            id: failed[0].id,
+            server: failed[0].server,
+            recipients: failed.flatMap(r => r.recipients),
+            reason,
+        });
+    }
+    /** Whether an extension took recipients whose domain names no SIMON server (a bridge to email, say). */
+    async #bridged(row) {
+        const record = await this.#ctx.store.message(row.id);
+        return (!!record &&
+            this.#ctx.hooks.emit('unroutable', {
+                id: row.id,
+                domain: row.domain,
+                recipients: row.recipients,
+                message: record.message,
+            }));
+    }
+    /** Try again after backoff or Retry-After, before the deadline; past the last chance, the rows expire. */
+    async #retry(rows, reason, token, afterS) {
+        const now = this.#ctx.now().getTime();
+        const expired = [];
+        for (const r of rows) {
+            const backoff = Math.min(MIN_BACKOFF_MS * 2 ** r.attempts, MAX_BACKOFF_MS);
+            const delay = afterS === undefined ? backoff : Math.max(afterS * 1000, MIN_BACKOFF_MS);
+            const move = await this.#ctx.store.value(MOVE_CUTOFF, rowKey(r), iso(new Date(now)));
+            const deadline = Math.min(Date.parse(r.deadline), move ? Date.parse(move.until) : Infinity);
+            const next = Math.min(now + delay, deadline - LAST_CHANCE_MS);
+            if (next <= now && move && Date.parse(move.until) <= Date.parse(r.deadline))
+                await this.#fail([r], move.reason, token);
+            else if (next <= now)
+                expired.push(r);
+            else {
+                const row = { ...r, reason, attempts: r.attempts + 1, nextAttemptAt: iso(new Date(next)) };
+                await this.#ctx.store.settleDelivery(row, token);
+            }
+        }
+        if (expired.length)
+            await this.#fail(expired, 'expired', token);
+    }
+}
+/** A delivery row's key in the store: its message and recipient domain. */
+const rowKey = (row) => `${row.id} ${row.domain}`;
+/** When a key stops being blocked, or 0 when it is not (an entry past its time is forgotten). */
+function blockedUntil(map, key, now) {
+    const until = map.get(key) ?? 0;
+    if (until > now.getTime())
+        return until;
+    map.delete(key);
+    return 0;
+}
+/** Retry-After in seconds, from seconds or an HTTP date, at most an hour; undefined when absent or unreadable. */
+function retryAfter(value, now) {
+    if (!value)
+        return undefined;
+    const s = /^\d+$/.test(value) ? Number(value) : (Date.parse(value) - now.getTime()) / 1000;
+    return Number.isFinite(s) ? Math.min(MAX_RETRY_AFTER_S, Math.max(0, Math.ceil(s))) : undefined;
+}

package/dist/parts.d.ts

@@ -0,0 +1,29 @@
+import { type Context } from './context.ts';
+import type { Route } from './http.ts';
+import type { StoredMessage } from './store/store.ts';
+export declare const PREFETCH_JITTER_MS: number;
+export declare const MAX_FETCHES = 4;
+export declare const MAX_FETCHES_PER_CARRIER = 2;
+export declare const MAX_FETCH_BYTES: number;
+/**
+ * Fetches waiting for one carrier, at most. A part past it is given up at once, as a fetch that failed is,
+ * so no carrier's messages grow the queue without bound; its rows go with the messages that name them.
+ */
+export declare const MAX_PENDING_FETCHES_PER_CARRIER = 4096;
+/** Queue the fetch of an accepted message's content-addressed parts from its carrier. */
+export declare function queuePrefetch(ctx: Context, record: StoredMessage): Promise<void>;
+/**
+ * Runs due part fetches within the caps, in turn across carriers. A carrier's hourly budget and its share of
+ * the fetches underway are kept in the store, so every process running fetches on it shares them, and a
+ * restart starts over on neither; a fetch is claimed in the store before it runs, so no two processes run it.
+ * MAX_FETCHES and MAX_FETCH_BYTES bound each process's own work.
+ */
+export declare class Fetcher {
+    #private;
+    constructor(ctx: Context);
+    /** Start due fetches, as far as the caps allow. Returns how many it started. */
+    tick(): Promise<number>;
+    /** Wait until every fetch underway has ended. */
+    idle(): Promise<void>;
+}
+export declare function partRoutes(ctx: Context): Route[];