@ciromaciel/contract 1.2.0 → 1.3.0

package/README.md

@@ -133,6 +133,14 @@ Both return `{ ok: true, subject, email, name, scopes, claims }` or
 the `riligar-auth` issuer only. `jose` (5 or 6) is an **optional** peer, needed
 only by this entrypoint.
 
+### `@ciromaciel/contract/outbox`
+
+```js
+import { OUTBOX_SQL, envelope, enqueueEvent, deliverDue, releaseHeld, retryDelivery } from '@ciromaciel/contract/outbox'
+```
+
+The delivery queue every product shares. Each event is a row in `outbox` before it leaves, one per subscribed endpoint, signed with `./webhooks`. A 2xx is `sent`; 401 and 403 are `held` (probed hourly, released at once by `releaseHeld` when the secret changes); 410 is `failed`; anything else retries after 1 min, 5 min, 30 min, 2 h and 12 h. `OUTBOX_SQL` is the schema to put in the product's first migration. The product supplies a D1-shaped `db` and `decryptSecret(ciphertext)`.
+
 ## Links
 
 - [Website](https://ciromaciel.click)

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@ciromaciel/contract",
-    "version": "1.2.0",
+    "version": "1.3.0",
     "license": "MIT",
     "type": "module",
     "description": "A single response envelope, a closed catalogue of error codes, and helpers that make answering outside it impossible.",
@@ -15,7 +15,8 @@
         "./schemas": "./src/schemas.js",
         "./plan": "./src/plan/index.js",
         "./webhooks": "./src/webhooks.js",
-        "./token": "./src/token.js"
+        "./token": "./src/token.js",
+        "./outbox": "./src/outbox.js"
     },
     "scripts": {
         "test": "bun test tests/"

package/src/outbox.js

@@ -0,0 +1,237 @@
+// ============================================================================
+// @ciromaciel/contract/outbox — the delivery queue every product shares
+// ============================================================================
+//
+//     import { OUTBOX_SQL, enqueueEvent, deliverDue, releaseHeld } from '@ciromaciel/contract/outbox'
+//
+// Every event a product emits leaves through webhook endpoints the account
+// registers, signed with `./webhooks`. WRITTEN BEFORE IT LEAVES: each delivery
+// is a row in `outbox` first (one per endpoint); trying it right away is a
+// courtesy, and the cron is the guarantee.
+//
+// WHAT HAPPENS TO AN ATTEMPT:
+//
+//     2xx              sent
+//     401, 403         held: almost always a rotated secret or a revoked key on
+//                      the other side. Retrying five times would only kill the
+//                      delivery in 14 hours, while the fix is one paste away.
+//                      It waits, probed once an hour, and leaves on its own
+//                      when the endpoint answers 2xx again — or at once when
+//                      the secret is replaced here (`releaseHeld`).
+//     410              failed: the receiver said this address is gone.
+//     anything else    retried after 1 min, 5 min, 30 min, 2 h and 12 h, then
+//                      failed — visible on the screen and resendable by hand.
+//
+// WHY IT LIVES HERE. Videos wrote it, Community copied it, and four products
+// were about to copy it again. One fix should not become six.
+//
+// IT ASKS THE PRODUCT FOR TWO THINGS ONLY: a D1-shaped `db` (prepare, bind,
+// first, all, run, batch) and a `decryptSecret(ciphertext)` that returns the
+// endpoint's `whsec_`. Envelopes are built by the product with `envelope()`.
+
+import { webhookHeaders } from './webhooks.js'
+
+/** Seconds before each retry of an ordinary failure. */
+export const RETRY_DELAYS = [60, 300, 1800, 7200, 43_200]
+
+/** How often a held delivery is probed, in seconds. */
+export const HOLD_PROBE_SECONDS = 3600
+
+const DELIVERY_TIMEOUT_MS = 10_000
+
+/**
+ * The two tables, as products put them in their first migration. Same columns
+ * as Videos' `0002_webhook_endpoints.sql`, plus the `held` status.
+ */
+export const OUTBOX_SQL = `
+CREATE TABLE webhook_endpoints (
+    id                 TEXT PRIMARY KEY,
+    account_id         TEXT NOT NULL,
+    url                TEXT NOT NULL,
+    description        TEXT,
+    events             TEXT NOT NULL DEFAULT '["*"]',      -- JSON array of types, or ["*"]
+    secret_ciphertext  TEXT NOT NULL,
+    secret_prefix      TEXT NOT NULL,
+    secret_source      TEXT NOT NULL DEFAULT 'generated',  -- generated | provided
+    is_active          INTEGER NOT NULL DEFAULT 1,
+    last_status        INTEGER,
+    last_at            INTEGER,
+    created_at         INTEGER NOT NULL,
+    updated_at         INTEGER NOT NULL
+);
+CREATE INDEX idx_webhook_endpoints_account ON webhook_endpoints (account_id, created_at);
+
+CREATE TABLE outbox (
+    id           TEXT PRIMARY KEY,
+    account_id   TEXT NOT NULL,
+    target       TEXT NOT NULL,                           -- webhook_endpoints.id
+    event_id     TEXT NOT NULL,                           -- x-ccm-id: resending is a no-op on the other side
+    type         TEXT NOT NULL,
+    subject_id   TEXT,                                    -- what the event is about (a response, a link), to list deliveries per item
+    body         TEXT NOT NULL,
+    status       TEXT NOT NULL DEFAULT 'pending',         -- pending | sent | held | failed
+    attempts     INTEGER NOT NULL DEFAULT 0,
+    next_at      INTEGER NOT NULL,
+    last_status  INTEGER,
+    last_error   TEXT,
+    created_at   INTEGER NOT NULL,
+    sent_at      INTEGER,
+    UNIQUE (target, event_id)
+);
+CREATE INDEX idx_outbox_due ON outbox (status, next_at);
+CREATE INDEX idx_outbox_account ON outbox (account_id, created_at DESC);
+CREATE INDEX idx_outbox_target ON outbox (target, created_at DESC);
+CREATE INDEX idx_outbox_subject ON outbox (account_id, subject_id);
+`
+
+/** Whether an endpoint's subscription covers `type`. */
+export function subscribes(endpoint, type) {
+    let events
+    try {
+        events = JSON.parse(endpoint.events ?? '["*"]')
+    } catch {
+        events = ['*']
+    }
+    return events.includes('*') || events.includes(type)
+}
+
+/**
+ * The envelope People reads from any source: `{type, timestamp, data}`, the
+ * person in `data.person`, where they came through in `data.channel`, the rest
+ * in `data.properties`. Empty person fields are dropped.
+ */
+export function envelope({ type, at, channel = 'web', person = null, properties = {} }) {
+    const data = { channel, properties }
+    if (person) {
+        const clean = Object.fromEntries(Object.entries(person).filter(([, value]) => value !== undefined && value !== null && value !== ''))
+        if (Object.keys(clean).length) data.person = clean
+    }
+    return JSON.stringify({ type, timestamp: new Date(at * 1000).toISOString(), data })
+}
+
+/**
+ * What one attempt turns into. `attempts` counts this one.
+ *
+ * @returns {{status: 'sent'|'pending'|'held'|'failed', nextIn: number|null}}
+ */
+export function classifyAttempt({ ok, status }, attempts) {
+    if (ok) return { status: 'sent', nextIn: null }
+    if (status === 401 || status === 403) return { status: 'held', nextIn: HOLD_PROBE_SECONDS }
+    if (status === 410) return { status: 'failed', nextIn: null }
+    if (attempts > RETRY_DELAYS.length) return { status: 'failed', nextIn: null }
+    return { status: 'pending', nextIn: RETRY_DELAYS[attempts - 1] }
+}
+
+/**
+ * Queues one event for every active endpoint of the account subscribed to its
+ * type. Returns the outbox ids, so the caller can try them at once.
+ *
+ * @param {object} db D1 or anything shaped like it
+ * @param {{accountId: string, eventId: string, type: string, body: string, at: number, subjectId?: string}} event
+ */
+export async function enqueueEvent(db, { accountId, eventId, type, body, at, subjectId = null }) {
+    const { results: active } = await db.prepare('SELECT id, events FROM webhook_endpoints WHERE account_id = ? AND is_active = 1').bind(accountId).all()
+    const targets = active.filter(endpoint => subscribes(endpoint, type))
+    if (!targets.length) return []
+    const ids = []
+    const statements = targets.map(endpoint => {
+        const id = crypto.randomUUID()
+        ids.push(id)
+        return db
+            .prepare('INSERT OR IGNORE INTO outbox (id, account_id, target, event_id, type, subject_id, body, next_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')
+            .bind(id, accountId, endpoint.id, eventId, type, subjectId, body, at, at)
+    })
+    await db.batch(statements)
+    return ids
+}
+
+async function post(fetchImpl, url, headers, body, userAgent) {
+    const started = Date.now()
+    try {
+        const response = await fetchImpl(url, { method: 'POST', headers: { 'content-type': 'application/json', 'user-agent': userAgent, ...headers }, body, signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS) })
+        return { status: response.status, ok: response.ok, ms: Date.now() - started, error: response.ok ? null : (await response.text()).slice(0, 300) }
+    } catch (error) {
+        return { status: 0, ok: false, ms: Date.now() - started, error: error?.name === 'TimeoutError' ? 'Sem resposta em 10 s.' : 'Não foi possível conectar.' }
+    }
+}
+
+/**
+ * One signed POST to an endpoint: a delivery, or the "Enviar teste" button.
+ *
+ * @param {object} endpoint a `webhook_endpoints` row
+ * @param {{eventId: string, body: string}} delivery
+ * @param {{decryptSecret: (ciphertext: string) => Promise<string>, fetch?: typeof fetch, userAgent?: string}} options
+ */
+export async function sendSigned(endpoint, { eventId, body }, { decryptSecret, fetch: fetchImpl = fetch, userAgent = 'ciromaciel-webhooks/1.0' }) {
+    const secret = await decryptSecret(endpoint.secret_ciphertext)
+    return post(fetchImpl, endpoint.url, await webhookHeaders(secret, { id: eventId, body }), body, userAgent)
+}
+
+/**
+ * Delivers due rows: those `ids`, or every pending or held row whose time has
+ * come. Returns counts for the cron's log.
+ *
+ * @param {object} db
+ * @param {{now: number, ids?: string[]|null, limit?: number, decryptSecret: Function, fetch?: typeof fetch, userAgent?: string}} options
+ */
+export async function deliverDue(db, { now, ids = null, limit = 50, decryptSecret, fetch: fetchImpl = fetch, userAgent }) {
+    const rows = ids
+        ? ids.length
+            ? (
+                  await db
+                      .prepare(`SELECT * FROM outbox WHERE status IN ('pending', 'held') AND id IN (${ids.map(() => '?').join(',')})`)
+                      .bind(...ids)
+                      .all()
+              ).results
+            : []
+        : (await db.prepare("SELECT * FROM outbox WHERE status IN ('pending', 'held') AND next_at <= ? ORDER BY next_at LIMIT ?").bind(now, limit).all()).results
+    const endpoints = new Map()
+    const counts = { due: rows.length, sent: 0, held: 0, failed: 0, retrying: 0 }
+    for (const row of rows) {
+        if (!endpoints.has(row.target)) endpoints.set(row.target, await db.prepare('SELECT * FROM webhook_endpoints WHERE id = ? AND account_id = ?').bind(row.target, row.account_id).first())
+        const endpoint = endpoints.get(row.target)
+        if (!endpoint || !endpoint.is_active) {
+            await db
+                .prepare("UPDATE outbox SET status = 'failed', last_error = ? WHERE id = ?")
+                .bind(endpoint ? 'Endpoint desligado.' : 'Endpoint removido.', row.id)
+                .run()
+            counts.failed++
+            continue
+        }
+        const result = await sendSigned(endpoint, { eventId: row.event_id, body: row.body }, { decryptSecret, fetch: fetchImpl, userAgent })
+        // A held row probing again does not spend the ordinary retries.
+        const attempts = row.status === 'held' ? row.attempts : row.attempts + 1
+        const next = classifyAttempt(result, Math.max(attempts, 1))
+        if (next.status === 'sent') {
+            await db.prepare("UPDATE outbox SET status = 'sent', attempts = ?, last_status = ?, last_error = NULL, sent_at = ? WHERE id = ?").bind(row.attempts + 1, result.status, now, row.id).run()
+            counts.sent++
+        } else if (next.status === 'failed') {
+            await db.prepare("UPDATE outbox SET status = 'failed', attempts = ?, last_status = ?, last_error = ? WHERE id = ?").bind(row.attempts + 1, result.status, result.error, row.id).run()
+            counts.failed++
+        } else {
+            await db
+                .prepare('UPDATE outbox SET status = ?, attempts = ?, last_status = ?, last_error = ?, next_at = ? WHERE id = ?')
+                .bind(next.status, attempts, result.status, result.error, now + next.nextIn, row.id)
+                .run()
+            if (next.status === 'held') counts.held++
+            else counts.retrying++
+        }
+        await db.prepare('UPDATE webhook_endpoints SET last_status = ?, last_at = ? WHERE id = ?').bind(result.status, now, endpoint.id).run()
+    }
+    return counts
+}
+
+/**
+ * Lets held deliveries of an endpoint go now — called when its secret or URL
+ * changes here, so the fix is felt at once instead of at the next probe.
+ */
+export async function releaseHeld(db, { accountId, endpointId, now }) {
+    const result = await db.prepare("UPDATE outbox SET status = 'pending', next_at = ? WHERE account_id = ? AND target = ? AND status = 'held'").bind(now, accountId, endpointId).run()
+    return result?.meta?.changes ?? 0
+}
+
+/** Puts one failed or held delivery back in line, for the "Reenviar" button. */
+export async function retryDelivery(db, { accountId, id, now }) {
+    const result = await db.prepare("UPDATE outbox SET status = 'pending', attempts = 0, next_at = ? WHERE id = ? AND account_id = ? AND status IN ('failed', 'held')").bind(now, id, accountId).run()
+    return (result?.meta?.changes ?? 0) > 0
+}