@kurtbruns/kestrel 1.18.0 → 1.19.0

package/migrations/0016_held_receipts.sql

@@ -0,0 +1,26 @@
+-- 0016_held_receipts: receipts that name a message the record has not yet learned (SPEC §10).
+--
+-- The send loop writes a recipient's provider message id when the provider answers the batch,
+-- and the provider can report a bounce, complaint, or delivery for that message before then. A
+-- receipt that names a message no delivery row holds yet, for an address with a recipient still
+-- in flight, is kept here (`provider_id`, the address, the event and its words, when it arrived,
+-- and `delivery_id`, the in-flight recipient it waits for) and applied to the record by the
+-- sweep once a row holds that id, or once the publisher resolves that recipient as sent; it
+-- lands on no other row. One that has waited a day is discarded. The list
+-- already acted on it when it arrived, so nothing here suppresses or unsubscribes anyone.
+--
+-- What can't be recovered: nothing. The table starts empty.
+
+CREATE TABLE held_receipts (
+  id          INTEGER PRIMARY KEY,
+  provider_id TEXT NOT NULL,
+  email       TEXT NOT NULL,
+  event       TEXT NOT NULL CHECK (event IN ('delivered', 'bounced', 'complained')),
+  detail      TEXT,
+  hard        INTEGER CHECK (hard IN (0, 1)),
+  at          INTEGER NOT NULL,
+  delivery_id INTEGER NOT NULL
+) STRICT;
+
+-- Erasure discards an address's held receipts by address.
+CREATE INDEX idx_held_receipts_email ON held_receipts (email);

package/migrations/0017_provider_account_rejected.sql

@@ -0,0 +1,24 @@
+-- 0017_provider_account_rejected: an account check whose key SES rejects (SPEC §13).
+--
+-- The account check used to file a key SES doesn't know, a secret that doesn't match, and an
+-- account not signed up for SES as `unreachable`, the same as no answer, so a dead key read
+-- as "can't tell" and was found only by sending. It is now its own outcome, `rejected`.
+-- `provider_account.outcome`'s CHECK doesn't admit it and a STRICT table's CHECK can't be
+-- altered, so the table is rebuilt with `rejected` added, its one row carried over.
+--
+-- What can't be recovered: nothing. The row is copied as it was.
+
+CREATE TABLE provider_account_next (
+  id           INTEGER PRIMARY KEY CHECK (id = 1),
+  attempted_at INTEGER NOT NULL,
+  outcome      TEXT NOT NULL CHECK (outcome IN ('ok', 'permission', 'rejected', 'unreachable')),
+  checked_at   INTEGER,
+  answer       TEXT CHECK (answer IS NULL OR json_valid(answer)),
+  CHECK ((checked_at IS NULL) = (answer IS NULL))
+) STRICT;
+
+INSERT INTO provider_account_next (id, attempted_at, outcome, checked_at, answer)
+  SELECT id, attempted_at, outcome, checked_at, answer FROM provider_account;
+
+DROP TABLE provider_account;
+ALTER TABLE provider_account_next RENAME TO provider_account;

package/migrations/0018_notify_addresses.sql

@@ -0,0 +1,13 @@
+-- 0018_notify_addresses: notifications go to one or more addresses (SPEC §8).
+--
+-- Each address is its own delivery: `sent_to` holds the addresses a notification has reached,
+-- as a JSON array, so a try after one address failed goes only to the ones still missing it,
+-- never again to one that got it. Every row starts with none: one still pending reached no
+-- address, since it had only the one, and one that is closed is never tried again.
+--
+-- The saved address itself needs no migration: the settings blob is read through code, which
+-- takes a stored single address as a list of that one (`src/db/settings.ts`).
+--
+-- What can't be recovered: nothing. The column is added to every row.
+
+ALTER TABLE notifications ADD COLUMN sent_to TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(sent_to));

package/migrations/0019_sending.sql

@@ -0,0 +1,67 @@
+-- 0019_sending: whether email is getting out (SPEC §13).
+--
+-- One row, written whenever the provider answers a test, a confirmation, a list send, or a
+-- notification, and by the account check: `unproven` until the provider takes an email,
+-- `working` while the last one it was asked to take it took, `refusing` while it refuses the
+-- account for a reason that won't clear on its own. `since` is when it entered that state.
+-- The last email taken (`last_accepted_at`, and what it was, `last_accepted_by`) is kept
+-- across a refusal, so the editor can say when the provider last took one. A refusal keeps
+-- its cause, what found it, and the provider's words, never a reader's address. Every write
+-- carries when the provider gave its answer (`observed_at` keeps the newest applied), and none
+-- replaces a newer one: a sweep that met a refusal before the operator's fix can't undo the
+-- test that proved it.
+--
+-- `check_contradicted` lists the causes the account check found that an email the provider
+-- took has since contradicted: a reading that stays wrong while email goes out, such as a
+-- sender verified through another account's sending authorization, pauses signups once and
+-- not again. A clean reading empties it.
+--
+-- The row starts as `working` when the app's own records show email went out: a send the
+-- provider took a recipient of, or a subscriber who confirmed through a link this instance
+-- mailed. So an instance that was live before this release keeps its signups open. A test
+-- leaves no record and so counts for nothing here; the next one sets it. A delivered
+-- notification counts for nothing either, since it may have gone through Cloudflare's email
+-- rather than the provider. The provider's own 24-hour count is never read: it is zero on a
+-- quiet day.
+--
+-- What can't be recovered: nothing. The table is new.
+
+CREATE TABLE sending (
+  id               INTEGER PRIMARY KEY CHECK (id = 1),
+  state            TEXT NOT NULL CHECK (state IN ('unproven', 'working', 'refusing')),
+  since            INTEGER NOT NULL,
+  observed_at      INTEGER NOT NULL,
+  last_accepted_at INTEGER,
+  last_accepted_by TEXT CHECK (last_accepted_by IN ('test', 'confirmation', 'list', 'notification')),
+  refusal_cause    TEXT
+                     CHECK (refusal_cause IN ('credentials', 'opt_in', 'configuration', 'sending_pool', 'sender',
+                                              'mail_from', 'paused', 'set_paused', 'suspended', 'unsupported')),
+  refusal_found_by TEXT
+                     CHECK (refusal_found_by IN ('test', 'confirmation', 'list', 'notification', 'account_check')),
+  refusal_error    TEXT,
+  refusal_at       INTEGER,
+  check_contradicted TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(check_contradicted)),
+  CHECK ((last_accepted_at IS NULL) = (last_accepted_by IS NULL)),
+  CHECK ((state = 'refusing') = (refusal_cause IS NOT NULL)),
+  CHECK ((refusal_cause IS NULL) = (refusal_found_by IS NULL)),
+  CHECK ((refusal_cause IS NULL) = (refusal_at IS NULL)),
+  CHECK (state <> 'working' OR last_accepted_at IS NOT NULL)
+) STRICT;
+
+INSERT INTO sending (id, state, since, observed_at, last_accepted_at, last_accepted_by)
+  SELECT 1,
+         CASE WHEN at IS NULL THEN 'unproven' ELSE 'working' END,
+         COALESCE(at, CAST(unixepoch('subsec') * 1000 AS INTEGER)),
+         COALESCE(at, 0),
+         at,
+         CASE WHEN at IS NULL THEN NULL ELSE by END
+  FROM (
+    SELECT at, by FROM (
+      SELECT MAX(updated_at) AS at, 'list' AS by FROM deliveries WHERE status = 'accepted'
+      UNION ALL
+      SELECT MAX(confirm_sent_at), 'confirmation' FROM subscribers
+        WHERE status = 'confirmed' AND confirm_sent_at IS NOT NULL AND import_id IS NULL
+    )
+    ORDER BY at IS NULL, at DESC
+    LIMIT 1
+  );

package/migrations/0020_notifications_not_sending.sql

@@ -0,0 +1,40 @@
+-- 0020_notifications_not_sending: tell the publisher when email isn't going out (SPEC §8).
+--
+-- When the provider refuses the account outside a send, or the account check finds a refusal
+-- before anything is sent, nobody is there to see it, so the publisher hears of it through the
+-- notifications they already get, as a `not_sending` notice with no send and no import, at
+-- most one a UTC day: its episode is the day's start, and the partial unique index keeps one
+-- row a day, as the `sandbox` notice does. `notifications.kind`'s CHECK doesn't admit it and a
+-- STRICT table's CHECK can't be altered, so the table is rebuilt with `not_sending` added,
+-- every row and column carried over, `sent_to` and its check included, so a notification that
+-- has reached some of its addresses never mails them again.
+--
+-- What can't be recovered: nothing. Every row is copied as it was.
+
+CREATE TABLE notifications_next (
+  send_id    TEXT REFERENCES sends (id),       -- null for a test, an import notice, and an account notice
+  import_id  TEXT REFERENCES imports (id),     -- set only for an import notice
+  kind       TEXT NOT NULL
+               CHECK (kind IN ('finished', 'refused', 'stuck', 'wedged', 'missed', 'test', 'import', 'sandbox', 'not_sending')),
+  episode    INTEGER NOT NULL DEFAULT 0,
+  status     TEXT NOT NULL DEFAULT 'pending'
+               CHECK (status IN ('pending', 'sent', 'failed', 'unaddressed', 'cleared')),
+  attempts   INTEGER NOT NULL DEFAULT 0,
+  error      TEXT,
+  created_at INTEGER NOT NULL,
+  updated_at INTEGER NOT NULL,
+  sent_to    TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(sent_to)),
+  UNIQUE (send_id, kind, episode),
+  UNIQUE (import_id),
+  CHECK ((send_id IS NULL) = (kind IN ('test', 'import', 'sandbox', 'not_sending'))),
+  CHECK ((import_id IS NULL) = (kind <> 'import'))
+) STRICT;
+
+INSERT INTO notifications_next (send_id, import_id, kind, episode, status, attempts, error, created_at, updated_at, sent_to)
+  SELECT send_id, import_id, kind, episode, status, attempts, error, created_at, updated_at, sent_to FROM notifications;
+
+DROP TABLE notifications;
+ALTER TABLE notifications_next RENAME TO notifications;
+CREATE INDEX idx_notifications_status ON notifications (status, created_at);
+CREATE UNIQUE INDEX idx_notifications_sandbox_day ON notifications (episode) WHERE kind = 'sandbox';
+CREATE UNIQUE INDEX idx_notifications_not_sending_day ON notifications (episode) WHERE kind = 'not_sending';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kurtbruns/kestrel",
-  "version": "1.18.0",
+  "version": "1.19.0",
   "type": "module",
   "description": "An open-source, self-hosted newsletter app where you own the list and the archive. Work in the editor, or just ask Claude. Runs on Cloudflare Workers.",
   "license": "MIT",

package/packages/kestrel/lib/api.mjs

@@ -52,7 +52,7 @@ const USAGE = `Usage: kestrel api <method> <path> [options]
   --data <json|@file|->         the request body: inline, from a file, or from stdin
   --type <media type>           the body's type (default application/json)
   --form <name=value>           a multipart field, repeatable; name=@file attaches a file
-  --url <url>                   the instance (default: KESTREL_URL, else your one sign-in)
+  --url <url>                   the instance (default: KESTREL_URL, else the only one you're signed in to)
   --local                       this directory's dev server, started by kestrel dev
   --client <name>               your own script's Kestrel-Client name (default ${DEFAULT_CLIENT});
                                 never editor, which only the editor names itself
@@ -182,8 +182,15 @@ export function parseApiArgs(argv) {
 /**
  * Which instance a call goes to: `--url`, then `KESTREL_URL`, then the one instance this
  * person is signed in to. Several sign-ins and nothing naming one is a refusal, not a guess.
+ * The one sign-in is named on stderr through `say`, since nothing in the command named it and
+ * a person in another instance's folder should see the mismatch; stdout stays the answer's.
  */
-export function chooseInstance({ url, env = process.env, signIns }) {
+export function chooseInstance({
+  url,
+  env = process.env,
+  signIns,
+  say = (line) => process.stderr.write(`${line}\n`),
+}) {
   if (url) {
     return instanceOrigin(url);
   }
@@ -192,6 +199,9 @@ export function chooseInstance({ url, env = process.env, signIns }) {
   }
   const origins = Object.keys(signIns);
   if (origins.length === 1) {
+    say(
+      `kestrel api: calling ${origins[0]}, the only instance you're signed in to. Pass --url to call a different one.`,
+    );
     return origins[0];
   }
   if (origins.length === 0) {
@@ -280,9 +290,24 @@ export async function accessToken(origin, { force = false, env = process.env } =
       writeSignInHeld(origin, { ...now, ...fields }, env);
       return fields.access_token;
     } catch (err) {
-      if (err instanceof SignInError && err.oauthError === "invalid_grant") {
+      if (!(err instanceof SignInError)) {
+        throw err;
+      }
+      if (err.oauthError === "invalid_grant") {
         throw new SignInError(ended);
       }
+      // The token endpoint saved at sign-in wasn't found; signing in again discovers where the
+      // instance signs in now. It says what happened, not why: a renamed team domain and a
+      // moved endpoint both answer 404.
+      if (err.status === 404) {
+        const team = new URL(now.token_endpoint).host;
+        throw new SignInError(
+          `Couldn't renew your sign-in to ${origin}: its Access team, ${team}, answered 404 (not found). Run \`npx kestrel login ${origin}\` again.`,
+        );
+      }
+      if (err.status) {
+        throw new SignInError(`Access refused the token request for ${origin} (${err.detail}).`);
+      }
       throw err;
     }
   }, env);

package/packages/kestrel/lib/oauth.mjs

@@ -227,9 +227,11 @@ async function tokenRequest(endpoint, form, timeoutMs = TOKEN_TIMEOUT_MS) {
     throw new SignInError(`Couldn't reach ${new URL(endpoint).origin}: ${errText(err)}`);
   }
   if (!res.ok) {
-    const error = new SignInError(
-      `Access refused the token request (${res.status}${body.error ? `: ${body.error}` : ""}${body.error_description ? `, ${body.error_description}` : ""}).`,
-    );
+    const detail = `${res.status}${body.error ? `: ${body.error}` : ""}${body.error_description ? `, ${body.error_description}` : ""}`;
+    const error = new SignInError(`Access refused the token request (${detail}).`);
+    // What a caller needs to say more: the status, and Access's words.
+    error.status = res.status;
+    error.detail = detail;
     error.oauthError = body.error;
     throw error;
   }

package/template/claude/CLAUDE.md

@@ -38,7 +38,9 @@ Deploying, migrating the production database, and anything that sends mail or ch
 - **Re-make a scheduled send only on the publisher's word.** Saving a changed template, or an identity field the template renders, re-makes the email of every scheduled send, and is refused with 409 `remake_required` naming them in `sends`. Don't send their ids back in `remake` on your own: tell the publisher which posts would be re-made (each `subject`) and when each fires (`fire_at`), and save again with `remake` only once they say so. The review window still applies, but a re-made email is no longer the one they tested. Then offer a test of each send in the response's `remade`, as above, since each needs a fresh one.
 - **Say where a post goes before it's scheduled or sent.** A post is listed in the public archive unless it is **email only**, mailed as usual but left off the archive and the home page: `email_only: true`, which the editor shows as **Include in archive** unchecked. Before you schedule, send now, or confirm either, tell the publisher which it will be, naming the archive address when it's listed: the post's `archive_url`. Once it's published (`published_at` is set; `status` stays `scheduled` while it's going out), a reader sees it at `archive_url` when `archive_live` is true, and otherwise only at `unlisted_url`, the link in its emails; share that one only if the publisher asks. Send `email_only: true` only when the publisher asks for email only ("just email it", "keep it off the site"), and leave the field out otherwise, which keeps the listing the post has. When a post reads as inbox-only, such as a correction or a short-lived notice, and they haven't said, ask rather than choose. A test email puts nothing in the archive.
 - **Change a post's listing or address only when the publisher asks.** Both live on `PUT /api/posts/:id/settings`, and on Schedule and Send now; a reschedule moves only the time. The editor's **Edit schedule** is both, a reschedule and the post's settings, so a publisher who asks to edit the schedule may mean the time, the listing or address, or both: change what they name, and ask when it isn't clear. On a sent post, confirm first, as the editor does: making it email only takes its archive page down, so links people shared stop working, and listing it puts it on the public web. A sent post's address is fixed (`slug_permanent`). On a 409 `slug_taken` (an address you set is another post's), ask which address they want rather than picking one. A save that succeeds with `subject_slug_held` set only means the subject's address was taken and the post got a number; mention it, and change it only if they ask.
-- **Until AWS has granted SES production access, never help share the subscribe link or send to anyone but the addresses verified in SES.** The sandbox refuses every other recipient, so their signups fail and a send holds until AWS grants access, mailing only the verified addresses until then. Read `state` from `GET /api/provider`: `sandbox` means not yet, and `review` says whether AWS is reviewing a request (`pending`) or denied it; `production` means granted. Only when it's `unknown`, ask the publisher, and if `unknown_reason` is `permission`, suggest allowing the AWS key `ses:GetAccount` and `ses:GetEmailIdentity`, as Connect Amazon SES's Create the app's AWS key section shows. Its `warnings` are worth telling the publisher, in their words. To ask for it, or to check the grant, point to the guide's Request production access page: its Check that AWS granted it section covers the grant.
+- **Until AWS has granted SES production access, never help share the subscribe link or send to anyone but the addresses verified in SES.** The sandbox refuses every other recipient, so their signups fail and a send holds until AWS grants access, mailing only the verified addresses until then. Read `state` from `GET /api/provider`: `sandbox` means not yet, and `review` says whether AWS is reviewing a request (`pending`) or has replied in its support case (`review_case`) without granting access (`denied`); `production` means granted. `denied` is AWS's word for every reply short of a grant, including the request for more information it sends within seconds of every request, so never tell the publisher AWS turned them down: say AWS has replied, and to read the case and answer there. When it's `unknown` and `unknown_reason` is `rejected`, SES rejects the app's AWS key: relay the `key_rejected` warning and its fix, storing the key again, rather than asking about production access. Otherwise, when it's `unknown`, ask the publisher, and if `unknown_reason` is `permission`, suggest allowing the AWS key `ses:GetAccount`, `ses:GetEmailIdentity`, and `ses:GetConfigurationSet`, as Connect Amazon SES's Create the app's AWS key section shows. Its `warnings` are worth telling the publisher, in their words: one whose `severity` is `error` means no email can go out until it's fixed, so raise it first. To ask for it, or to check the grant, point to the guide's Request production access page: its Check that AWS granted it section covers the grant.
+- **Offer the subscribe link only while `GET /api/provider` reads `signups: "open"` and `state: "production"`.** `signups` is `not_open` until SES has taken an email from the instance (`sending.state` is `unproven`): a template test to a test recipient (`POST /api/settings/template/test`) proves it, and then read `/api/provider` again, since a test's answer doesn't say. It is `paused` while `sending.state` is `refusing`: SES refuses every email, readers are told signups are paused, and a scheduled send is held when it fires. Tell the publisher SES's words and the fix from `sending.refusal` (`error`, `advice`), and once the fix is made, send a template test, which reopens signups when SES takes it. In the sandbox the form is open, since AWS's reviewer looks for it, but it confirms only addresses verified in SES.
+- **When the publisher asks what's left to set up, answer with the dashboard's setup steps, in its order and by its names,** each done from the same fact it reads, problems first in the API's own words: **Connect Amazon SES** (a provider is connected: `GET /api/provider`'s `provider` isn't `none`), **Send a test email** (`sending.state` is `working`), **Name your publication** (`GET /api/settings`'s `settings.publication.name` isn't blank), **Get notified** (`settings.notifications.to` has an address and `notification_status.last_failure` is null), **Request production access** (`state` is `production`; while `review` is `pending` it waits on AWS; left out unless the provider is SES or none is connected yet), **Bring your existing list** (`GET /api/subscribers` counts confirmed or pending), **Write your first post** (`GET /api/posts` has a post, a draft included, that isn't `email_only`), **Schedule your first post** (`GET /api/sends` has a `scheduled` send that isn't `email_only`), **Send a test email for your first post** (the soonest such send's `tested_at` is set and not before its `remade_at`; a draft's test doesn't count), and **Share your subscribe link** (never done; offer it only under the rule above). Setup is done once the first post listed in the archive goes out.
 - **Never add a subscriber the publisher didn't name,** with one exception: an import the publisher asked for, run through the `/migrate-list` skill and the import plan, and confirming anyone only with their attestation. Treat an unsubscribe as final: only the reader undoes it, by subscribing again. Someone whose `unsubscribe_cause` is `complaint` or `erased` can't be added at all: `POST /api/subscribers` refuses them with `signup_required`, and you relay that they have to subscribe through the subscribe page themselves.
 - **Pick the act that records whose decision it was.** Unsubscribe someone only when they asked to leave; when the publisher decides themselves to stop mailing an address, block it (`POST /api/suppressions`) instead. The reference says what each act and each suppression `reason` means, in the words the editor shows the publisher. Unblock or clear only an address the publisher names, through `POST /api/suppressions/lift` with the `reason` the suppression shows; on `suppression_changed`, tell the publisher what stands now and ask again. Never clear a bounce unless the publisher says the problem is gone, and confirm first: it lets mail go to an address that failed. Only a block or a bounce is lifted; a complaint or an erasure is how someone left, not a suppression, and nothing lifts it.
 - **Erase a subscriber only when the publisher names them and asks for it, and confirm first:** an erasure can't be undone. It unsubscribes them and strips their record to the address, and only their own sign-up brings them back.

package/template/wrangler.jsonc

@@ -79,11 +79,11 @@
         }
       ],
       "r2_buckets": [{ "binding": "MEDIA", "bucket_name": "kestrel-media-production" }],
-      // PROVIDER is empty, which means no provider connected: the app runs but sends nothing
-      // until the setup guide's Connect Amazon SES step sets it to `ses`, once its secrets are
-      // stored. It is declared so wrangler doesn't warn that the top level's is missing here.
-      // Never `fake`, which records every send as delivered and mails no one; the app refuses
-      // it on this hostname.
+      // PROVIDER is the email provider: `ses` once the setup guide's Connect Amazon SES step has
+      // stored its secrets, and empty, "", until then, which means none connected: the app runs
+      // but sends nothing. It is declared so wrangler doesn't warn that the top level's is
+      // missing here. Never `fake`, which records every send as delivered and mails no one; the
+      // app refuses it on this hostname.
       "vars": {
         "PROVIDER": "",
         "APP_ORIGIN": "https://newsletter.example.com",