@bugmole/cli 0.4.0

package/src/registry-worker/notifications/alerts.ts

@@ -0,0 +1,212 @@
+// Alert rules: which events go to which Slack channel or email address, and
+// the outbox handlers that deliver them.
+import type { Env } from "../core.js";
+import { emailSender, escapeHtml, type EmailMessage } from "../email/sender.js";
+import { getEntitlements } from "../billing/entitlements.js";
+import { onUsageAlert } from "../billing/thresholds.js";
+import { runChangedHooks } from "../hooks.js";
+import { enqueue, registerOutboxHandler } from "./outbox.js";
+import { openSecret } from "./secrets.js";
+import { postSlackMessage, slackText, SlackRevokedError, type SlackBlock } from "./slack.js";
+import { postTeamsMessage, teamsFetchEnv, TeamsRevokedError } from "./teams.js";
+import { molesAte, molesFound } from "../../runtime/mole-voice.js";
+
+export const ALERT_EVENTS = {
+  run_failed: "A run fails",
+  run_fixed: "A failing plan passes again",
+  usage_80: "80% of an allowance is used",
+  usage_100: "An allowance is used up",
+  spending_limit: "The spending limit is near or reached",
+} as const;
+export type AlertEvent = keyof typeof ALERT_EVENTS;
+export type AlertChannel = "slack" | "teams" | "email";
+/** Slack and Teams are paid-plan features; email works on every plan. */
+const PAID_CHANNELS = new Set<AlertChannel>(["slack", "teams"]);
+
+export function isAlertEvent(value: unknown): value is AlertEvent {
+  return typeof value === "string" && Object.hasOwn(ALERT_EVENTS, value);
+}
+
+type Rule = { id: string; channel: AlertChannel; target: string };
+export type Alert = { text: string; title: string; lines: string[]; link?: { label: string; url: string }; tone: "bad" | "good" | "info" };
+
+function dashboard(env: Env): string {
+  return (env.DASHBOARD_URL || "https://app.bugmole.com").replace(/\/$/, "");
+}
+
+async function matchingRules(env: Env, workspaceId: string, event: AlertEvent, projectId: string | null): Promise<Rule[]> {
+  const rows = await env.DB.prepare(`
+    SELECT r.id, r.channel, r.target FROM alert_rules r
+    LEFT JOIN slack_installations s ON r.channel = 'slack' AND s.id = r.target
+    LEFT JOIN teams_installations t ON r.channel = 'teams' AND t.id = r.target
+    WHERE r.workspace_id = ? AND r.event = ? AND r.enabled = 1
+      AND (r.project_id IS NULL OR r.project_id = ?)
+      AND (
+        r.channel = 'email'
+        OR (r.channel = 'slack' AND s.id IS NOT NULL AND s.revoked_at IS NULL)
+        OR (r.channel = 'teams' AND t.id IS NOT NULL AND t.revoked_at IS NULL)
+      )
+  `).bind(workspaceId, event, projectId).all<Rule>();
+  if (!rows.results.some((rule) => PAID_CHANNELS.has(rule.channel))) return rows.results;
+  const { features } = await getEntitlements(env, workspaceId);
+  return rows.results.filter((rule) => {
+    if (rule.channel === "slack") return features.slack;
+    if (rule.channel === "teams") return features.teams;
+    return true;
+  });
+}
+
+export function slackBlocks(alert: Alert): SlackBlock[] {
+  const icon = alert.tone === "bad" ? ":x:" : alert.tone === "good" ? ":white_check_mark:" : ":information_source:";
+  const blocks: SlackBlock[] = [
+    { type: "section", text: { type: "mrkdwn", text: `${icon} *${slackText(alert.title)}*` } },
+  ];
+  if (alert.lines.length) blocks.push({ type: "section", text: { type: "mrkdwn", text: alert.lines.map(slackText).join("\n") } });
+  if (alert.link) {
+    blocks.push({ type: "actions", elements: [{ type: "button", text: { type: "plain_text", text: alert.link.label }, url: alert.link.url }] });
+  }
+  return blocks;
+}
+
+/** Teams Workflows webhooks take plain text, not blocks — the same content as email's text body. */
+export function teamsText(alert: Alert): string {
+  const icon = alert.tone === "bad" ? "❌" : alert.tone === "good" ? "✅" : "ℹ️";
+  return [`${icon} ${alert.title}`, ...alert.lines, alert.link ? `${alert.link.label}: ${alert.link.url}` : ""].filter(Boolean).join("\n");
+}
+
+export function alertEmail(to: string, alert: Alert): EmailMessage {
+  return {
+    to: [{ email: to }],
+    subject: alert.title,
+    tag: "alert",
+    text: [...alert.lines, alert.link ? `${alert.link.label}: ${alert.link.url}` : ""].filter(Boolean).join("\n\n"),
+    html: `${alert.lines.map((line) => `<p>${escapeHtml(line)}</p>`).join("")}${alert.link ? `<p><a href="${escapeHtml(alert.link.url)}">${escapeHtml(alert.link.label)}</a></p>` : ""}`,
+  };
+}
+
+/** Queues an alert for every rule that wants it. */
+export async function dispatchAlert(env: Env, workspaceId: string, event: AlertEvent, projectId: string | null, alert: Alert, dedupe: string | null): Promise<number> {
+  let queued = 0;
+  for (const rule of await matchingRules(env, workspaceId, event, projectId)) {
+    const dedupeKey = dedupe ? `alert:${rule.id}:${dedupe}` : undefined;
+    const ok = rule.channel === "slack"
+      ? await enqueue(env, "slack.message", { installationId: rule.target, text: alert.text, blocks: slackBlocks(alert) }, { dedupeKey })
+      : rule.channel === "teams"
+        ? await enqueue(env, "teams.message", { installationId: rule.target, text: teamsText(alert) }, { dedupeKey })
+        : await enqueue(env, "email.message", alertEmail(rule.target, alert), { dedupeKey });
+    if (ok) queued += 1;
+  }
+  return queued;
+}
+
+// --- Events ----------------------------------------------------------------------------------
+
+const FAILED = new Set(["error", "partial"]);
+
+export async function runAlert(env: Env, runId: string): Promise<{ event: AlertEvent; alert: Alert; projectId: string; workspaceId: string } | null> {
+  const run = await env.DB.prepare(`
+    SELECT r.id, r.project_id AS projectId, r.workspace_id AS workspaceId, r.plan_id AS planId, r.status,
+           r.steps_passed AS passed, r.steps_failed AS failed, r.error_message AS error, r.failure_code AS failureCode,
+           r.completed_at AS completedAt, r.execution_mode AS mode, p.name AS projectName,
+           e.name AS environmentName, r.environment_base_url AS baseUrl
+    FROM runs r JOIN projects p ON p.id = r.project_id
+    LEFT JOIN project_environments e ON e.project_id = r.project_id AND e.id = r.environment_id
+    WHERE r.id = ?
+  `).bind(runId).first<Record<string, string | number | null>>();
+  if (!run?.workspaceId) return null;
+  const previous = await env.DB.prepare(`
+    SELECT status, steps_failed AS failed FROM runs
+    WHERE project_id = ? AND plan_id = ? AND id != ?
+      AND status IN ('success', 'partial', 'error') AND completed_at IS NOT NULL
+      AND (completed_at < ? OR (completed_at = ? AND rowid < (SELECT rowid FROM runs WHERE id = ?)))
+    ORDER BY completed_at DESC, rowid DESC LIMIT 1
+  `).bind(run.projectId, run.planId, runId, run.completedAt, run.completedAt, runId).first<{ status: string; failed: number | null }>();
+  const status = String(run.status);
+  const event: AlertEvent | null = FAILED.has(status)
+    ? "run_failed"
+    : status === "success" && previous && FAILED.has(previous.status) ? "run_fixed" : null;
+  if (!event) return null;
+  const where = run.environmentName ? ` on ${run.environmentName}` : run.baseUrl ? ` on ${String(run.baseUrl).replace(/^https?:\/\//, "")}` : "";
+  // A run that errored before any step failed (it couldn't start, say) found no bugs; say it failed.
+  const failed = Number(run.failed ?? 0);
+  const title = event === "run_failed"
+    ? failed > 0
+      ? `${molesFound(failed)} in ${run.planId}${where} — ${run.projectName}`
+      : `${run.planId} failed${where} — ${run.projectName}`
+    : `${molesAte(previous?.failed)} in ${run.planId}${where} — ${run.projectName}`;
+  const lines = event === "run_fixed"
+    ? [`${run.planId} passes again: ${run.passed ?? 0} steps passed.`]
+    : [`${run.passed ?? 0} steps passed, ${failed} failed.`];
+  if (event === "run_failed" && run.error) lines.push(String(run.error).slice(0, 300));
+  return {
+    event,
+    projectId: String(run.projectId),
+    workspaceId: String(run.workspaceId),
+    alert: {
+      title,
+      text: title,
+      lines,
+      tone: event === "run_failed" ? "bad" : "good",
+      link: { label: "Open the run", url: `${dashboard(env)}/runs?${new URLSearchParams({ project: String(run.projectId), run: runId })}` },
+    },
+  };
+}
+
+runChangedHooks.push(async (env, event) => {
+  if (!event.terminal || !event.workspaceId) return;
+  const found = await runAlert(env, event.runId);
+  if (found) await dispatchAlert(env, found.workspaceId, found.event, found.projectId, found.alert, `${event.runId}:${found.event}`);
+});
+
+onUsageAlert(async (env, workspaceId, usage) => {
+  const event: AlertEvent = usage.key.startsWith("spending_limit") ? "spending_limit" : usage.key.endsWith(":100") ? "usage_100" : "usage_80";
+  await dispatchAlert(env, workspaceId, event, null, {
+    title: usage.subject,
+    text: usage.subject,
+    lines: [usage.body],
+    tone: event === "usage_80" ? "info" : "bad",
+    link: { label: "Open billing", url: `${dashboard(env)}/billing?${new URLSearchParams({ workspace: workspaceId })}` },
+  }, null);
+});
+
+// --- Delivery --------------------------------------------------------------------------------
+
+registerOutboxHandler("slack.message", async (env, payload: { installationId: string; text: string; blocks?: SlackBlock[] }) => {
+  const install = await env.DB.prepare(`
+    SELECT channel_id AS channel, bot_token_ciphertext AS ciphertext, bot_token_iv AS iv, revoked_at AS revokedAt
+    FROM slack_installations WHERE id = ?
+  `).bind(payload.installationId).first<{ channel: string; ciphertext: string; iv: string; revokedAt: string | null }>();
+  if (!install || install.revokedAt) return; // disconnected since it was queued; nothing to send
+  const token = await openSecret(env, install.ciphertext, install.iv);
+  try {
+    await postSlackMessage(env, token, install.channel, { text: payload.text, blocks: payload.blocks });
+  } catch (error) {
+    if (error instanceof SlackRevokedError) {
+      await env.DB.prepare("UPDATE slack_installations SET revoked_at = ?, revoked_reason = ? WHERE id = ? AND revoked_at IS NULL")
+        .bind(new Date().toISOString(), error.code, payload.installationId).run();
+    }
+    throw error;
+  }
+});
+
+registerOutboxHandler("teams.message", async (env, payload: { installationId: string; text: string }) => {
+  const install = await env.DB.prepare(`
+    SELECT webhook_url_ciphertext AS ciphertext, webhook_url_iv AS iv, revoked_at AS revokedAt
+    FROM teams_installations WHERE id = ?
+  `).bind(payload.installationId).first<{ ciphertext: string; iv: string; revokedAt: string | null }>();
+  if (!install || install.revokedAt) return; // disconnected since it was queued; nothing to send
+  const webhookUrl = await openSecret(env, install.ciphertext, install.iv);
+  try {
+    await postTeamsMessage(teamsFetchEnv(env), webhookUrl, payload.text);
+  } catch (error) {
+    if (error instanceof TeamsRevokedError) {
+      await env.DB.prepare("UPDATE teams_installations SET revoked_at = ?, revoked_reason = 'webhook_gone' WHERE id = ? AND revoked_at IS NULL")
+        .bind(new Date().toISOString(), payload.installationId).run();
+    }
+    throw error;
+  }
+});
+
+registerOutboxHandler("email.message", async (env, payload: EmailMessage) => {
+  await emailSender(env).send(payload);
+});

package/src/registry-worker/notifications/notifications.test.ts

@@ -0,0 +1,298 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import type { Env } from "../core.js";
+import { call, createTestEnv, seedPlan, seedProject } from "../testing/harness.js";
+import { recordUsage } from "../billing/ledger.js";
+import { checkUsageThresholds } from "../billing/thresholds.js";
+import { backoffMs, drainOutbox, enqueue, MAX_ATTEMPTS, registerOutboxHandler, PermanentDeliveryError } from "./outbox.js";
+import { signState, verifyState } from "./secrets.js";
+
+function fakeSlack() {
+  const state = { posts: [] as Array<{ token: string; body: any }>, reply: { ok: true } as Record<string, unknown>, status: 200 };
+  const doFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+    const url = String(input);
+    if (url.endsWith("/oauth.v2.access")) {
+      const form = new URLSearchParams(String(init?.body));
+      if (form.get("code") !== "good-code") return Response.json({ ok: false, error: "invalid_code" });
+      return Response.json({ ok: true, access_token: "xoxb-secret-token", team: { id: "T1", name: "Acme" }, incoming_webhook: { channel: "#qa", channel_id: "C1" } });
+    }
+    if (url.endsWith("/chat.postMessage")) {
+      state.posts.push({ token: new Headers(init?.headers).get("authorization") ?? "", body: JSON.parse(String(init?.body)) });
+      return Response.json(state.reply, { status: state.status });
+    }
+    return new Response("not found", { status: 404 });
+  }) as unknown as typeof fetch;
+  return { state, doFetch };
+}
+
+function slackEnv(slack: ReturnType<typeof fakeSlack>, extra: Record<string, unknown> = {}) {
+  return createTestEnv({
+    FEATURE_SLACK: "on",
+    SLACK_CLIENT_ID: "client",
+    SLACK_CLIENT_SECRET: "secret",
+    SLACK_FETCH: slack.doFetch,
+    SECRETS_ENCRYPTION_KEY: "test-encryption-key",
+    DASHBOARD_URL: "https://dash.test",
+    ...extra,
+  } as Partial<Env>);
+}
+
+async function connected(slack: ReturnType<typeof fakeSlack>, plan = "team") {
+  const setup = slackEnv(slack);
+  const project = await seedProject(setup.env);
+  setup.db.db.prepare("INSERT INTO workspace_plans (workspace_id, plan) VALUES (?, ?)").run(project.workspaceId, plan);
+  const started = await call(setup.env, "POST", `/api/workspaces/${project.workspaceId}/integrations/slack/install`, { as: project.owner });
+  assert.equal(started.status, 200, JSON.stringify(started.json));
+  const authorize = new URL(started.json.authorizeUrl);
+  assert.equal(authorize.searchParams.get("redirect_uri"), "https://dash.test/api/integrations/slack/callback");
+  assert.equal(authorize.searchParams.get("scope"), "chat:write,incoming-webhook");
+  const state = authorize.searchParams.get("state")!;
+  const done = await call(setup.env, "POST", "/api/integrations/slack/complete", { as: project.owner, body: { code: "good-code", state } });
+  assert.equal(done.status, 201, JSON.stringify(done.json));
+  return { ...setup, ...project, installationId: done.json.installation.id as string, state };
+}
+
+const TEAMS_WEBHOOK = "https://acme.logic.azure.com/workflows/abc123/triggers/manual/paths/invoke";
+
+function fakeTeams() {
+  const state = { posts: [] as Array<{ text: string }>, status: 200 };
+  const doFetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
+    const body = JSON.parse(String(init?.body));
+    state.posts.push({ text: body.text });
+    return new Response(state.status === 200 ? "1" : "error", { status: state.status });
+  }) as unknown as typeof fetch;
+  return { state, doFetch };
+}
+
+function teamsEnv(teams: ReturnType<typeof fakeTeams>, extra: Record<string, unknown> = {}) {
+  return createTestEnv({
+    FEATURE_TEAMS: "on",
+    TEAMS_FETCH: teams.doFetch,
+    SECRETS_ENCRYPTION_KEY: "test-encryption-key",
+    DASHBOARD_URL: "https://dash.test",
+    ...extra,
+  } as Partial<Env>);
+}
+
+async function connectedTeams(teams: ReturnType<typeof fakeTeams>, plan = "team") {
+  const setup = teamsEnv(teams);
+  const project = await seedProject(setup.env);
+  setup.db.db.prepare("INSERT INTO workspace_plans (workspace_id, plan) VALUES (?, ?)").run(project.workspaceId, plan);
+  const connected = await call(setup.env, "POST", `/api/workspaces/${project.workspaceId}/integrations/teams`, { as: project.owner, body: { webhookUrl: TEAMS_WEBHOOK, name: "QA channel" } });
+  assert.equal(connected.status, 201, JSON.stringify(connected.json));
+  return { ...setup, ...project, installationId: connected.json.installation.id as string };
+}
+
+async function finishRun(env: Env, projectId: string, owner: string, key: string, status: string) {
+  const run = (await call(env, "POST", `/api/projects/${projectId}/runs`, { as: owner, body: { planId: "checkout", idempotencyKey: key } })).json.run.id;
+  await call(env, "POST", `/api/runs/${run}`, { as: owner, body: { status: "running" } });
+  await call(env, "POST", `/api/runs/${run}`, { as: owner, body: { status, stepsPassed: 3, stepsFailed: status === "success" ? 0 : 1, errorMessage: status === "success" ? undefined : "Expected <Payment> to be visible" } });
+  return run;
+}
+
+test("the outbox retries with backoff, dead-letters, and never delivers the same key twice", async () => {
+  const { env, db } = createTestEnv();
+  const seen: string[] = [];
+  let failures = 2;
+  registerOutboxHandler("test.flaky", async (_env, payload: { n: string }) => {
+    if (failures-- > 0) throw new Error("service down");
+    seen.push(payload.n);
+  });
+  registerOutboxHandler("test.broken", async () => {
+    throw new PermanentDeliveryError("bad request");
+  });
+  assert.equal(await enqueue(env, "test.flaky", { n: "a" }, { dedupeKey: "k1" }), true);
+  assert.equal(await enqueue(env, "test.flaky", { n: "a" }, { dedupeKey: "k1" }), false);
+  await enqueue(env, "test.broken", {});
+  await enqueue(env, "test.unknown", {});
+  let now = Date.now();
+  assert.deepEqual(await drainOutbox(env, new Date(now)), { delivered: 0, retried: 1, dead: 2 });
+  assert.deepEqual(await drainOutbox(env, new Date(now)), { delivered: 0, retried: 0, dead: 0 }, "waits out the backoff");
+  now += backoffMs(1) + 1;
+  await drainOutbox(env, new Date(now));
+  now += backoffMs(2) + 1;
+  assert.equal((await drainOutbox(env, new Date(now))).delivered, 1);
+  assert.deepEqual(seen, ["a"]);
+  assert.equal(backoffMs(20), 60 * 60_000);
+
+  registerOutboxHandler("test.always", async () => {
+    throw new Error("still down");
+  });
+  await enqueue(env, "test.always", {});
+  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+    now += 2 * 60 * 60_000;
+    await drainOutbox(env, new Date(now));
+  }
+  const row = db.db.prepare("SELECT status, attempts FROM outbox WHERE kind = 'test.always'").get() as any;
+  assert.deepEqual({ ...row }, { status: "dead", attempts: MAX_ATTEMPTS });
+});
+
+test("OAuth state is signed, scoped to its purpose, and expires", async () => {
+  const { env } = createTestEnv({ SECRETS_ENCRYPTION_KEY: "k" } as Partial<Env>);
+  const state = await signState(env, "slack-install", { workspaceId: "acme" });
+  assert.equal((await verifyState(env, "slack-install", state))?.workspaceId, "acme");
+  assert.equal(await verifyState(env, "github-install", state), null);
+  assert.equal(await verifyState(env, "slack-install", `${state.split(".")[0]}.00`), null);
+  const expired = await signState(env, "slack-install", {}, -1);
+  assert.equal(await verifyState(env, "slack-install", expired), null);
+});
+
+test("connecting Slack stores the token encrypted, posts a welcome, and sets default alerts", async () => {
+  const slack = fakeSlack();
+  const { env, db, workspaceId, owner, installationId, state } = await connected(slack);
+  const stored = db.db.prepare("SELECT bot_token_ciphertext AS c, channel_name AS channel FROM slack_installations WHERE id = ?").get(installationId) as any;
+  assert.equal(stored.channel, "#qa");
+  assert.ok(!String(stored.c).includes("xoxb"), "not stored in plain text");
+  assert.equal(slack.state.posts.length, 1, "welcome message sent right away");
+  assert.equal(slack.state.posts[0].token, "Bearer xoxb-secret-token");
+  assert.equal(slack.state.posts[0].body.channel, "C1");
+
+  const view = await call(env, "GET", `/api/workspaces/${workspaceId}/notifications`, { as: owner });
+  assert.equal(view.json.slack.available, true);
+  assert.deepEqual(view.json.rules.map((rule: { event: string }) => rule.event).sort(), ["run_failed", "run_fixed"]);
+  assert.ok(!JSON.stringify(view.json).includes("xoxb"), "the token never leaves the registry");
+
+  // The state is bound to the person who started the install.
+  assert.equal((await call(env, "POST", "/api/integrations/slack/complete", { as: "user_other", body: { code: "good-code", state } })).status, 400);
+  assert.equal((await call(env, "POST", "/api/integrations/slack/complete", { as: owner, body: { code: "bad-code", state } })).status, 502);
+});
+
+test("Slack is a paid feature, and only owners and admins manage it", async () => {
+  const slack = fakeSlack();
+  const { env } = slackEnv(slack);
+  const project = await seedProject(env);
+  const free = await call(env, "POST", `/api/workspaces/${project.workspaceId}/integrations/slack/install`, { as: project.owner });
+  assert.equal(free.status, 402);
+  assert.match(free.json.error, /Team and Business/);
+  assert.equal((await call(env, "POST", `/api/workspaces/${project.workspaceId}/integrations/slack/install`, { as: "user_stranger" })).status, 403);
+
+  const unconfigured = slackEnv(slack, { SLACK_CLIENT_SECRET: "" });
+  const other = await seedProject(unconfigured.env);
+  const view = await call(unconfigured.env, "GET", `/api/workspaces/${other.workspaceId}/notifications`, { as: other.owner });
+  assert.equal(view.json.slack.available, false);
+});
+
+test("a failed run and its fix reach Slack once each", async () => {
+  const slack = fakeSlack();
+  const { env, projectId, owner } = await connected(slack);
+  await seedPlan(env, projectId, owner);
+  slack.state.posts.length = 0;
+
+  const failed = await finishRun(env, projectId, owner, "r1", "error");
+  await drainOutbox(env);
+  assert.equal(slack.state.posts.length, 1);
+  const message = slack.state.posts[0].body;
+  assert.match(message.text, /Our moles found 1 bug in checkout/);
+  assert.ok(JSON.stringify(message.blocks).includes("Expected &lt;Payment&gt; to be visible"), "text is escaped");
+  assert.ok(JSON.stringify(message.blocks).includes(`run=${failed}`));
+
+  // A late duplicate update doesn't send again.
+  await call(env, "POST", `/api/runs/${failed}`, { as: owner, body: { status: "error" } });
+  await finishRun(env, projectId, owner, "r2", "success");
+  await finishRun(env, projectId, owner, "r3", "success");
+  await drainOutbox(env);
+  assert.equal(slack.state.posts.length, 2, "one failure, one fix, nothing for an ordinary pass");
+  assert.match(slack.state.posts[1].body.text, /Our moles ate (1 bug|the bugs) in checkout/);
+});
+
+test("a revoked Slack token disconnects the channel instead of retrying forever", async () => {
+  const slack = fakeSlack();
+  const { env, db, installationId, workspaceId, owner } = await connected(slack);
+  slack.state.reply = { ok: false, error: "token_revoked" };
+  const sent = await call(env, "POST", `/api/workspaces/${workspaceId}/integrations/slack/${installationId}/test`, { as: owner });
+  assert.equal(sent.status, 202);
+  const row = db.db.prepare("SELECT revoked_reason AS reason FROM slack_installations WHERE id = ?").get(installationId) as any;
+  assert.equal(row.reason, "token_revoked");
+  const dead = db.db.prepare("SELECT COUNT(*) AS c FROM outbox WHERE status = 'dead'").get() as any;
+  assert.equal(dead.c, 1);
+  const view = await call(env, "GET", `/api/workspaces/${workspaceId}/notifications`, { as: owner });
+  assert.ok(view.json.slack.installations[0].revokedAt);
+});
+
+test("email alerts go only to workspace members; usage alerts follow the rules", async () => {
+  const slack = fakeSlack();
+  const { env, email, workspaceId, owner, installationId, projectId } = await connected(slack);
+  await call(env, "GET", "/api/workspaces", { as: owner, headers: { "x-aegis-email": "owner@acme.test", "x-aegis-email-verified": "true" } });
+  const rule = (body: Record<string, unknown>) => call(env, "POST", `/api/workspaces/${workspaceId}/alert-rules`, { as: owner, body });
+  assert.equal((await rule({ event: "usage_80", channel: "email", target: "someone@elsewhere.test" })).status, 422);
+  assert.equal((await rule({ event: "usage_80", channel: "email", target: "Owner@Acme.test" })).status, 201);
+  assert.equal((await rule({ event: "usage_80", channel: "email", target: "owner@acme.test" })).status, 409);
+  assert.equal((await rule({ event: "usage_80", channel: "slack", target: installationId })).status, 201);
+  assert.equal((await rule({ event: "nope", channel: "slack", target: installationId })).status, 422);
+  assert.equal((await rule({ event: "run_failed", channel: "slack", target: "slk_other" })).status, 422);
+  assert.equal((await rule({ event: "run_failed", channel: "slack", target: installationId, projectId: "not-mine" })).status, 422);
+  const scoped = await rule({ event: "run_failed", channel: "email", target: "owner@acme.test", projectId });
+  assert.equal(scoped.status, 201);
+  assert.equal((await call(env, "PATCH", `/api/workspaces/${workspaceId}/alert-rules/${scoped.json.rule.id}`, { as: owner, body: { enabled: false } })).status, 200);
+
+  slack.state.posts.length = 0;
+  const before = email.sent.length;
+  await recordUsage(env, { workspaceId, kind: "browser_minute", quantity: 2_500, source: "cloud_run", idempotencyKey: "u" });
+  await checkUsageThresholds(env, workspaceId);
+  await drainOutbox(env);
+  assert.equal(slack.state.posts.length, 1);
+  assert.match(slack.state.posts[0].body.text, /80% of this month's cloud minutes/);
+  // The billing email itself, plus the rule's copy.
+  assert.equal(email.sent.length - before, 2);
+
+  assert.equal((await call(env, "DELETE", `/api/workspaces/${workspaceId}/integrations/slack/${installationId}`, { as: owner })).status, 200);
+  const view = await call(env, "GET", `/api/workspaces/${workspaceId}/notifications`, { as: owner });
+  assert.ok(view.json.rules.every((item: { channel: string }) => item.channel === "email"), "Slack rules go with the connection");
+});
+
+test("connecting Teams verifies the webhook, stores it encrypted, and sets default alerts", async () => {
+  const teams = fakeTeams();
+  const { env, db, workspaceId, owner, installationId } = await connectedTeams(teams);
+  assert.equal(teams.state.posts.length, 1, "a verification message is sent right away");
+  assert.match(teams.state.posts[0].text, /Bugmole is connected/);
+  const stored = db.db.prepare("SELECT webhook_url_ciphertext AS c FROM teams_installations WHERE id = ?").get(installationId) as any;
+  assert.ok(!String(stored.c).includes("logic.azure.com"), "not stored in plain text");
+
+  const view = await call(env, "GET", `/api/workspaces/${workspaceId}/notifications`, { as: owner });
+  assert.equal(view.json.teams.available, true);
+  assert.deepEqual(view.json.rules.filter((r: { channel: string }) => r.channel === "teams").map((r: { event: string }) => r.event).sort(), ["run_failed", "run_fixed"]);
+  assert.ok(!JSON.stringify(view.json).includes("logic.azure.com"), "the webhook URL never leaves the registry");
+
+  assert.equal((await call(env, "POST", `/api/workspaces/${workspaceId}/integrations/teams`, { as: owner, body: { webhookUrl: "https://example.com/not-a-workflow" } })).status, 422);
+});
+
+test("Teams is a paid feature, and only owners and admins manage it", async () => {
+  const teams = fakeTeams();
+  const { env } = teamsEnv(teams);
+  const project = await seedProject(env);
+  const free = await call(env, "POST", `/api/workspaces/${project.workspaceId}/integrations/teams`, { as: project.owner, body: { webhookUrl: TEAMS_WEBHOOK } });
+  assert.equal(free.status, 402);
+  assert.match(free.json.error, /Team and Business/);
+  assert.equal((await call(env, "POST", `/api/workspaces/${project.workspaceId}/integrations/teams`, { as: "user_stranger", body: { webhookUrl: TEAMS_WEBHOOK } })).status, 403);
+});
+
+test("a failed run reaches both Slack and Teams; a revoked Teams webhook disconnects instead of retrying forever", async () => {
+  const slack = fakeSlack();
+  const teams = fakeTeams();
+  const setup = slackEnv(slack, { FEATURE_TEAMS: "on", TEAMS_FETCH: teams.doFetch });
+  const project = await seedProject(setup.env);
+  setup.db.db.prepare("INSERT INTO workspace_plans (workspace_id, plan) VALUES (?, ?)").run(project.workspaceId, "team");
+  await seedPlan(setup.env, project.projectId, project.owner);
+  const slackInstall = await call(setup.env, "POST", `/api/workspaces/${project.workspaceId}/integrations/slack/install`, { as: project.owner });
+  const authorize = new URL(slackInstall.json.authorizeUrl);
+  await call(setup.env, "POST", "/api/integrations/slack/complete", { as: project.owner, body: { code: "good-code", state: authorize.searchParams.get("state") } });
+  const teamsInstall = await call(setup.env, "POST", `/api/workspaces/${project.workspaceId}/integrations/teams`, { as: project.owner, body: { webhookUrl: TEAMS_WEBHOOK } });
+  const teamsInstallationId = teamsInstall.json.installation.id as string;
+  slack.state.posts.length = 0;
+  teams.state.posts.length = 0;
+
+  await finishRun(setup.env, project.projectId, project.owner, "r1", "error");
+  await drainOutbox(setup.env);
+  assert.equal(slack.state.posts.length, 1);
+  assert.equal(teams.state.posts.length, 1);
+  assert.match(teams.state.posts[0].text, /Our moles found 1 bug in checkout/);
+
+  teams.state.status = 404;
+  await call(setup.env, "POST", `/api/workspaces/${project.workspaceId}/integrations/teams/${teamsInstallationId}/test`, { as: project.owner });
+  await drainOutbox(setup.env);
+  const row = setup.db.db.prepare("SELECT revoked_reason AS reason FROM teams_installations WHERE id = ?").get(teamsInstallationId) as any;
+  assert.equal(row.reason, "webhook_gone");
+
+  // Already revoked by the failed delivery above, so there's nothing left to disconnect.
+  assert.equal((await call(setup.env, "DELETE", `/api/workspaces/${project.workspaceId}/integrations/teams/${teamsInstallationId}`, { as: project.owner })).status, 404);
+});

package/src/registry-worker/notifications/outbox.ts

@@ -0,0 +1,83 @@
+// A durable queue for calls to other services. A request (or a hook) adds a
+// row; the minute job delivers it, retrying with backoff, and gives up after
+// MAX_ATTEMPTS so one broken destination can't clog the queue.
+import type { Env } from "../core.js";
+import { newId } from "../core.js";
+import { registerJob } from "../jobs/index.js";
+
+export const MAX_ATTEMPTS = 8;
+const BATCH = 50;
+const LOCK_MS = 2 * 60_000;
+
+/** A handler decides whether a failure is worth retrying. */
+export class PermanentDeliveryError extends Error {}
+
+type Handler = (env: Env, payload: any) => Promise<void>;
+const handlers = new Map<string, Handler>();
+
+export function registerOutboxHandler(kind: string, handler: Handler): void {
+  handlers.set(kind, handler);
+}
+
+/** Queues work. A repeated `dedupeKey` is ignored, so hooks that fire twice send once. */
+export async function enqueue(env: Env, kind: string, payload: unknown, options: { dedupeKey?: string; notBefore?: Date } = {}): Promise<boolean> {
+  const result = await env.DB.prepare(`
+    INSERT INTO outbox (id, kind, payload_json, dedupe_key, next_attempt_at) VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(dedupe_key) DO NOTHING
+  `).bind(newId("out"), kind, JSON.stringify(payload), options.dedupeKey ?? null, (options.notBefore ?? new Date()).toISOString()).run();
+  return (result.meta?.changes ?? 0) > 0;
+}
+
+export function backoffMs(attempts: number): number {
+  return Math.min(30_000 * 2 ** Math.max(0, attempts - 1), 60 * 60_000);
+}
+
+export type DrainResult = { delivered: number; retried: number; dead: number };
+
+export async function drainOutbox(env: Env, now = new Date(), limit = BATCH): Promise<DrainResult> {
+  const result: DrainResult = { delivered: 0, retried: 0, dead: 0 };
+  const nowIso = now.toISOString();
+  const due = await env.DB.prepare(`
+    SELECT id, kind, payload_json AS payload, attempts FROM outbox
+    WHERE status = 'pending' AND next_attempt_at <= ? AND (locked_until IS NULL OR locked_until <= ?)
+    ORDER BY next_attempt_at ASC LIMIT ?
+  `).bind(nowIso, nowIso, limit).all<{ id: string; kind: string; payload: string; attempts: number }>();
+  for (const item of due.results) {
+    // Claim it so an overlapping run doesn't deliver it twice.
+    const claimed = await env.DB.prepare(`
+      UPDATE outbox SET locked_until = ?, attempts = attempts + 1
+      WHERE id = ? AND status = 'pending' AND (locked_until IS NULL OR locked_until <= ?)
+    `).bind(new Date(now.getTime() + LOCK_MS).toISOString(), item.id, nowIso).run();
+    if ((claimed.meta?.changes ?? 0) !== 1) continue;
+    const attempts = item.attempts + 1;
+    const handler = handlers.get(item.kind);
+    try {
+      if (!handler) throw new PermanentDeliveryError(`No handler for ${item.kind}`);
+      await handler(env, JSON.parse(item.payload));
+      await env.DB.prepare("UPDATE outbox SET status = 'done', done_at = ?, locked_until = NULL, last_error = NULL WHERE id = ?")
+        .bind(nowIso, item.id).run();
+      result.delivered += 1;
+    } catch (error) {
+      const message = (error instanceof Error ? error.message : String(error)).slice(0, 500);
+      if (error instanceof PermanentDeliveryError || attempts >= MAX_ATTEMPTS) {
+        await env.DB.prepare("UPDATE outbox SET status = 'dead', locked_until = NULL, last_error = ? WHERE id = ?").bind(message, item.id).run();
+        console.error(`[outbox] giving up on ${item.kind} ${item.id}: ${message}`);
+        result.dead += 1;
+      } else {
+        await env.DB.prepare("UPDATE outbox SET locked_until = NULL, last_error = ?, next_attempt_at = ? WHERE id = ?")
+          .bind(message, new Date(now.getTime() + backoffMs(attempts)).toISOString(), item.id).run();
+        result.retried += 1;
+      }
+    }
+  }
+  return result;
+}
+
+registerJob("minute", "outbox", async (env, _ctx, now) => {
+  await drainOutbox(env, now);
+});
+
+/** Delivers queued work right after the request that queued it, instead of waiting for the next minute. */
+export function drainSoon(env: Env, ctx: ExecutionContext): void {
+  ctx.waitUntil(drainOutbox(env).then(() => undefined));
+}