@catalyst-cloud/schema 0.1.3 → 0.1.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catalyst-cloud/schema",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "type": "module",
   "description": "Typed Drizzle schema = single source of truth for the per-tenant Mirror DO SQLite store (CTC-13 / ADR-0002). Shared by the mirror Worker, the host-sync replica, and the browser OPFS replica.",
   "license": "MIT",

package/src/index.ts

@@ -27,7 +27,10 @@ import {
   reviews,
   processed_events,
   change_log,
+  event_log,
   sync_cursors,
+  fleet_activity,
+  pull_sweeps,
 } from "./mirror.js";
 
 export * from "./mirror.js";
@@ -61,13 +64,29 @@ export const mirrorSchema = {
   reviews,
   processed_events,
   change_log,
+  event_log,
   sync_cursors,
+  fleet_activity,
+  // The per-PR enrichment receipt. In `mirrorSchema` (it is a real table the DO creates and the
+  // migration bundle must produce) but DELIBERATELY ABSENT from `mirrorEntityTables` below — see the
+  // no-feed arithmetic on the table definition in ./mirror.ts.
+  pull_sweeps,
 } as const;
 
 /**
  * The domain-entity tables that flow over the change-feed (the keys are the wire `entity` names). The
- * infra tables — processed_events / change_log / sync_cursors — are deliberately excluded: they never
- * ride a delta.
+ * infra tables — processed_events / change_log / event_log / sync_cursors — are deliberately excluded:
+ * they never ride a delta.
+ *
+ * `event_log` in particular must NEVER appear here. It is the raw provider-event passthrough feed
+ * (CTC-294 / ADR-0017), served on its own route and its own cursor; leaking it into the entity feed
+ * would push an unknown `entity` at every already-deployed replica consumer, which is exactly the
+ * fleet-wide breakage this cutover exists to avoid.
+ *
+ * ⛔ `pull_sweeps` must never appear here either, for a different reason: its clock advances on every
+ * reconcile pass, so as an entity it would emit ~60k deltas/day (7 repos × 30 open PRs × 288 passes)
+ * into a 200k/7-day ring — the ring would hold under a day and every replica cursor would fall off
+ * the end. The arithmetic and the rest of the posture live on the table definition in ./mirror.ts.
  */
 export const mirrorEntityTables = {
   issues,
@@ -85,6 +104,9 @@ export const mirrorEntityTables = {
   check_runs,
   commit_statuses,
   reviews,
+  // CTC-350: fleet coordination current-state — replicates like any other entity (one row per
+  // host_id/ticket), unlike the infra tables above which never ride a delta.
+  fleet_activity,
 } as const;
 
 /** The wire `entity` name of a change-feed-carried table (keys of mirrorEntityTables). */

package/src/migrate.ts

@@ -118,8 +118,11 @@ export function applyMigrations(
  * non-replay-safe when the run-once ledger guard is bypassed — e.g. a crash between a statement
  * succeeding and the tag being recorded, which reruns the whole migration on the next boot:
  *   • "duplicate column name" on `ALTER TABLE ... ADD COLUMN` — the column is already present.
- *   • "index ... already exists" on `CREATE INDEX` — the index is already present (drizzle-kit emits
- *     bare `CREATE INDEX`, not `IF NOT EXISTS`, so the runner makes the replay idempotent here).
+ *   • "index ... already exists" on `CREATE INDEX` **or `CREATE UNIQUE INDEX`** — the index is already
+ *     present (drizzle-kit emits a bare form, not `IF NOT EXISTS`, so the runner makes the replay
+ *     idempotent here). The `UNIQUE` variant is NOT optional to handle: the pattern must match the
+ *     statement drizzle-kit actually emitted, and a unique index that fell through this guard would
+ *     throw on replay and leave the DO unable to boot (CTC-294 added the first such index).
  *   • "table ... already exists" on `CREATE TABLE` — a NEW entity table added in a LATER migration
  *     (CTC-USERS): drizzle-kit emits a bare `CREATE TABLE` (no IF NOT EXISTS), so a live DB whose
  *     tables were already created from a newer `schema.sql` (which carries the new table) would throw
@@ -135,7 +138,7 @@ function execReplaySafe(db: MigrationDb, stmt: string): void {
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
     if (/duplicate column name/i.test(msg) && /\bADD\b/i.test(stmt)) return;
-    if (/already exists/i.test(msg) && /\bCREATE\s+INDEX\b/i.test(stmt)) return;
+    if (/already exists/i.test(msg) && /\bCREATE\s+(UNIQUE\s+)?INDEX\b/i.test(stmt)) return;
     if (/already exists/i.test(msg) && /\bCREATE\s+TABLE\b/i.test(stmt)) return;
     throw err;
   }

package/src/migrations.generated.ts

@@ -74,6 +74,55 @@ export const MIRROR_MIGRATIONS = {
         tag: "0008_optimal_rattler",
         breakpoints: true,
       },
+      {
+        idx: 9,
+        version: "6",
+        when: 1784832568791,
+        tag: "0009_deep_bill_hollister",
+        breakpoints: true,
+      },
+      {
+        idx: 10,
+        version: "6",
+        when: 1785084189458,
+        tag: "0010_red_abomination",
+        breakpoints: true,
+      },
+      {
+        idx: 11,
+        version: "6",
+        when: 1785090228974,
+        tag: "0011_hesitant_korath",
+        breakpoints: true,
+      },
+      {
+        idx: 12,
+        version: "6",
+        when: 1786097717165,
+        tag: "0012_ordinary_infant_terrible",
+        breakpoints: true,
+      },
+      {
+        idx: 13,
+        version: "6",
+        when: 1786133360284,
+        tag: "0013_fair_mach_iv",
+        breakpoints: true,
+      },
+      {
+        idx: 14,
+        version: "6",
+        when: 1786583196993,
+        tag: "0014_reviews_state_canonical",
+        breakpoints: true,
+      },
+      {
+        idx: 15,
+        version: "6",
+        when: 1786594623244,
+        tag: "0015_brainy_lady_ursula",
+        breakpoints: true,
+      },
     ],
   },
   migrations: {
@@ -95,5 +144,18 @@ export const MIRROR_MIGRATIONS = {
       "CREATE TABLE `issue_history` (\n\t`id` text PRIMARY KEY NOT NULL,\n\t`issue_id` text NOT NULL,\n\t`actor_id` text,\n\t`created_at` integer,\n\t`updated_at` integer,\n\t`from_state` text,\n\t`to_state` text,\n\t`from_assignee_id` text,\n\t`to_assignee_id` text,\n\t`from_priority` integer,\n\t`to_priority` integer,\n\t`from_estimate` real,\n\t`to_estimate` real,\n\t`from_title` text,\n\t`to_title` text,\n\t`from_cycle_id` text,\n\t`to_cycle_id` text,\n\t`from_project_id` text,\n\t`to_project_id` text,\n\t`from_parent_id` text,\n\t`to_parent_id` text,\n\t`from_team_id` text,\n\t`to_team_id` text,\n\t`from_due_date` text,\n\t`to_due_date` text,\n\t`added_label_ids` text,\n\t`removed_label_ids` text,\n\t`updated_description` integer,\n\t`archived` integer,\n\t`auto_archived` integer,\n\t`auto_closed` integer,\n\t`trashed` integer,\n\t`raw` text\n);\n--> statement-breakpoint\nCREATE INDEX `idx_issue_history_issue_created` ON `issue_history` (`issue_id`,`created_at`);",
     "0008_optimal_rattler":
       "ALTER TABLE `issues` ADD `state_id` text;--> statement-breakpoint\nALTER TABLE `issues` ADD `team_key` text;--> statement-breakpoint\nALTER TABLE `issues` ADD `team_name` text;",
+    "0009_deep_bill_hollister":
+      "ALTER TABLE `pull_requests` ADD `head_ref` text;--> statement-breakpoint\nALTER TABLE `pull_requests` ADD `linear_issue_identifier` text;--> statement-breakpoint\nCREATE INDEX `idx_pulls_linear_ident` ON `pull_requests` (`linear_issue_identifier`);",
+    "0010_red_abomination":
+      "CREATE TABLE `event_log` (\n\t`seq` integer PRIMARY KEY AUTOINCREMENT NOT NULL,\n\t`delivery_id` text NOT NULL,\n\t`source` text NOT NULL,\n\t`event_type` text NOT NULL,\n\t`action` text,\n\t`received_at` integer NOT NULL,\n\t`backend_ts` integer,\n\t`payload` text NOT NULL,\n\t`payload_omitted_bytes` integer\n);\n--> statement-breakpoint\nCREATE UNIQUE INDEX `idx_event_log_delivery` ON `event_log` (`delivery_id`);",
+    "0011_hesitant_korath": "ALTER TABLE `event_log` ADD `identity` text;",
+    "0012_ordinary_infant_terrible":
+      "ALTER TABLE `cycles` ADD `team_id` text;--> statement-breakpoint\nALTER TABLE `cycles` ADD `team_key` text;--> statement-breakpoint\nALTER TABLE `cycles` ADD `team_name` text;",
+    "0013_fair_mach_iv":
+      "CREATE TABLE `fleet_activity` (\n\t`host_id` text NOT NULL,\n\t`ticket` text NOT NULL,\n\t`phase` text,\n\t`status` text,\n\t`disposition` text,\n\t`pr_number` integer,\n\t`pr_url` text,\n\t`orch_id` text,\n\t`last_event_name` text,\n\t`last_event_ts` integer,\n\t`updated_at` integer,\n\tPRIMARY KEY(`host_id`, `ticket`)\n);\n",
+    "0014_reviews_state_canonical":
+      "-- CTC-451 — canonicalise `reviews.state` to UPPERCASE.\n--\n-- The two ingestion legs disagreed at the source: REST (`/pulls/{n}/reviews`, the reconcile) returns\n-- `COMMENTED`, while the `pull_request_review` webhook delivers `commented`. Both legs stored the\n-- value verbatim, so the column carried BOTH casings of the same state. Measured on the live corpus\n-- before this migration: COMMENTED 3957 / commented 921 — a raw `WHERE state = 'COMMENTED'` missed\n-- 19% of reviews. Three read paths worked around it in application code, and none of those\n-- workarounds travel to the host replica, which is the fleet's only data path.\n--\n-- Both writers are fixed (apps/mirror/src/normalize/github.ts buildReviewRow), but that alone\n-- converges only rows the reconcile re-sweeps — and the reconcile walks OPEN PRs only, so every\n-- lowercase row on an already-closed PR would keep its casing forever. This backfill is what makes\n-- the corpus canonical rather than just the new writes.\n--\n-- ⭐ Deliberately a DATA migration in the SHARED bundle: the DO and every host replica apply the\n-- same MIRROR_MIGRATIONS through the same runner, so each side canonicalises its OWN rows locally.\n-- That converges the replica WITHOUT emitting ~921 change_log deltas, which would otherwise evict\n-- real deltas from the 200k/7d ring (the CTC-279 flood).\n--\n-- Idempotent by construction: upper() of an already-uppercase value is a no-op, and the WHERE clause\n-- makes the write a no-op too. Safe to re-run.\nUPDATE `reviews` SET `state` = upper(`state`) WHERE `state` IS NOT NULL AND `state` <> upper(`state`);\n",
+    "0015_brainy_lady_ursula":
+      "CREATE TABLE `pull_sweeps` (\n\t`repo_id` text NOT NULL,\n\t`pr_number` integer NOT NULL,\n\t`head_sha` text NOT NULL,\n\t`swept_at` integer NOT NULL,\n\t`check_run_count` integer,\n\t`checks_error` integer,\n\t`commit_status_count` integer,\n\t`statuses_error` integer,\n\t`review_count` integer,\n\t`reviews_error` integer,\n\tPRIMARY KEY(`repo_id`, `pr_number`)\n);\n",
   },
 } as const;

package/src/mirror.ts

@@ -16,7 +16,15 @@
 //   • `check_runs` join a PR by head_sha, NOT PR number.
 //   • PR mergeable / mergeable_state are nullable until a reconcile resolves them.
 
-import { sqliteTable, text, integer, real, primaryKey, index } from "drizzle-orm/sqlite-core";
+import {
+  sqliteTable,
+  text,
+  integer,
+  real,
+  primaryKey,
+  index,
+  uniqueIndex,
+} from "drizzle-orm/sqlite-core";
 
 // ── Linear ───────────────────────────────────────────────────────────────────────────────────────
 
@@ -217,6 +225,12 @@ export const cycles = sqliteTable("cycles", {
   scope: real("scope"), // last(scopeHistory) — current total scope points.
   completed_issue_count: integer("completed_issue_count"), // last(completedIssueCountHistory).
   issue_count: integer("issue_count"), // last(issueCountHistory).
+  team_id: text("team_id"),
+  // CTC-358: denormalized team key + name (off the cycle's `team` ref) so a cycle labels its team —
+  // several teams' "Cycle 7" rows are otherwise indistinguishable. Mirrors the issues precedent
+  // (issues.team_key/team_name). team_id is KEPT.
+  team_key: text("team_key"),
+  team_name: text("team_name"),
   starts_at: integer("starts_at"),
   ends_at: integer("ends_at"),
   updated_at: integer("updated_at"),
@@ -300,11 +314,20 @@ export const pull_requests = sqliteTable(
     milestone_title: text("milestone_title"),
     updated_at: integer("updated_at"),
     synced_at: integer("synced_at"),
+    // CTC-269 ticket↔PR link: the PR's source branch (now persisted — previously read only for
+    // head_sha and dropped at normalize) and the Linear ticket key parsed from the PR title/body
+    // (e.g. "CTC-268"). Both nullable (a pre-backfill row, or a PR whose text names no ticket). The
+    // identifier backs the reverse lookup that surfaces a ticket's PRs and a PR's ticket — existence
+    // is gated at READ time (a false-positive parse never surfaces because no issue matches it).
+    head_ref: text("head_ref"),
+    linear_issue_identifier: text("linear_issue_identifier"),
   },
   (t) => [
     primaryKey({ columns: [t.repo_id, t.number] }),
     // CTC-59: the pulls list — `ORDER BY updated_at DESC` (buildPullsView).
     index("idx_pulls_updated").on(t.updated_at),
+    // CTC-269: the ticket→PR reverse lookup — `WHERE linear_issue_identifier = ?` (buildIssueDetail).
+    index("idx_pulls_linear_ident").on(t.linear_issue_identifier),
   ],
 );
 
@@ -363,6 +386,38 @@ export const change_log = sqliteTable("change_log", {
   ts: integer("ts"),
 });
 
+/**
+ * CTC-294 / ADR-0017: the raw provider-webhook passthrough log — the event-delivery feed that
+ * replaces the per-host smee tunnels. DELIBERATELY NOT an entity table: it never rides the
+ * change-feed, never appears in /snapshot, and carries its OWN `seq` sequence (consumers of
+ * /events/stream track a cursor SEPARATE from the change_log cursor — crossing them is a bug).
+ *
+ * `payload` is the provider's JSON re-serialized VERBATIM — deliberately un-normalized, so a host
+ * can feed it through the same mapper it already applies to a direct webhook. `delivery_id` is the
+ * provider's own idempotency key (x-github-delivery / linear-delivery) and is UNIQUE, so a queue
+ * re-delivery is an INSERT OR IGNORE no-op rather than a duplicate event on the wire.
+ */
+export const event_log = sqliteTable(
+  "event_log",
+  {
+    seq: integer("seq").primaryKey({ autoIncrement: true }),
+    delivery_id: text("delivery_id").notNull(),
+    source: text("source").notNull(), // github | linear
+    event_type: text("event_type").notNull(), // x-github-event value | Linear payload.type
+    action: text("action"), // payload.action when the provider sends one
+    received_at: integer("received_at").notNull(), // webhook receipt ms (edge clock)
+    backend_ts: integer("backend_ts"), // provider event time when supplied (Linear webhookTimestamp)
+    payload: text("payload").notNull(), // RAW provider JSON, verbatim ("null" when elided)
+    // NULL = payload stored intact. Non-null = the payload exceeded the cap and was ELIDED, and this
+    // is its original byte size; the record still ships so the consumer sees an attributable gap.
+    payload_omitted_bytes: integer("payload_omitted_bytes"),
+    // CTC-296: JSON identity tuple extracted BEFORE an over-cap payload is discarded, so the host
+    // can scope its reconcile to one entity instead of sweeping the board. Null unless elided.
+    identity: text("identity"),
+  },
+  (t) => [uniqueIndex("idx_event_log_delivery").on(t.delivery_id)],
+);
+
 export const sync_cursors = sqliteTable(
   "sync_cursors",
   {
@@ -373,3 +428,97 @@ export const sync_cursors = sqliteTable(
   },
   (t) => [primaryKey({ columns: [t.source, t.entity] })],
 );
+
+/**
+ * CTC-350: bounded current-state projection of fleet coordination events — one row per
+ * (host_id, ticket). Rides change_log (NOT a new cursor); NOT an infra table (it replicates, so it is
+ * in mirrorEntityTables). Folded by reduceFleetActivity (apps/mirror/src/do/fleet-activity.ts);
+ * terminal rows pruned on ingest. `updated_at` = the event's last_event_ts, so upsertRow's
+ * last-write-wins guard (`excluded.updated_at > stored`) drops stale/replayed events by construction
+ * (out-of-order tolerance).
+ */
+export const fleet_activity = sqliteTable(
+  "fleet_activity",
+  {
+    host_id: text("host_id").notNull(),
+    ticket: text("ticket").notNull(),
+    phase: text("phase"),
+    status: text("status"),
+    disposition: text("disposition"), // non-null == terminal (drives ingest-time prune)
+    pr_number: integer("pr_number"),
+    pr_url: text("pr_url"),
+    orch_id: text("orch_id"),
+    last_event_name: text("last_event_name"),
+    last_event_ts: integer("last_event_ts"), // ms epoch — the reducer watermark
+    updated_at: integer("updated_at"), // ms epoch — mirrors last_event_ts; upsertRow guard column
+  },
+  (t) => [primaryKey({ columns: [t.host_id, t.ticket] })],
+);
+
+/**
+ * ⭐ THE PER-PR ENRICHMENT RECEIPT — the row that makes "this PR has no CI" distinguishable from
+ * "nobody ever swept this PR". CTC-436 follow-up.
+ *
+ * ⛔ THE DEFECT. The GitHub reconcile writes one `check_runs` row per run GitHub returns and writes
+ * NOTHING when it returns `[]`. So a PR with no CI configured and a PR the walk never reached produce
+ * the IDENTICAL observation — zero rows — and no column anywhere separates them. Measured: 4 of a
+ * repo's 20 newest open PRs had zero check rows and nothing could say which kind of zero they were.
+ * `synced_at` does not help: it lives on `pull_requests` and is stamped by step 1's PR sweep, which
+ * runs for EVERY PR whether or not the enrichment walk ever reached it.
+ *
+ * ⭐ HOW A READER DECODES IT — three states, and the third is the one that used to be unrepresentable:
+ *   • no row for (repo_id, pr_number)                → NEVER swept. Not "no CI".
+ *   • row.head_sha <> pull_requests.head_sha         → never swept AT THIS SHA; a push invalidated the
+ *                                                      receipt. (This is why a bare timestamp is a LIE:
+ *                                                      step 1 advances head_sha every pass, so a stale
+ *                                                      per-PR clock reads as "swept" at a SHA nobody
+ *                                                      ever looked at.)
+ *   • row.head_sha matches AND check_run_count = 0   → SWEPT, and GitHub genuinely reported zero.
+ *   • row.head_sha matches AND check_run_count IS NULL
+ *     AND checks_error = 422                         → the fetch FAILED. ⛔ NOT a zero.
+ *
+ * ⛔ A COUNT COLUMN IS NULLABLE ON PURPOSE AND NULL IS NEVER 0. After CTC-436 each enrichment stage
+ * CATCHES an object-scoped 4xx and continues with the empty array it was initialised to, and the
+ * resume cursor advances anyway. A receipt that recorded `0` there would be a FORGED PROOF — "swept,
+ * source reported zero", with a fresh clock, for a PR nobody successfully swept — which is strictly
+ * worse than the missing receipt, because a merge gate would read it as verified-clean. The `*_error`
+ * columns carry the HTTP status so the absence states its reason instead of just being absent.
+ *
+ * ⛔ THIS TABLE MUST NEVER RIDE THE CHANGE FEED — a load-bearing decision, with arithmetic. The sweep
+ * clock advances on EVERY pass, so feeding it would emit one delta per open PR per pass:
+ * ~7 repos × ~30 open PRs × ~288 passes/day ≈ 60k deltas/day against a 200k-row / 7-day change_log
+ * ring. The ring would hold under a day, every host replica's `/changes` cursor would fall off the
+ * end, and the fleet would resync-storm. It is therefore deliberately NOT in `mirrorEntityTables`
+ * (so it has no MIRROR_TABLE_META, no `EntityName`, no PK_COLUMNS entry), NOT in SNAPSHOT_TABLES,
+ * NOT in GITHUB_ENTITIES, and nothing ever calls `appendChange` for it. It is a HUB-READ-ONLY table,
+ * the same posture as `sync_cursors` / `event_log`. Do not "finish the job" by wiring it to the feed.
+ *
+ * No pruner: it is self-limiting at |pull_requests| — one row per PR, overwritten in place.
+ *
+ * ⚠️ NO `updated_at` COLUMN, DELIBERATELY. `upsertRow` applies its last-write-wins guard
+ * (`WHERE excluded.updated_at > stored.updated_at`) iff the row object carries an `updated_at` key.
+ * Here the advancing clock IS the payload, so a guard could only ever reject the write that proves
+ * freshness — and two sweeps landing in the same millisecond would silently drop the second, which is
+ * exactly the write that carries a new `head_sha`. The clock is named `swept_at` so no future edit
+ * re-acquires that guard by renaming a column.
+ */
+export const pull_sweeps = sqliteTable(
+  "pull_sweeps",
+  {
+    repo_id: text("repo_id").notNull(),
+    pr_number: integer("pr_number").notNull(),
+    /** The head SHA the enrichment walk actually swept — NOT re-read from pull_requests. */
+    head_sha: text("head_sha").notNull(),
+    /** When the walk reached this PR (ms). Advances only when a walk actually ran. */
+    swept_at: integer("swept_at").notNull(),
+    /** Runs GitHub RETURNED (not rows changed). NULL ⇒ the fetch did not succeed — never 0. */
+    check_run_count: integer("check_run_count"),
+    /** HTTP status when the check-runs fetch failed; NULL when it succeeded. */
+    checks_error: integer("checks_error"),
+    commit_status_count: integer("commit_status_count"),
+    statuses_error: integer("statuses_error"),
+    review_count: integer("review_count"),
+    reviews_error: integer("reviews_error"),
+  },
+  (t) => [primaryKey({ columns: [t.repo_id, t.pr_number] })],
+);