@catalyst-cloud/schema 0.1.16 → 0.1.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catalyst-cloud/schema",
-  "version": "0.1.16",
+  "version": "0.1.18",
   "type": "module",
   "description": "Typed Drizzle schema = single source of truth for the per-tenant Mirror DO SQLite store (CTC-13 / ADR-0002). Shared by the mirror Worker, the host-sync replica, and the browser OPFS replica.",
   "license": "MIT",

package/src/index.ts

@@ -45,9 +45,11 @@ import {
   pr_ancillary_backfill,
   push_subscriptions,
   pushes,
+  push_events,
   deployments,
   deployment_statuses,
   pr_review_threads,
+  check_suites,
 } from "./mirror.js";
 
 export * from "./mirror.js";
@@ -126,11 +128,17 @@ export const mirrorSchema = {
   push_subscriptions,
   // CTC-667: the current head of a pushed git ref — the orchestrator's rebase-detection signal.
   pushes,
+  // CTC-704: one row per push DELIVERY, append-only next to the per-ref row above — what lets the
+  // feed emit an edge per push instead of collapsing two pushes to one ref into one.
+  push_events,
   // CTC-667: a GitHub deployment, its status transitions, and a PR review thread's resolution state
   // — the remaining payload types host dispatch consumes (CTL-1929's last smee tunnel).
   deployments,
   deployment_statuses,
   pr_review_threads,
+  // CTC-667 item 4: the check SUITE's own rollup — the highest-volume signal in CTL's census and the
+  // one every phase agent's CI wait blocks on.
+  check_suites,
 } as const;
 
 /**
@@ -190,6 +198,10 @@ export const feedEntityTables = {
   // rebase-detection signal, and the first of the four payload types CTL's dispatch consumes that
   // the mirror did not carry (CTL-1929's last smee tunnel).
   pushes,
+  // CTC-704: the per-DELIVERY push row. ⛔ A feed entity precisely because `pushes` cannot be one
+  // faithfully — a producer emits one edge per row, so a per-ref row caps the feed at one edge per
+  // ref per tick (CTL-48: 101 of 138 unmatched smee events were pushes).
+  push_events,
   // CTC-667: a GitHub deployment and its status history — what `phase-monitor-deploy` and the deploy
   // state machine key on (`environment`, `state`, `target_url`, `environment_url`).
   deployments,
@@ -197,6 +209,9 @@ export const feedEntityTables = {
   // CTC-667: a PR review thread's RESOLUTION state — the merge gate AGENTS.md names, and the one
   // fact `pr_review_comments` cannot express (it has no resolution column).
   pr_review_threads,
+  // CTC-667 item 4: GitHub's OWN check-suite rollup. ⛔ The row is contractual; a derivation from
+  // `check_runs` is NOT — see the table's own header for the two reasons it cannot agree with GitHub.
+  check_suites,
 } as const;
 
 /**

package/src/migrations.generated.ts

@@ -200,6 +200,20 @@ export const MIRROR_MIGRATIONS = {
         tag: "0026_curvy_micromacro",
         breakpoints: true,
       },
+      {
+        idx: 27,
+        version: "6",
+        when: 1787034085255,
+        tag: "0027_closed_miek",
+        breakpoints: true,
+      },
+      {
+        idx: 28,
+        version: "6",
+        when: 1787038129996,
+        tag: "0028_burly_nemesis",
+        breakpoints: true,
+      },
     ],
   },
   migrations: {
@@ -255,5 +269,8 @@ export const MIRROR_MIGRATIONS = {
       "CREATE TABLE `pushes` (\n\t`repo_id` text NOT NULL,\n\t`ref` text NOT NULL,\n\t`before` text,\n\t`after` text,\n\t`forced` integer,\n\t`created` integer,\n\t`deleted` integer,\n\t`base_ref` text,\n\t`pusher_id` text,\n\t`head_commit_sha` text,\n\t`updated_at` integer,\n\tPRIMARY KEY(`repo_id`, `ref`)\n);\n",
     "0026_curvy_micromacro":
       "CREATE TABLE `deployment_statuses` (\n\t`id` text PRIMARY KEY NOT NULL,\n\t`repo_id` text NOT NULL,\n\t`deployment_id` text NOT NULL,\n\t`state` text,\n\t`environment` text,\n\t`target_url` text,\n\t`environment_url` text,\n\t`description` text,\n\t`creator_id` text,\n\t`created_at` integer,\n\t`updated_at` integer\n);\n--> statement-breakpoint\nCREATE INDEX `idx_deployment_statuses_deployment` ON `deployment_statuses` (`deployment_id`,`created_at`);--> statement-breakpoint\nCREATE TABLE `deployments` (\n\t`id` text PRIMARY KEY NOT NULL,\n\t`repo_id` text NOT NULL,\n\t`ref` text,\n\t`sha` text,\n\t`task` text,\n\t`environment` text,\n\t`production_environment` integer,\n\t`transient_environment` integer,\n\t`description` text,\n\t`creator_id` text,\n\t`created_at` integer,\n\t`updated_at` integer\n);\n--> statement-breakpoint\nCREATE INDEX `idx_deployments_repo_env` ON `deployments` (`repo_id`,`environment`,`created_at`);--> statement-breakpoint\nCREATE TABLE `pr_review_threads` (\n\t`id` text PRIMARY KEY NOT NULL,\n\t`repo_id` text NOT NULL,\n\t`pr_number` integer NOT NULL,\n\t`resolved` integer,\n\t`resolved_at` integer,\n\t`resolver_id` text,\n\t`first_comment_id` text,\n\t`comment_count` integer,\n\t`updated_at` integer\n);\n--> statement-breakpoint\nCREATE INDEX `idx_pr_review_threads_pr` ON `pr_review_threads` (`repo_id`,`pr_number`);",
+    "0027_closed_miek":
+      "CREATE TABLE `check_suites` (\n\t`repo_id` text NOT NULL,\n\t`check_suite_id` text PRIMARY KEY NOT NULL,\n\t`head_sha` text,\n\t`head_branch` text,\n\t`status` text,\n\t`conclusion` text,\n\t`app_slug` text,\n\t`latest_check_runs_count` integer,\n\t`updated_at` integer\n);\n--> statement-breakpoint\nCREATE INDEX `idx_check_suites_sha` ON `check_suites` (`head_sha`);--> statement-breakpoint\nCREATE TABLE `push_events` (\n\t`delivery_id` text PRIMARY KEY NOT NULL,\n\t`repo_id` text NOT NULL,\n\t`ref` text NOT NULL,\n\t`before` text,\n\t`after` text,\n\t`forced` integer,\n\t`created` integer,\n\t`deleted` integer,\n\t`base_ref` text,\n\t`pusher_id` text,\n\t`head_commit_sha` text,\n\t`updated_at` integer\n);\n--> statement-breakpoint\nCREATE INDEX `idx_push_events_repo_ref` ON `push_events` (`repo_id`,`ref`,`updated_at`);--> statement-breakpoint\nALTER TABLE `pull_requests` ADD `merge_commit_sha` text;",
+    "0028_burly_nemesis": "ALTER TABLE `check_suites` ADD `pull_request_numbers` text;",
   },
 } as const;

package/src/mirror.ts

@@ -465,6 +465,11 @@ export const pull_requests = sqliteTable(
     // is gated at READ time (a false-positive parse never surfaces because no issue matches it).
     head_ref: text("head_ref"),
     linear_issue_identifier: text("linear_issue_identifier"),
+    // CTC-691: the merge commit SHA GitHub assigns when a PR merges — the join key the host uses
+    // to match a github.pr.merged event to its deployment (setFilterStateMerged). Nullable: GitHub
+    // also sends a throwaway test-merge SHA on UNMERGED PRs, and pre-backfill rows lack it; stored
+    // verbatim because the host discriminates on `merged`, not on SHA presence.
+    merge_commit_sha: text("merge_commit_sha"),
   },
   (t) => [
     primaryKey({ columns: [t.repo_id, t.number] }),
@@ -666,6 +671,14 @@ export const pr_events = sqliteTable(
  * distinguishes a fast-forward from a force-push when read alongside `forced`, and a consumer that
  * only ever sees the latest row would otherwise have to infer it.
  *
+ * ⚠️ WEBHOOK-ONLY, NO BOOTSTRAP — a ref whose last push predates this rollout has NO row until its
+ * next push, and on a quiet branch that is never. Raised on #758 (Codex P2). Adding this table to
+ * `SNAPSHOT_TABLES` seeds a fresh replica from the DO, which is right and necessary, but it cannot
+ * invent rows the DO never received: an unpushed-since-rollout ref is currently indistinguishable
+ * from one that never existed — the exact ambiguity the `deleted = 1` rule below exists to avoid for
+ * the OTHER case. The fix is the same backfill as `pr_review_threads` above (CTC-677): seed from
+ * GitHub's current refs rather than only from future webhooks.
+ *
  * ⚠️ `created` / `deleted` are the webhook's own booleans for branch creation and deletion. A DELETED
  * ref keeps its row with `deleted = 1` rather than being removed: "this branch is gone" is a fact a
  * consumer needs, and a vanished row is indistinguishable from one that never existed.
@@ -697,6 +710,175 @@ export const pushes = sqliteTable(
   (t) => [primaryKey({ columns: [t.repo_id, t.ref] })],
 );
 
+/**
+ * CTC-704 — ONE ROW PER PUSH DELIVERY, append-only, alongside the per-ref `pushes` row above.
+ *
+ * ⛔ WHY THIS EXISTS, and it is a parity blocker rather than a nicety. `pushes` is keyed
+ * `(repo_id, ref)` and carries that ref's CURRENT head — deliberately, and that choice is still
+ * right for the question it answers. But a feed producer emits one edge per ROW, so it can never
+ * emit more than one edge per ref: **two pushes to the same ref between two producer ticks collapse
+ * to one edge**, and the second is unrecoverable from the mirror. CTL-48 measured the consequence
+ * against real traffic — **101 of 138** unmatched smee events were `github.push`, which makes GitHub
+ * parity structurally INCONCLUSIVE and the last smee tunnel un-retirable (CTL-1929).
+ *
+ * ⚠️ The 101 is an UPPER BOUND from a replay, not the live rate: a replay sees one row per ref, while
+ * live only pushes inside a single tick collapse. The number that matters is the shadow window's.
+ *
+ * ⭐ WHY A SECOND TABLE RATHER THAN RE-KEYING `pushes`. This is the same shape `deployment_statuses`
+ * takes next to `deployments`, and for the same stated reason: a consumer that acts on the ARRIVAL of
+ * a transition needs every arrival, while a consumer asking "where is this ref now" needs exactly one
+ * row. Re-keying `pushes` per push would serve the first and break the second — rebase detection
+ * (broker/router.mjs:1582) reads the per-ref head, `SNAPSHOT_TABLES` seeds it as CURRENT STATE, and
+ * both would then have to page an unbounded log to answer a one-row question. Splitting keeps
+ * ⭐ ref-latest EXACTLY derivable — it is the untouched `pushes` row — and costs one additive table.
+ *
+ * ⚠️ NOT SEEDED (absent from `SNAPSHOT_TABLES`), and that is the deliberate half of the trade. This
+ * table grows with every push — 3,746/month on mini-2 — and it is the unboundedness argument in
+ * `pushes`' own doc that makes it wrong to hand a fresh replica the entire history. A new host reads
+ * ref-latest from the seeded `pushes` row and per-push edges from its cursor forward, which is what a
+ * feed CONSUMER needs; nobody in the census asks a fresh replica for last month's push sequence.
+ *
+ * ⭐ THE PK IS THE WEBHOOK DELIVERY ID, which makes a redelivery idempotent for free. GitHub resends
+ * with the same `x-github-delivery`, and `processed_events.delivery_id` already drops a redelivery
+ * BEFORE the normalizer runs (MirrorDO.ts:2276) — so this table cannot double-count even if that
+ * upstream guard were bypassed. ⛔ A synthetic/random id would NOT have this property, and
+ * `(repo_id, ref, after)` would not either: the `pushes` doc records that two separate pushes can
+ * carry the SAME head sha (a revert-and-repush, a branch reset onto an existing sha), so keying on
+ * `after` silently collapses exactly the force-push case this table exists to preserve.
+ *
+ * ⚠️ WEBHOOK-ONLY, NO BACKFILL — like `pushes`, and here it is honest rather than a gap: an event log
+ * cannot invent deliveries that were never received. Parity is measured from the cutover forward.
+ */
+export const push_events = sqliteTable(
+  "push_events",
+  {
+    /** GitHub's `x-github-delivery` for the push webhook — see the header on why this, not a uuid. */
+    delivery_id: text("delivery_id").primaryKey(),
+    repo_id: text("repo_id").notNull(),
+    /** Full git ref as GitHub sends it, e.g. `refs/heads/main` — NOT shortened, so tags are unambiguous. */
+    ref: text("ref").notNull(),
+    /** The ref's head BEFORE this push (all-zero sha when the ref was just created). */
+    before: text("before"),
+    /** The ref's head AFTER this push (all-zero sha when the ref was deleted). */
+    after: text("after"),
+    /** True when the push was a force-push — the discriminator rebase detection actually keys on. */
+    forced: integer("forced"),
+    /** True when this push created the ref. */
+    created: integer("created"),
+    /** True when this push deleted the ref. */
+    deleted: integer("deleted"),
+    /** The ref this branch was created FROM, when GitHub reports one (`base_ref`); usually null. */
+    base_ref: text("base_ref"),
+    /** Who pushed — the resolvable `github:<login>` key, same convention as pr_events.actor_id. */
+    pusher_id: text("pusher_id"),
+    /** Head commit sha of the pushed head, when the payload carried a head_commit. */
+    head_commit_sha: text("head_commit_sha"),
+    /**
+     * ⛔ DELIVERY TIME, not the head commit's timestamp — the same rule `pushes.updated_at` follows
+     * and for the same reason (a force-push to an older commit moves a commit clock BACKWARD). Here
+     * it is also the producer's ORDERING key, so it must be monotonic in arrival order.
+     */
+    updated_at: integer("updated_at"),
+  },
+  (t) => [index("idx_push_events_repo_ref").on(t.repo_id, t.ref, t.updated_at)],
+);
+
+/**
+ * CTC-667 item 4 — a GitHub CHECK SUITE's completion, one row per suite id.
+ *
+ * ⛔ WHY THIS EXISTS. `github.check_suite.completed` is the orchestrator's CI wait
+ * (broker/router.mjs:1497) and the HIGHEST-VOLUME signal in CTL's census — 20,400 events on mini-2 in
+ * 2026-08 — and the mirror stored NO row for it: `normalizeGithub`'s `check_suite` case was a
+ * deliberate `return []`. So the one thing every phase agent blocks on had nothing to read, and the
+ * last GitHub smee tunnel could not be retired.
+ *
+ * ⭐⭐ THE ANSWER TO CTL'S QUESTION — "a suite row or an agreed derivation? just say which is
+ * contractual" — IS **THIS ROW**, AND A DERIVATION FROM `check_runs` IS EXPLICITLY *NOT*
+ * CONTRACTUAL. GitHub computes the rollup itself, and its answer is not reproducible from the
+ * constituent runs without replicating rules we do not have:
+ *
+ *   • ⛔ A DERIVATION MUST NOT BE "every run concluded `success`". `neutral` and `skipped` are
+ *     NON-FAILING under GitHub's own rollup, and this fleet has real rows of both — check run
+ *     `84696876797` is a genuine `neutral` (measured while fixing CTC-673). The naive rule reports a
+ *     GREEN suite as not-green, which for a CI wait means a phase agent blocking forever on a build
+ *     that already passed.
+ *   • ⛔ REQUIRED vs NON-REQUIRED checks are a BRANCH-PROTECTION fact, not a check_runs fact. A suite
+ *     can conclude `success` with a failing non-required run in it. Nothing in `check_runs` carries
+ *     the required-ness, so no derivation over that table can agree with GitHub in general.
+ *
+ * Storing GitHub's own `conclusion` sidesteps both. ⚠️ IF a consumer ever must derive one anyway
+ * (e.g. for suites predating this table), the ONLY defensible rule is **"no run failed, and all runs
+ * completed"** — never "all concluded success". Written down here so the second definition CTL was
+ * right to fear does not get invented independently.
+ *
+ * ⭐ CURRENT STATE, one row per suite id, same as every table here. A suite's interesting movement is
+ * `status`/`conclusion` converging, and the ORDERED history is the raw passthrough feed's job
+ * (ADR-0017) — exactly the argument `pushes` above makes.
+ *
+ * ⚠️ `conclusion` IS MUTABLE AFTER COMPLETION and this is the CTC-673 trap in a new place: a re-run
+ * or a late correction changes `conclusion` while `updated_at` (derived from an EVENT time) can stay
+ * put. CTC-673 fixed the upsert guard to accept an equal-timestamp write whose CONTENT differs, which
+ * is what lets this row converge at all — do not re-narrow that guard.
+ *
+ * ⚠️ `app_slug` distinguishes suites from different CI providers on one SHA (GitHub Actions, CodeQL,
+ * a third-party app). A consumer that waits on "the" suite for a SHA without it would pick one
+ * arbitrarily and report the wrong verdict on any repo with two providers — which this one has.
+ */
+export const check_suites = sqliteTable(
+  "check_suites",
+  {
+    repo_id: text("repo_id").notNull(),
+    /** GitHub's own check-suite id, as a string (same convention as check_runs.check_run_id). */
+    check_suite_id: text("check_suite_id").primaryKey(),
+    /** The commit this suite ran against — the join key every CI wait uses. */
+    head_sha: text("head_sha"),
+    /** The branch GitHub attributes the suite to, when it reports one. */
+    head_branch: text("head_branch"),
+    /** `queued` | `in_progress` | `completed` — a conclusion is only meaningful once completed. */
+    status: text("status"),
+    /** GitHub's OWN rollup: success | failure | neutral | cancelled | timed_out | action_required |
+     *  stale | skipped | startup_failure, or null while the suite is still running. THE contract. */
+    conclusion: text("conclusion"),
+    /** Which app produced this suite — see the app_slug note in the header. */
+    app_slug: text("app_slug"),
+    /** GitHub's own count of the runs in the suite; useful for spotting a suite still filling up. */
+    latest_check_runs_count: integer("latest_check_runs_count"),
+    /**
+     * CTC-712 — the PRs GitHub itself attributed this suite to, as a JSON array of numbers
+     * (`"[123,124]"`). ⛔ THE CONSUMER'S ONLY KEY. `broker/router.mjs:1497` reaches an interest for
+     * `github.check_suite.completed` exclusively through `detail.prNumbers`; an empty one means the
+     * CI wait is never resolved, and under `enforce` there is no smee copy to recover it from.
+     *
+     * ⛔⛔ DO NOT DERIVE THIS FROM `head_sha` → `pull_requests.head_sha`. `pull_requests.head_sha` is
+     * LAST-STATE, so a suite that ran against a head the PR has since moved past can never be joined
+     * back. Measured by CTL on mini-2 (CTC-712): of 202 suites, 93 joined (46%) and 109 missed (54%)
+     * — and the misses were active PR branches (`ryan/ctl-1944-direnv-fleet` 25, `ryan/ctl-1949-ask-
+     * traps` 19), not main. ⭐ The sharpest row: `ctc-pin-sdk-0.8.13` appears in BOTH the hits (10)
+     * and the misses (10) — one branch, where only the suites matching the PR's CURRENT head join.
+     * That is the superseded-head pattern isolated in a single branch, so the 46% is structural.
+     *
+     * ⚠️ Worse than a coverage gap, it is a RACE: the join cannot tell a legitimately-superseded
+     * suite from a current one whose PR moved a moment later, and it drops both — silently, as an
+     * emit-decline rather than an error. Same class as the `pushes` collapse (CTC-704), fixed the
+     * same way: store the EDGE the payload carries, not a projection of state.
+     *
+     * ⚠️ `null` AND `"[]"` ARE DIFFERENT ANSWERS and the distinction is deliberate. `null` = GitHub's
+     * payload carried no `pull_requests` field at all (we do not know); `"[]"` = GitHub told us this
+     * suite belongs to no PR (a `main` push, `release-please--*`). A consumer that must not
+     * distinguish them collapses both to the empty list — which is exactly what the tunnel producer
+     * already does (`webhook-events.ts` `parseCheckSuite` builds `prNumbers: []` for both), so this
+     * column is at PARITY with the tunnel it replaces and strictly more informative.
+     *
+     * ⚠️ KNOWN LIMIT, stated rather than discovered later: GitHub delivers an EMPTY `pull_requests`
+     * for a suite on a FORK's PR. That is a real blind spot — but it is the tunnel's blind spot too,
+     * for the same reason, so retiring the tunnel loses nothing here.
+     */
+    pull_request_numbers: text("pull_request_numbers"),
+    updated_at: integer("updated_at"),
+  },
+  (t) => [index("idx_check_suites_sha").on(t.head_sha)],
+);
+
 /**
  * CTC-667 — a GitHub DEPLOYMENT, one row per deployment id.
  *
@@ -805,6 +987,21 @@ export const deployment_statuses = sqliteTable(
  * key is a node id, which is why it is said out loud: a reader assuming the numeric convention would
  * write a joiner that never matches.
  *
+ * ⛔⛔ WEBHOOK-ONLY, NO BACKFILL — SO THIS TABLE IS NOT YET A MERGE GATE, AND READING IT AS ONE IS A
+ * FALSE GREEN. Raised by a reviewer on #758 (Codex P1) and it is right: GitHub emits
+ * `pull_request_review_thread` ONLY on `resolved` / `unresolved`. A thread that has been OPENED and
+ * never touched again produces no event at all, and nothing else writes here — the
+ * `pull_request_review_comment` webhook carries no thread id, and there is no reconcile leg for this
+ * table. So a PR whose threads are all still open holds ZERO rows, which is byte-identical to a PR
+ * with nothing outstanding.
+ *
+ * ⚠️ That error points the WORST possible direction for the question this table exists to answer:
+ * "does this PR have unresolved threads" would read NO for exactly the PRs that most need a yes. Until
+ * a backfill lands (CTC-677 — seed a PR's thread set from GitHub's GraphQL `pullRequest.reviewThreads`
+ * on the same reconcile pass that already sweeps its comments), this table answers only the NARROWER
+ * question it can actually answer: "of the threads we have observed a resolution event for, what is
+ * their current state". Do not wire the AGENTS.md merge gate to it before then.
+ *
  * ⚠️ `first_comment_id` is the anchor INTO `pr_review_comments` (the earliest comment the thread's
  * payload carries), because the reverse link does not exist — the `pull_request_review_comment`
  * webhook carries no thread id, so a comment cannot name its thread. Nullable: a thread payload