@catalyst-cloud/schema 0.1.5 → 0.1.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catalyst-cloud/schema",
-  "version": "0.1.5",
+  "version": "0.1.9",
   "type": "module",
   "description": "Typed Drizzle schema = single source of truth for the per-tenant Mirror DO SQLite store (CTC-13 / ADR-0002). Shared by the mirror Worker, the host-sync replica, and the browser OPFS replica.",
   "license": "MIT",
@@ -30,7 +30,7 @@
     "drizzle-orm": "0.44.7"
   },
   "devDependencies": {
-    "@catalyst-cloud/typescript-config": "workspace:*",
+    "@catalyst-cloud/typescript-config": "0.0.0",
     "@types/node": "^25.9.3",
     "drizzle-kit": "^0.31.0",
     "prettier": "^3.8.4",

package/src/index.ts

@@ -30,6 +30,9 @@ import {
   event_log,
   sync_cursors,
   fleet_activity,
+  agent_sessions,
+  agent_activities,
+  fleet_activity_pruned,
   pull_sweeps,
 } from "./mirror.js";
 
@@ -45,6 +48,18 @@ export {
   type MigrationJournalEntry,
 } from "./migrate.js";
 
+// CTC-471 — schema-skew detection. A consumer reports the bundle it LOADED (not the one its lockfile
+// claims) and the hub places that tail in its own journal. Exported here so the SDK, the replica and
+// the Worker all compare the same way; see ./skew.ts for why a version string cannot do this job.
+export {
+  loadedSchemaIdentity,
+  schemaSkew,
+  schemaSkewNeedsAttention,
+  schemaIncludesMigration,
+  type SchemaIdentity,
+  type SchemaSkew,
+} from "./skew.js";
+
 /** Every table in the Mirror DO store — pass as `drizzle(storage, { schema: mirrorSchema })`. */
 export const mirrorSchema = {
   issues,
@@ -71,6 +86,13 @@ export const mirrorSchema = {
   // migration bundle must produce) but DELIBERATELY ABSENT from `mirrorEntityTables` below — see the
   // no-feed arithmetic on the table definition in ./mirror.ts.
   pull_sweeps,
+  // CTC-393: the fleet_activity anti-resurrection tombstone. Same posture as pull_sweeps — a real
+  // table the DO creates, but pure internal bookkeeping that must NEVER ride the change-feed (a
+  // consumer has no use for "a row was once terminal here"; see the doc on the table itself).
+  // CTC-524: agent sessions + their activity stream.
+  agent_sessions,
+  agent_activities,
+  fleet_activity_pruned,
 } as const;
 
 /**
@@ -107,6 +129,9 @@ export const mirrorEntityTables = {
   // CTC-350: fleet coordination current-state — replicates like any other entity (one row per
   // host_id/ticket), unlike the infra tables above which never ride a delta.
   fleet_activity,
+  // CTC-524: agent sessions + their activities.
+  agent_sessions,
+  agent_activities,
 } as const;
 
 /** The wire `entity` name of a change-feed-carried table (keys of mirrorEntityTables). */

package/src/migrations.generated.ts

@@ -123,6 +123,27 @@ export const MIRROR_MIGRATIONS = {
         tag: "0015_brainy_lady_ursula",
         breakpoints: true,
       },
+      {
+        idx: 16,
+        version: "6",
+        when: 1786723845155,
+        tag: "0016_vengeful_the_stranger",
+        breakpoints: true,
+      },
+      {
+        idx: 17,
+        version: "6",
+        when: 1786770512483,
+        tag: "0017_clammy_doctor_octopus",
+        breakpoints: true,
+      },
+      {
+        idx: 18,
+        version: "6",
+        when: 1786783490609,
+        tag: "0018_brainy_clint_barton",
+        breakpoints: true,
+      },
     ],
   },
   migrations: {
@@ -157,5 +178,10 @@ export const MIRROR_MIGRATIONS = {
       "-- CTC-451 — canonicalise `reviews.state` to UPPERCASE.\n--\n-- The two ingestion legs disagreed at the source: REST (`/pulls/{n}/reviews`, the reconcile) returns\n-- `COMMENTED`, while the `pull_request_review` webhook delivers `commented`. Both legs stored the\n-- value verbatim, so the column carried BOTH casings of the same state. Measured on the live corpus\n-- before this migration: COMMENTED 3957 / commented 921 — a raw `WHERE state = 'COMMENTED'` missed\n-- 19% of reviews. Three read paths worked around it in application code, and none of those\n-- workarounds travel to the host replica, which is the fleet's only data path.\n--\n-- Both writers are fixed (apps/mirror/src/normalize/github.ts buildReviewRow), but that alone\n-- converges only rows the reconcile re-sweeps — and the reconcile walks OPEN PRs only, so every\n-- lowercase row on an already-closed PR would keep its casing forever. This backfill is what makes\n-- the corpus canonical rather than just the new writes.\n--\n-- ⭐ Deliberately a DATA migration in the SHARED bundle: the DO and every host replica apply the\n-- same MIRROR_MIGRATIONS through the same runner, so each side canonicalises its OWN rows locally.\n-- That converges the replica WITHOUT emitting ~921 change_log deltas, which would otherwise evict\n-- real deltas from the 200k/7d ring (the CTC-279 flood).\n--\n-- Idempotent by construction: upper() of an already-uppercase value is a no-op, and the WHERE clause\n-- makes the write a no-op too. Safe to re-run.\nUPDATE `reviews` SET `state` = upper(`state`) WHERE `state` IS NOT NULL AND `state` <> upper(`state`);\n",
     "0015_brainy_lady_ursula":
       "CREATE TABLE `pull_sweeps` (\n\t`repo_id` text NOT NULL,\n\t`pr_number` integer NOT NULL,\n\t`head_sha` text NOT NULL,\n\t`swept_at` integer NOT NULL,\n\t`check_run_count` integer,\n\t`checks_error` integer,\n\t`commit_status_count` integer,\n\t`statuses_error` integer,\n\t`review_count` integer,\n\t`reviews_error` integer,\n\tPRIMARY KEY(`repo_id`, `pr_number`)\n);\n",
+    "0016_vengeful_the_stranger":
+      "CREATE TABLE `fleet_activity_pruned` (\n\t`host_id` text NOT NULL,\n\t`ticket` text NOT NULL,\n\t`last_event_ts` integer NOT NULL,\n\t`pruned_at` integer NOT NULL,\n\tPRIMARY KEY(`host_id`, `ticket`)\n);\n",
+    "0017_clammy_doctor_octopus": "ALTER TABLE `users` ADD `email` text;",
+    "0018_brainy_clint_barton":
+      "CREATE TABLE `agent_activities` (\n\t`id` text PRIMARY KEY NOT NULL,\n\t`session_id` text NOT NULL,\n\t`type` text,\n\t`body` text,\n\t`signal` text,\n\t`created_at` integer,\n\t`updated_at` integer,\n\t`raw` text\n);\n--> statement-breakpoint\nCREATE INDEX `idx_agent_activities_session_created` ON `agent_activities` (`session_id`,`created_at`);--> statement-breakpoint\nCREATE TABLE `agent_sessions` (\n\t`id` text PRIMARY KEY NOT NULL,\n\t`issue_id` text,\n\t`status` text,\n\t`app_user_id` text,\n\t`creator_id` text,\n\t`created_at` integer,\n\t`updated_at` integer,\n\t`removed_at` integer,\n\t`raw` text\n);\n--> statement-breakpoint\nCREATE INDEX `idx_agent_sessions_issue` ON `agent_sessions` (`issue_id`,`updated_at`);",
   },
 } as const;

package/src/mirror.ts

@@ -116,6 +116,10 @@ export const users = sqliteTable("users", {
   avatar_url: text("avatar_url"),
   type: text("type"), // human | agent | bot | app
   source: text("source"), // linear | github
+  // CTC-506 — only ever populated for source='linear' rows (the roster sweep is the one caller that
+  // fetches it; every other upsert site omits the column entirely rather than writing null over it,
+  // see users/identity.ts's userRow()). Feeds the registry-side identity join, never read for github.
+  email: text("email"),
   updated_at: integer("updated_at"),
 });
 
@@ -197,6 +201,65 @@ export const issue_history = sqliteTable(
 // progress, health, and presentation (icon/color). `target_date` / `start_date` are Linear
 // TimelessDates ("YYYY-MM-DD") — stored AS-IS as TEXT, NEVER through isoToMs (the same trap as
 // issues.due_date). `progress` is a Float (0..1).
+/**
+ * CTC-524 — a Linear agent session: one delegated unit of agent work on an issue.
+ *
+ * ⛔ WHY THIS TABLE EXISTS AT ALL. CTC-499's responder ACKNOWLEDGES an `AgentSessionEvent` inside
+ * Linear's 10-second deadline and enqueues it — and the queue then dropped it, because
+ * `normalizeLinearEvent`'s switch had no case and fell through to `default: return []`. Acknowledged,
+ * queued, normalized to nothing, discarded. So nothing could render a session: CTC-503's own first
+ * payload — 14 dead sessions since Aug 10 — lived only in Linear.
+ *
+ * ⚠️ `status` IS LINEAR'S, NOT OURS. `active` / `awaitingInput` / `complete` / `error` — stored as the
+ * provider sends it rather than mapped into a local vocabulary, so a reader can tell "Linear says
+ * awaitingInput" from any judgement we layer on top. The abandoned-vs-awaiting distinction CTC-503
+ * needs is DERIVED from `status` + `updated_at`, and derivation belongs above this row, not in it.
+ */
+export const agent_sessions = sqliteTable(
+  "agent_sessions",
+  {
+    id: text("id").primaryKey(), // Linear agentSession id
+    issue_id: text("issue_id"), // FK → issues.id; the session is issue-scoped (no session above it)
+    /** Linear's own session status, verbatim. */
+    status: text("status"),
+    /** The app/user the session belongs to (FK → users.id) — the agent that was delegated to. */
+    app_user_id: text("app_user_id"),
+    /** Who created it. ⚠️ MEASURED NULL in practice (ARCHITECT-7) — a session created by the app
+     *  itself has no human creator, which is exactly why CTC-500 needs an explicit target. */
+    creator_id: text("creator_id"),
+    created_at: integer("created_at"),
+    updated_at: integer("updated_at"), // drives the last-write-wins upsert guard
+    removed_at: integer("removed_at"),
+    raw: text("raw"), // full fidelity / forward-compat, like issues.raw
+  },
+  (t) => [index("idx_agent_sessions_issue").on(t.issue_id, t.updated_at)],
+);
+
+/**
+ * CTC-524 — one activity within a session: the thought/action/response/elicitation stream.
+ *
+ * ⚠️ `type` CARRIES THE NOTIFY SEMANTICS and must not be flattened. `elicitation` and `error` reach a
+ * human's Inbox; `thought`/`action`/`response` do not (CTC-501, and Linear's behaviour rather than our
+ * preference). A reader that cannot tell an elicitation from a thought cannot tell a question from
+ * narration.
+ */
+export const agent_activities = sqliteTable(
+  "agent_activities",
+  {
+    id: text("id").primaryKey(),
+    session_id: text("session_id").notNull(), // FK → agent_sessions.id
+    /** thought | action | response | elicitation | error — Linear's content type, verbatim. */
+    type: text("type"),
+    body: text("body"),
+    /** Linear's `signal` on the activity (e.g. stop), when present. */
+    signal: text("signal"),
+    created_at: integer("created_at"),
+    updated_at: integer("updated_at"),
+    raw: text("raw"),
+  },
+  (t) => [index("idx_agent_activities_session_created").on(t.session_id, t.created_at)],
+);
+
 export const projects = sqliteTable("projects", {
   id: text("id").primaryKey(),
   name: text("name"),
@@ -455,6 +518,33 @@ export const fleet_activity = sqliteTable(
   (t) => [primaryKey({ columns: [t.host_id, t.ticket] })],
 );
 
+/**
+ * CTC-393 — a tombstone of the last `last_event_ts` a (host_id, ticket) reached TERMINAL disposition
+ * at, kept for a while AFTER `fleet_activity`'s own row is hard-pruned.
+ *
+ * ⛔ THE BUG THIS CLOSES. `fleet_activity` has no soft-delete column — a terminal row past its
+ * retention window is genuinely DELETEd, not marked removed. Without this table, a stale/delayed
+ * pre-terminal event arriving AFTER that deletion finds no existing row (`readFleetRow` → null), the
+ * reducer treats that as "first seen", and a DEAD ticket is silently RESURRECTED as an active row
+ * with no disposition — orphaned forever, since nothing prunes an active row.
+ *
+ * Bounded the same way `fleet_activity` itself is bounded: a tombstone is written once per prune and
+ * pruned again after its own retention window (see `fleet-activity.ts`'s prune step), so this table
+ * never grows past "recently terminal tickets, twice over" — never proportional to all-time history.
+ */
+export const fleet_activity_pruned = sqliteTable(
+  "fleet_activity_pruned",
+  {
+    host_id: text("host_id").notNull(),
+    ticket: text("ticket").notNull(),
+    // The watermark the deleted row reached terminal at — a later event must beat THIS, not just
+    // "any row", to be accepted as a genuine reopening rather than a stale replay.
+    last_event_ts: integer("last_event_ts").notNull(),
+    pruned_at: integer("pruned_at").notNull(), // ms epoch — when the live row was deleted; ages this out
+  },
+  (t) => [primaryKey({ columns: [t.host_id, t.ticket] })],
+);
+
 /**
  * ⭐ THE PER-PR ENRICHMENT RECEIPT — the row that makes "this PR has no CI" distinguishable from
  * "nobody ever swept this PR". CTC-436 follow-up.

package/src/skew.ts

@@ -0,0 +1,181 @@
+// CTC-471 — "is this consumer's schema bundle older than mine?", answered from the bundle a consumer
+// ACTUALLY LOADED rather than from what its lockfile claims.
+//
+// THE DEFECT THIS CLOSES. A host ran `@catalyst-cloud/schema@0.1.3` for 21+ days while the published
+// latest was 0.1.5. Every column added since 0.1.3 — `head_ref`, `linear_issue_identifier` — was
+// dropped from every replica upsert, and the host reported healthy the entire time: the replica's
+// migration tail and the installed bundle's journal tail agreed (`0008_optimal_rattler`), because the
+// host was faithfully applying the stale bundle it had. Nothing anywhere compared that tail to the
+// hub's.
+//
+// ⛔ WHY A VERSION STRING CANNOT BE THE ANSWER. The obvious fix — report `package.json`'s version and
+// compare semver — fails on the exact fleet that motivated the ticket, for three independent reasons:
+//
+//   1. `bun.lock` pins a RESOLUTION, not a range. `^0.1.3` would admit 0.1.5, but the lock records
+//      0.1.3 exactly, so `bun install` re-installs 0.1.3 deterministically, forever.
+//   2. A nested `node_modules` SHADOWS the root install. This fleet has already run one SDK version
+//      while the lockfile claimed another — so an install-time check would have PASSED while the
+//      replica sat at 0008.
+//   3. The daemon holds old code in memory; anything on disk is inert until restart.
+//
+// Every one of those is a gap between LOCKED and LOADED. So the reported value here is derived from
+// the `MIRROR_MIGRATIONS` array **at runtime**, in the same module graph that will do the applying. A
+// shadowed install reports the shadowed bundle's tail, because that is genuinely the bundle whose
+// columns the replica will have. That property is the whole point — see {@link loadedSchemaIdentity}.
+//
+// ⛔ AND "DID NOT REPORT" IS NOT "UP TO DATE". `unreported` is its own state and must never fold into
+// `current`. CTC-479 cost 27 hours precisely because a null read as a clean bill of health: an
+// absence with no reason is indistinguishable from proven-healthy, and an operator (and an agent
+// actively hunting the bug) both read it as fine. The invariant `team-liveness.ts` states for the
+// Linear legs — "every way of not proving liveness resolves to a RECORDED REASON, never to a clean
+// bill of health inferred from silence" — is the same invariant, one layer up.
+
+import { MIRROR_MIGRATIONS } from "./migrations.generated.js";
+import type { MigrationBundle } from "./migrate.js";
+
+/**
+ * What a consumer reports about the migration bundle it LOADED — not the one it installed, locked, or
+ * intended.
+ *
+ * `count` is not redundant with `tail`: it is what distinguishes a consumer running AHEAD of the hub
+ * (a newer bundle, whose tail the hub has never heard of) from a consumer reporting garbage (a tail
+ * the hub has never heard of, from a bundle no bigger than its own). Without it both collapse into
+ * "unrecognized" and a mid-deploy fleet looks broken.
+ */
+export interface SchemaIdentity {
+  /** The last journal tag in the loaded bundle, e.g. `0015_brainy_lady_ursula`. Null iff empty. */
+  tail: string | null;
+  /** How many migrations the loaded bundle carries. */
+  count: number;
+}
+
+/**
+ * The hub's verdict on one consumer's reported identity.
+ *
+ * ⚠️ Five states, deliberately. Collapsing any of them into a boolean recreates the defect: a
+ * consumer that is behind, a consumer that never reported, and a consumer whose bundle the hub cannot
+ * place are three DIFFERENT operational situations, and exactly one of them (`current`) is healthy.
+ */
+export type SchemaSkew =
+  /** The consumer's tail is the hub's tail. The only healthy state. */
+  | { state: "current"; tail: string }
+  /** The consumer is missing migrations the hub already has — it WILL drop those columns. */
+  | { state: "behind"; tail: string; missing: string[] }
+  /**
+   * The consumer carries migrations the hub does not — normal and transient mid-deploy (a host
+   * upgraded first), so it is NOT an error, but it is not `current` either and must stay visible.
+   */
+  | { state: "ahead"; tail: string; extra: number }
+  /**
+   * The consumer reported a tail the hub cannot place, from a bundle no larger than its own. Neither
+   * behind nor ahead: the question is unanswerable, and the unanswerable-ness is what gets recorded.
+   */
+  | { state: "unrecognized"; tail: string }
+  /**
+   * ⛔ The consumer reported nothing. NOT healthy, NOT `current` — see the header. Pre-CTC-471 SDKs
+   * land here, which is correct: the hub genuinely does not know what they are running.
+   */
+  | { state: "unreported" };
+
+/**
+ * The identity of the bundle THIS process actually loaded.
+ *
+ * ⭐ Derived from the imported `MIRROR_MIGRATIONS` array, never from `package.json`, a lockfile, or a
+ * build-time constant. That is what makes it a LOADED fact rather than a LOCKED claim: if a nested
+ * `node_modules` shadows the root install, this returns the shadowing bundle's tail — which is
+ * genuinely the schema the replica will have, and genuinely what the hub needs to know.
+ *
+ * The default argument exists so tests can pass a synthetic bundle; production callers pass nothing.
+ */
+export function loadedSchemaIdentity(bundle: MigrationBundle = MIRROR_MIGRATIONS): SchemaIdentity {
+  const entries = bundle.journal.entries;
+  const last = entries.length > 0 ? entries[entries.length - 1] : undefined;
+  return { tail: last?.tag ?? null, count: entries.length };
+}
+
+/**
+ * Compare a consumer's reported identity against the hub's own bundle.
+ *
+ * `reported` is deliberately typed to accept `null | undefined`: a consumer that predates CTC-471
+ * sends nothing, and the caller must not have to invent a value for it. That path returns
+ * `unreported`, never `current`.
+ */
+export function schemaSkew(
+  reported: SchemaIdentity | null | undefined,
+  bundle: MigrationBundle = MIRROR_MIGRATIONS,
+): SchemaSkew {
+  if (reported == null || reported.tail == null) return { state: "unreported" };
+
+  const tags = bundle.journal.entries.map((e) => e.tag);
+  const idx = tags.indexOf(reported.tail);
+
+  if (idx === -1) {
+    // Not in our journal. A LARGER bundle than ours is a consumer that upgraded first — expected
+    // during a rolling deploy. A same-or-smaller bundle we cannot place is genuinely unrecognized.
+    if (reported.count > tags.length) {
+      return { state: "ahead", tail: reported.tail, extra: reported.count - tags.length };
+    }
+    return { state: "unrecognized", tail: reported.tail };
+  }
+
+  // ⚠️ The tail alone is not enough to trust `current`/`behind` — see the doc on `SchemaIdentity.count`.
+  // A tail that matches our journal but a count that DISAGREES with that tail's position is an
+  // internally-inconsistent report (a corrupted client, or a future bug elsewhere), not a verified
+  // one: it cannot have come from actually walking this bundle. Codex flagged this — a mismatched
+  // count at the LAST tag would otherwise still read `current`, recreating the exact false-clean-bill
+  // condition this ticket exists to close.
+  if (reported.count !== idx + 1) return { state: "unrecognized", tail: reported.tail };
+
+  if (idx === tags.length - 1) return { state: "current", tail: reported.tail };
+
+  return { state: "behind", tail: reported.tail, missing: tags.slice(idx + 1) };
+}
+
+/**
+ * Is this verdict one an operator must act on? `behind` and `unrecognized` mean the replica cannot
+ * represent what the hub is sending. `unreported` is included ON PURPOSE — an unknown consumer is a
+ * thing to chase, not a thing to assume fine.
+ *
+ * ⚠️ `ahead` is excluded: it is expected mid-deploy and alerting on it would train operators to
+ * ignore this signal, which is strictly worse than the silence it replaces (the same argument
+ * `team-liveness.ts` makes for not black-flagging a quiet workspace).
+ */
+export function schemaSkewNeedsAttention(skew: SchemaSkew): boolean {
+  return skew.state === "behind" || skew.state === "unrecognized" || skew.state === "unreported";
+}
+
+/**
+ * CTC-393 — does a consumer's LOADED bundle include a specific migration (by journal tag)?
+ *
+ * ⭐ This is a different, more precise question than {@link schemaSkew}'s five-state verdict. A host
+ * can be `behind` the hub's exact tail (missing a LATER, unrelated migration) while still safely
+ * having every column a SPECIFIC earlier migration added — `schemaSkew` alone would wrongly treat it
+ * as ineligible. Use this when the question is "does this consumer support entity X", not "is this
+ * consumer on our exact tail" — e.g. gating a newly-replicated entity to only the hosts whose bundle
+ * already contains the migration that created its table, regardless of what shipped after.
+ *
+ * Reuses the identical journal-position lookup `schemaSkew` uses internally, so a host that reports
+ * nothing (`reported == null`) or a tail this hub cannot place resolves to `false` — the safe
+ * default-EXCLUDE this ticket requires, with no separate "unreported" branch to get wrong twice.
+ *
+ * `requiredTag` is asserted to exist in `bundle`'s own journal — a typo'd or since-renumbered tag
+ * would otherwise silently exclude EVERY consumer forever with no signal anything is wrong.
+ */
+export function schemaIncludesMigration(
+  reported: SchemaIdentity | null | undefined,
+  requiredTag: string,
+  bundle: MigrationBundle = MIRROR_MIGRATIONS,
+): boolean {
+  const tags = bundle.journal.entries.map((e) => e.tag);
+  const requiredIdx = tags.indexOf(requiredTag);
+  if (requiredIdx === -1) {
+    throw new Error(`schemaIncludesMigration: "${requiredTag}" is not in this bundle's journal`);
+  }
+  if (reported == null || reported.tail == null) return false;
+  const reportedIdx = tags.indexOf(reported.tail);
+  // Not in our journal at all: only a LARGER unrecognized bundle (a rolling-deploy "ahead" consumer,
+  // per schemaSkew's own reasoning) can be trusted to include an old migration; a same-or-smaller
+  // unplaceable tail is unrecognized garbage, never assumed to include anything.
+  if (reportedIdx === -1) return reported.count > tags.length;
+  return reportedIdx >= requiredIdx;
+}