@intentius/chant-lexicon-sql 0.114.0 → 0.115.0

package/src/clickhouse/rebuild/op.ts

@@ -25,6 +25,12 @@
  * | Drop | the old table, once its date has passed |
  * | onFailure | drop the new table and the dual-write view |
  *
+ * `gates: "outer"` (#3658) leaves out both approval gates and the Drop phase,
+ * for a caller that runs the Op under an approval of its own: the run goes
+ * from Plan to Retain, verification and the swap's own comparison included.
+ * `onFailure: "keep"` leaves out the onFailure, so a failed run's next run
+ * resumes the backfill from its receipts instead of starting again.
+ *
  * Run it until it is done: each run goes as far as the next gate. Every step
  * re-reads the server, so a run after a gate, a crash or an approval picks up
  * where the last stopped. A run killed during the backfill (Ctrl-C, a lost
@@ -55,6 +61,27 @@ export interface ClickHouseRebuildOpConfig {
   build?: boolean;
   /** How long the old table is kept after the swap, as a duration (`7d`, `36h`). Default: `7d`. */
   retain?: string;
+  /**
+   * Whose approval the swap and the drop wait for. `"own"` (the default): the
+   * Op's two gates. `"outer"`: an approval the caller already holds for a
+   * larger change that runs this Op as one of its steps. The swap gate, the
+   * drop gate and the Drop phase are left out, so the Op ends at Retain with
+   * the old table kept as `<table>__chant_old` with its retention date. Drop
+   * it after that date, or run the Op with `"own"`: its swap gate has
+   * nothing left to swap, and its drop gate binds that old table.
+   * The verification still runs, and the swap compares the tables again
+   * before the `EXCHANGE`: any difference fails the run with nothing swapped.
+   * App mode's writes-stopped gate is not an approval and stays.
+   */
+  gates?: "own" | "outer";
+  /**
+   * What a failed run does with the new table and the dual-write view.
+   * `"drop"` (the default): onFailure drops them, and the next run starts
+   * again from a new table. `"keep"`: there is no onFailure, and the next run
+   * resumes the backfill from its per-partition receipts. To start again
+   * instead, drop `<table>__chant_new` and `<table>__chant_dual`.
+   */
+  onFailure?: "drop" | "keep";
   /** The gate before the swap. Default name `approve-<name>`. */
   gate?: {
     gate?: string;
@@ -100,6 +127,8 @@ export interface ClickHouseRebuildArgs {
   stack?: string;
   ownershipEnv?: string;
   cwd?: string;
+  /** A failed run keeps the new table (`onFailure: "keep"`): a refusal says to drop it by hand. */
+  keepOnFailure?: boolean;
 }
 
 export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHouseRebuildOpResources {
@@ -112,6 +141,14 @@ export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHou
   if (config.dualWrite.mode === "materialized-view" && !config.dualWrite.cutoverColumn) {
     throw new Error(`ClickHouseRebuildOp "${config.name}": materialized-view mode needs cutoverColumn, the time column rows are cut over on`);
   }
+  if (config.gates !== undefined && config.gates !== "own" && config.gates !== "outer") {
+    throw new Error(`ClickHouseRebuildOp "${config.name}": gates must be "own" or "outer"`);
+  }
+  if (config.onFailure !== undefined && config.onFailure !== "drop" && config.onFailure !== "keep") {
+    throw new Error(`ClickHouseRebuildOp "${config.name}": onFailure must be "drop" or "keep"`);
+  }
+  const outer = config.gates === "outer";
+  const keep = config.onFailure === "keep";
 
   const args: ClickHouseRebuildArgs = {
     table: config.table,
@@ -124,6 +161,7 @@ export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHou
     ...(config.stack ? { stack: config.stack } : {}),
     ...(config.ownershipEnv ? { ownershipEnv: config.ownershipEnv } : {}),
     ...(config.path && config.path !== "." ? { cwd: config.path } : {}),
+    ...(keep ? { keepOnFailure: true } : {}),
   };
   const a = args as unknown as Record<string, unknown>;
   const swapGate = config.gate?.gate ?? `approve-${config.name}`;
@@ -133,6 +171,33 @@ export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHou
   const step = (fn: string, extra: Partial<StepDefinition & { id: string; outcomeAttribute: unknown; profile: string; timeout: string }> = {}): StepDefinition =>
     ({ ...activity(fn, a), ...extra }) as StepDefinition;
 
+  // The swap waits for this gate, unless the caller holds the approval (`gates: "outer"`).
+  const approve = phase("Approve", [
+    gate(swapGate, {
+      ...(config.gate?.timeout ? { timeout: config.gate.timeout } : {}),
+      plan: stepOutput("verify", "planDigest"),
+      description:
+        config.gate?.description ??
+        `Approve swapping the rebuilt ${config.table} in on ${config.env}: the run record's Verification line has the counts this approval binds`,
+      ...(config.gate?.approval
+        ? {
+            approval: {
+              ...config.gate.approval,
+              ...(policy
+                ? {
+                    context: {
+                      verifiedPartitions: stepOutput("verify", "partitions"),
+                      verifiedRows: stepOutput("verify", "rows"),
+                      ...config.gate.approval.context,
+                    },
+                  }
+                : {}),
+            },
+          }
+        : {}),
+    }),
+  ]);
+
   const phases = [
     ...(config.build === false ? [] : [phase("Build", [activity("chantBuild", { path: config.path ?? "." })])]),
     phase("Plan", [step("clickhouseRebuildPlan", { id: "plan", outcomeAttribute: { name: "RebuildState", from: "state" } })]),
@@ -158,49 +223,29 @@ export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHou
         ],
       }),
     ]),
-    phase("Approve", [
-      gate(swapGate, {
-        ...(config.gate?.timeout ? { timeout: config.gate.timeout } : {}),
-        plan: stepOutput("verify", "planDigest"),
-        description:
-          config.gate?.description ??
-          `Approve swapping the rebuilt ${config.table} in on ${config.env}: the run record's Verification line has the counts this approval binds`,
-        ...(config.gate?.approval
-          ? {
-              approval: {
-                ...config.gate.approval,
-                ...(policy
-                  ? {
-                      context: {
-                        verifiedPartitions: stepOutput("verify", "partitions"),
-                        verifiedRows: stepOutput("verify", "rows"),
-                        ...config.gate.approval.context,
-                      },
-                    }
-                  : {}),
-              },
-            }
-          : {}),
-      }),
-    ]),
+    ...(outer ? [] : [approve]),
     phase("Swap", [step("clickhouseRebuildSwap", { profile: "longInfra", outcomeAttribute: { name: "Dependents", from: "dependents" } })]),
     phase("Retain", [step("clickhouseRebuildRetain", { id: "retain", outcomeAttribute: { name: "RetainUntil", from: "retainUntil" } })]),
-    phase("Approve drop", [
-      gate(dropGate, {
-        ...(config.dropGate?.timeout ? { timeout: config.dropGate.timeout } : {}),
-        plan: stepOutput("retain", "dropDigest"),
-        description: config.dropGate?.description ?? `Approve dropping the old ${config.table} once its retention has passed`,
-      }),
-    ]),
-    phase("Drop", [step("clickhouseRebuildDrop", { outcomeAttribute: { name: "Dropped", from: "dropped" } })]),
+    ...(outer
+      ? []
+      : [
+          phase("Approve drop", [
+            gate(dropGate, {
+              ...(config.dropGate?.timeout ? { timeout: config.dropGate.timeout } : {}),
+              plan: stepOutput("retain", "dropDigest"),
+              description: config.dropGate?.description ?? `Approve dropping the old ${config.table} once its retention has passed`,
+            }),
+          ]),
+          phase("Drop", [step("clickhouseRebuildDrop", { outcomeAttribute: { name: "Dropped", from: "dropped" } })]),
+        ]),
   ];
 
   const op = Op({
     name: config.name,
-    overview: `Rebuild ${config.table} on ${config.env}: new table, ${config.dualWrite.mode === "app" ? "writes stopped" : "dual write"}, backfill, verify, gated swap`,
+    overview: `Rebuild ${config.table} on ${config.env}: new table, ${config.dualWrite.mode === "app" ? "writes stopped" : "dual write"}, backfill, verify, ${outer ? "swap under the caller's approval" : "gated swap"}`,
     labels: { Rebuild: "true", Env: config.env, Table: config.table },
     phases,
-    onFailure: [phase("Compensate", [step("clickhouseRebuildCompensate")])],
+    ...(keep ? {} : { onFailure: [phase("Compensate", [step("clickhouseRebuildCompensate")])] }),
   });
   return { op };
 }

package/src/clickhouse/rebuild/partitions.ts

@@ -15,6 +15,7 @@ import { clickhouseQuery } from "../http";
 import type { ClickHouseTarget } from "../live/bind";
 import { ident, sqlString } from "../apply/statements";
 import type { CopiedColumn, RebuildNames } from "./observe";
+import type { Sharding } from "./shards";
 
 /** The old table's active partitions, by id. */
 export async function sourcePartitions(target: ClickHouseTarget, names: RebuildNames): Promise<string[]> {
@@ -26,6 +27,16 @@ export async function sourcePartitions(target: ClickHouseTarget, names: RebuildN
   return rows.map((r) => r.partition_id);
 }
 
+/** The old table's active partitions on each shard of a cluster (#3663), through this server. */
+export async function shardPartitions(run: { target: ClickHouseTarget }, sharding: Sharding, database: string, table: string): Promise<Array<{ shard: number; partition: string }>> {
+  const rows = await clickhouseQuery<{ shard: number | string; partition_id: string }>(
+    run.target.endpoint,
+    `SELECT _shard_num AS shard, partition_id FROM cluster(${sqlString(sharding.cluster)}, system.parts) ` +
+      `WHERE database = ${sqlString(database)} AND table = ${sqlString(table)} AND active GROUP BY shard, partition_id ORDER BY shard, partition_id`,
+  );
+  return rows.map((r) => ({ shard: Number(r.shard), partition: r.partition_id }));
+}
+
 /** Rename the columns an expression names, old name to new, leaving function names and qualified names alone. */
 export function renameColumns(expr: string, copied: readonly CopiedColumn[]): string {
   let out = expr;

package/src/clickhouse/rebuild/rebuild-sharded.e2e.test.ts

@@ -0,0 +1,161 @@
+/**
+ * The rebuild migration on a cluster of two shards (#3663), against a
+ * scratch cluster of the pinned image: shard 1 of two replicas, shard 2 of
+ * one server, Keeper, and a cluster `chant` over all three. The profile
+ * names shard 1's first replica with topology `cluster:chant`.
+ *
+ * Each shard holds its own rows of `shop.events`, with a `Distributed`
+ * table over them. An `ORDER BY` change runs as a `ClickHouseRebuildOp`
+ * under an outer approval, keeping its new table on failure:
+ *
+ * - the first run copies shard 1's partitions and one of shard 2's, then
+ *   fails: each (shard, partition) copied has its own receipt;
+ * - the rerun skips those, copies the rest of shard 2, verifies every row of
+ *   both shards, and swaps: every shard's rows are in its live table, on
+ *   every replica, with the new sorting key.
+ *
+ * Runs the Op through core's local executor. Needs Docker; skips cleanly
+ * without it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { normalizeApply } from "@intentius/chant/apply";
+import { memoryGateLedgerPort, runOpLocally, loadProfiles, OpRunFailure, type ActivityFn, type OpConfig, type OpRunResult } from "@intentius/chant/op";
+import { dockerAvailable, startScratchCluster, type ScratchCluster } from "../container";
+import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { clickhouseApply, toApplyResult } from "../../op/activities/clickhouse-apply";
+import * as rebuildActivities from "../../op/activities/clickhouse-rebuild";
+import type { ClickHouseRebuildDeps } from "../../op/activities/clickhouse-rebuild";
+import { ClickHouseRebuildOp, type ClickHouseRebuildOpConfig } from "./op";
+
+const enabled = await dockerAvailable();
+let cluster: ScratchCluster | undefined;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-rebuild-sharded-"));
+
+beforeAll(async () => {
+  if (!enabled) return;
+  cluster = await startScratchCluster(clickhouseImage(), { shards: [2, 1], cluster: "chant", namePrefix: "chant-sql-rebuild-shards" });
+}, 600_000);
+
+afterAll(async () => {
+  await cluster?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+const MARKER = { stack: "e2e", env: "test" };
+const config = () => ({ ownership: MARKER, sql: { profiles: { e2e: { url: cluster!.replicas[0]!.url, topology: "cluster:chant" } } } });
+const deps = (extra: Partial<ClickHouseRebuildDeps> = {}): ClickHouseRebuildDeps => ({ config: config(), env: {}, log: () => undefined, ...extra });
+const q = <T = Record<string, unknown>>(endpoint: ClickHouseEndpoint, sql: string) => clickhouseQuery<T>(endpoint, sql);
+const count = async (endpoint: ClickHouseEndpoint, table: string) => Number((await q<{ n: string }>(endpoint, `SELECT count() AS n FROM ${table}`))[0]!.n);
+/** The servers of a shard, by its macro. */
+const servers = (shard: string) => cluster!.layout.flatMap((l, i) => (l.shard === shard ? [cluster!.replicas[i]!] : []));
+
+const eventsDdl = (orderBy: string) => `CREATE TABLE shop.events
+(
+  ts DateTime,
+  user_id UInt64,
+  kind LowCardinality(String)
+)
+ENGINE = MergeTree
+PARTITION BY toDate(ts)
+ORDER BY ${orderBy}`;
+const DISTRIBUTED = "CREATE TABLE shop.events_all (ts DateTime, user_id UInt64, kind LowCardinality(String)) ENGINE = Distributed(chant, shop, events, user_id)";
+
+function writeBuild(file: string, orderBy: string): string {
+  const objects = [
+    { export: "shop", type: "ClickHouse::Database", ddl: "CREATE DATABASE shop", dependsOn: [] },
+    { export: "events", type: "ClickHouse::Table", ddl: eventsDdl(orderBy), dependsOn: ["shop"] },
+    { export: "eventsAll", type: "ClickHouse::Table", ddl: DISTRIBUTED, dependsOn: ["events"] },
+  ];
+  writeFileSync(join(dir, file), JSON.stringify({ dialect: "clickhouse", applyOrder: objects.map((o) => o.export), objects }));
+  return file;
+}
+
+const opConfig = (): ClickHouseRebuildOpConfig => ({
+  name: "rebuild-events",
+  env: "e2e",
+  table: "shop.events",
+  dualWrite: { mode: "materialized-view", cutoverColumn: "ts", cutoverDelay: "2s" },
+  build: false,
+  path: dir,
+  output: "events-v2.json",
+  stack: MARKER.stack,
+  ownershipEnv: MARKER.env,
+  gates: "outer",
+  onFailure: "keep",
+});
+
+async function runOp(extra: Partial<ClickHouseRebuildDeps> = {}): Promise<OpRunResult> {
+  const props = (ClickHouseRebuildOp(opConfig()).op as unknown as { props: OpConfig }).props;
+  const activities = new Map<string, ActivityFn>();
+  for (const [name, fn] of Object.entries(rebuildActivities)) {
+    if (typeof fn !== "function" || !name.startsWith("clickhouseRebuild")) continue;
+    activities.set(name, ((args: Record<string, unknown>, signal?: AbortSignal) => (fn as (a: unknown, s?: AbortSignal, d?: unknown) => Promise<unknown>)(args, signal, deps(extra))) as ActivityFn);
+  }
+  // One attempt per step, so the failure below fails the run where it happens.
+  const profiles = Object.fromEntries(Object.entries(await loadProfiles()).map(([k, v]) => [k, { ...v, retry: { maximumAttempts: 1 } }]));
+  return runOpLocally(props, activities, profiles, undefined, { gates: memoryGateLedgerPort(), now: new Date().toISOString() });
+}
+
+const outcome = (r: OpRunResult, name: string) => r.records.flatMap((x) => x.outcomes ?? []).find((o) => o.name === name)?.value;
+
+describe.skipIf(!enabled)("a rebuild on a cluster of two shards (#3663)", () => {
+  test("every shard's rows survive the rebuild, and a rerun after a failure on shard 2 skips what was copied", async () => {
+    const applied = normalizeApply(toApplyResult(await clickhouseApply({ buildPath: join(dir, writeBuild("events-v1.json", "(ts, user_id)")), environment: "e2e" }, undefined, deps())));
+    expect(applied.notAttempted).toEqual([]);
+    // Each shard's own rows, three days of them, written on that shard.
+    const [s1] = servers("s1");
+    const [s2] = servers("s2");
+    await q(s1!, "INSERT INTO shop.events SELECT toDateTime('2026-02-01 00:00:00') + (number % 3) * 86400 + number, number, 'view' FROM numbers(30)");
+    await q(s2!, "INSERT INTO shop.events SELECT toDateTime('2026-02-01 00:00:00') + (number % 3) * 86400 + number, 1000 + number, 'click' FROM numbers(24)");
+    expect(await count(s1!, "shop.events_all")).toBe(54);
+    writeBuild("events-v2.json", "(user_id, ts)");
+
+    // Shard 2's first partition is copied and its receipt written, then the backfill fails.
+    const failure = await runOp({
+      backfill: {
+        afterPartition: (_p, shard) => {
+          if (shard === 2) throw new Error("the backfill failed on shard 2");
+        },
+      },
+    }).then(
+      () => undefined,
+      (e: unknown) => e,
+    );
+    expect(failure).toBeInstanceOf(OpRunFailure);
+    expect((failure as OpRunFailure).result.records.find((x) => x.fn === "clickhouseRebuildBackfill")?.error).toMatch(/failed on shard 2/);
+    const receipts = async () =>
+      (await q<{ address: string }>(s1!, "SELECT DISTINCT address FROM chant_receipts.receipts WHERE address LIKE 'e2e/test/rebuild/shop.events/%' ORDER BY address")).map((r) => r.address);
+    expect(await receipts()).toEqual([
+      "e2e/test/rebuild/shop.events/shard1/20260201",
+      "e2e/test/rebuild/shop.events/shard1/20260202",
+      "e2e/test/rebuild/shop.events/shard1/20260203",
+      "e2e/test/rebuild/shop.events/shard2/20260201",
+    ]);
+    // Each shard's new table holds that shard's copied rows, and nothing of the other's.
+    expect(await count(s1!, "shop.events__chant_new")).toBe(30);
+    expect(await count(s2!, "shop.events__chant_new")).toBe(8);
+
+    // The rerun resumes: four (shard, partition) copies skipped, two copied, every row verified, swapped.
+    const r = await runOp();
+    expect(r.status).toBe("ok");
+    expect(outcome(r, "Skipped")).toBe(4);
+    expect(outcome(r, "Copied")).toBe(2);
+    expect(outcome(r, "VerifiedRows")).toBe(54);
+    expect(outcome(r, "VerifiedPartitions")).toBe(6);
+
+    for (const [shard, rows] of [["s1", 30], ["s2", 24]] as const) {
+      for (const server of servers(shard)) {
+        await q(server, "SYSTEM SYNC REPLICA shop.events");
+        expect(await count(server, "shop.events")).toBe(rows);
+        expect((await q<{ statement: string }>(server, "SHOW CREATE TABLE shop.events"))[0]!.statement).toMatch(/ORDER BY \(user_id, ts\)/);
+        expect(await count(server, "shop.events__chant_old")).toBe(rows);
+      }
+    }
+    expect(await count(s1!, "shop.events_all")).toBe(54);
+  }, 600_000);
+});

package/src/clickhouse/rebuild/rebuild.e2e.test.ts

@@ -7,6 +7,13 @@
  *   table, keeps the old table, and drops it at the second gate.
  * - A failure: rows that land in the old table after the backfill fail the
  *   verification, and onFailure drops the new table.
+ * - Rows dated after the cut-over (sql-yodeler#45): the ones the table held
+ *   before the view are copied by the backfill, the ones written during the
+ *   rebuild arrive through the view, and the swap leaves none behind.
+ * - Under an outer approval with the new table kept on failure (#3658): a
+ *   backfill that fails part way leaves the new table and its receipts, and
+ *   the next run skips the partitions already copied, verifies and swaps
+ *   with no gate, and keeps the old table.
  * - An interrupted backfill: stopped after three partitions, then killed
  *   between a partition's INSERT and its receipt, then resumed: the receipts
  *   skip what was copied, the half-copied partition is cleared first, and the
@@ -28,6 +35,7 @@ import { clickhouseImage } from "../../spec/pin";
 import { planAgainstServer } from "../plan/commands";
 import { clickhouseApply, toApplyResult } from "../../op/activities/clickhouse-apply";
 import * as rebuildActivities from "../../op/activities/clickhouse-rebuild";
+import type { ClickHouseRebuildDeps } from "../../op/activities/clickhouse-rebuild";
 import { ClickHouseRebuildOp, type ClickHouseRebuildOpConfig } from "./op";
 import { clickhouseReceiptStore, RECEIPTS_DATABASE, RECEIPTS_TABLE, type ClickHouseReceiptStore } from "./receipts";
 
@@ -65,20 +73,20 @@ const exists = async (database: string, name: string) => Number((await q<{ n: st
 const showCreate = async (name: string) => (await q<{ statement: string }>(`SHOW CREATE TABLE ${name}`))[0]!.statement;
 
 /** The rebuild activities, with the test's config, server and a quiet log, as the local executor resolves them. */
-function activities(): Map<string, ActivityFn> {
+function activities(extra: Partial<ClickHouseRebuildDeps> = {}): Map<string, ActivityFn> {
   const map = new Map<string, ActivityFn>();
   for (const [name, fn] of Object.entries(rebuildActivities)) {
     if (typeof fn !== "function" || !name.startsWith("clickhouseRebuild")) continue;
     map.set(name, ((args: Record<string, unknown>, signal?: AbortSignal) =>
-      (fn as (a: unknown, s?: AbortSignal, d?: unknown) => Promise<unknown>)(args, signal, deps())) as ActivityFn);
+      (fn as (a: unknown, s?: AbortSignal, d?: unknown) => Promise<unknown>)(args, signal, { ...deps(), ...extra })) as ActivityFn);
   }
   return map;
 }
 
-async function runOp(config: ClickHouseRebuildOpConfig, gates: GateLedgerPort): Promise<OpRunResult> {
+async function runOp(config: ClickHouseRebuildOpConfig, gates: GateLedgerPort, extra: Partial<ClickHouseRebuildDeps> = {}): Promise<OpRunResult> {
   const { op } = ClickHouseRebuildOp(config);
   const props = (op as unknown as { props: OpConfig }).props;
-  return runOpLocally(props, activities(), await loadProfiles(), undefined, { gates, now: new Date().toISOString() });
+  return runOpLocally(props, activities(extra), await loadProfiles(), undefined, { gates, now: new Date().toISOString() });
 }
 
 /** An in-memory gate ledger a person approves on: each approval answers the gate the last run stopped at, for the plan it recorded. */
@@ -174,7 +182,7 @@ describe.skipIf(!enabled)("a sorting-key change, rebuilt through its gates", ()
     expect(outcome(r, "RebuildState")).toBe("rebuild");
     expect(outcome(r, "VerifiedRows")).toBe(1600);
     expect(outcome(r, "VerifiedPartitions")).toBe(4);
-    expect(String(outcome(r, "Verification"))).toMatch(/4 partition\(s\), 1600 row\(s\) before the cut-over/);
+    expect(String(outcome(r, "Verification"))).toMatch(/4 partition\(s\), 1600 row\(s\) \(1600 of them before the cut-over/);
     expect(await count("shop.events__chant_new")).toBe(1600);
     expect(await exists("shop", "events__chant_dual")).toBe(true);
     // Nothing visible to a plan or an apply: the working objects are left out.
@@ -235,6 +243,108 @@ describe.skipIf(!enabled)("a sorting-key change, rebuilt through its gates", ()
   }, 300_000);
 });
 
+describe.skipIf(!enabled)("rows dated after the cut-over that the table already held (sql-yodeler#45)", () => {
+  const bookingsDdl = (orderBy: string) => `CREATE TABLE shop.bookings
+(
+  id UInt64,
+  kind LowCardinality(String),
+  at DateTime
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(at)
+ORDER BY ${orderBy}`;
+  const V1: Obj = { export: "bookings", type: "ClickHouse::Table", dependsOn: ["shop"], ddl: bookingsDdl("(kind, at)") };
+  const V2: Obj = { ...V1, ddl: bookingsDdl("(kind, id)") };
+  const config = (): ClickHouseRebuildOpConfig => ({
+    name: "rebuild-bookings",
+    env: "e2e",
+    table: "shop.bookings",
+    dualWrite: { mode: "materialized-view", cutoverColumn: "at", cutoverDelay: "2s" },
+    build: false,
+    path: dir,
+    output: "bookings-v2.json",
+    retain: "0s",
+    stack: MARKER.stack,
+    ownershipEnv: MARKER.env,
+  });
+
+  test("every row is copied and compared, those written during the rebuild too, and the swap leaves none behind", async () => {
+    await clickhouseApply({ buildPath: join(dir, writeBuild("bookings-v1.json", [DB, V1])), environment: "e2e" }, undefined, deps());
+    // 600 rows seven hours apart, from four months ago to about two months
+    // ahead, with an identical pair among the future ones.
+    await q(
+      "INSERT INTO shop.bookings SELECT number, ['view','cart','buy'][number % 3 + 1], toStartOfHour(now()) - toIntervalDay(120) + toIntervalHour(number * 7) FROM numbers(600)",
+    );
+    await q("INSERT INTO shop.bookings VALUES (599, 'buy', toStartOfHour(now()) - toIntervalDay(120) + toIntervalHour(599 * 7))");
+    const future = Number((await q<{ n: string }>("SELECT count() AS n FROM shop.bookings WHERE at > now() + 60"))[0]!.n);
+    expect(future).toBeGreaterThan(150);
+    writeBuild("bookings-v2.json", [DB, V2]);
+
+    const ledger = new Ledger();
+    const first = await runOp(config(), ledger.port);
+    expect(first.status).toBe("gated");
+    expect(first.gate).toMatchObject({ gate: "approve-rebuild-bookings" });
+    expect(outcome(first, "VerifiedRows")).toBe(601);
+    expect(await count("shop.bookings__chant_new")).toBe(601);
+
+    // Written during the rebuild, after the cut-over: through the view.
+    await q("INSERT INTO shop.bookings VALUES (1000, 'buy', now() + toIntervalDay(40)), (1001, 'cart', now())");
+    const again = await runOp(config(), ledger.port);
+    expect(again.status).toBe("gated");
+    expect(outcome(again, "VerifiedRows")).toBe(603);
+    // The rows before the cut-over are what the approval binds; later rows do not change it.
+    expect(again.gate!.planDigest).toBe(first.gate!.planDigest);
+
+    ledger.approveLast();
+    await q("INSERT INTO shop.bookings VALUES (1002, 'view', now() + toIntervalDay(70))");
+    const swapped = await runOp(config(), ledger.port);
+    expect(swapped.status).toBe("gated");
+    expect(swapped.gate).toMatchObject({ gate: "approve-rebuild-bookings-drop" });
+    expect(await showCreate("shop.bookings")).toMatch(/ORDER BY \(kind, id\)/);
+    expect(await count("shop.bookings")).toBe(604);
+    expect(await count("shop.bookings__chant_old")).toBe(604);
+    const diff = await q("SELECT * FROM shop.bookings__chant_old EXCEPT ALL SELECT * FROM shop.bookings");
+    expect(diff).toEqual([]);
+  }, 300_000);
+
+  test("a row that reaches the old table alone after the verification stops the swap", async () => {
+    const clicksDdl = (orderBy: string) => `CREATE TABLE shop.visits
+(
+  id UInt64,
+  at DateTime
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(at)
+ORDER BY ${orderBy}`;
+    const W1: Obj = { export: "visits", type: "ClickHouse::Table", dependsOn: ["shop"], ddl: clicksDdl("at") };
+    const W2: Obj = { ...W1, ddl: clicksDdl("(id, at)") };
+    await clickhouseApply({ buildPath: join(dir, writeBuild("visits-v1.json", [DB, W1])), environment: "e2e" }, undefined, deps());
+    await q("INSERT INTO shop.visits SELECT number, now() - toIntervalDay(number % 90) FROM numbers(300)");
+    writeBuild("visits-v2.json", [DB, W2]);
+    const args = {
+      table: "shop.visits",
+      buildPath: "visits-v2.json",
+      environment: "e2e",
+      dualWrite: { mode: "materialized-view" as const, cutoverColumn: "at", cutoverDelay: "2s" },
+      stack: MARKER.stack,
+      ownershipEnv: MARKER.env,
+      cwd: dir,
+    };
+    await rebuildActivities.clickhouseRebuildCreate(args, undefined, deps());
+    await rebuildActivities.clickhouseRebuildDualWrite(args, undefined, deps());
+    await rebuildActivities.clickhouseRebuildBackfill(args, undefined, deps());
+    expect(await rebuildActivities.clickhouseRebuildVerify(args, undefined, deps())).toMatchObject({ rows: 300 });
+
+    // Late: a time well before the cut-over, written after the verification.
+    await q("INSERT INTO shop.visits VALUES (9999, now() - toIntervalDay(30))");
+    await expect(rebuildActivities.clickhouseRebuildSwap(args, undefined, deps())).rejects.toThrow(/does not match the old one in 1 partition/);
+    expect(await showCreate("shop.visits")).toMatch(/ORDER BY at/);
+    expect(await count("shop.visits")).toBe(301);
+    expect(await count("shop.visits__chant_new")).toBe(300);
+    await rebuildActivities.clickhouseRebuildCompensate(args, undefined, deps());
+  }, 300_000);
+});
+
 describe.skipIf(!enabled)("a failure runs onFailure, which drops the new table", () => {
   const clicksDdl = (orderBy: string) => `CREATE TABLE shop.clicks (id UInt64, url String) ENGINE = MergeTree ORDER BY ${orderBy}`;
   const V1: Obj = { export: "clicks", type: "ClickHouse::Table", dependsOn: ["shop"], ddl: clicksDdl("id") };
@@ -349,3 +459,104 @@ describe.skipIf(!enabled)("an interrupted backfill resumes from its receipts", (
     expect(dropped.dropped).toEqual(["shop.page_views__chant_new"]);
   }, 300_000);
 });
+
+describe.skipIf(!enabled)("under an outer approval, a failed rebuild keeps its new table and the rerun resumes (#3658)", () => {
+  const sessionsDdl = (orderBy: string) => `CREATE TABLE shop.sessions
+(
+  day Date,
+  user_id UInt64,
+  ts DateTime
+)
+ENGINE = MergeTree
+PARTITION BY day
+ORDER BY ${orderBy}`;
+  const V1: Obj = { export: "sessions", type: "ClickHouse::Table", dependsOn: ["shop"], ddl: sessionsDdl("(day, user_id)") };
+  const V2: Obj = { ...V1, ddl: sessionsDdl("(user_id, day)") };
+  const config = (): ClickHouseRebuildOpConfig => ({
+    name: "rebuild-sessions",
+    env: "e2e",
+    table: "shop.sessions",
+    dualWrite: { mode: "materialized-view", cutoverColumn: "ts", cutoverDelay: "2s" },
+    build: false,
+    path: dir,
+    output: "sessions-v2.json",
+    stack: MARKER.stack,
+    ownershipEnv: MARKER.env,
+    gates: "outer",
+    onFailure: "keep",
+  });
+  const receipts = async () =>
+    (await q<{ address: string }>(`SELECT DISTINCT address FROM ${RECEIPTS_DATABASE}.${RECEIPTS_TABLE} WHERE address LIKE 'e2e/test/rebuild/shop.sessions/%'`)).length;
+
+  test("the failed run keeps the new table and its receipts; the rerun skips what was copied, swaps without a gate and keeps the old table", async () => {
+    await clickhouseApply({ buildPath: join(dir, writeBuild("sessions-v1.json", [DB, V1])), environment: "e2e" }, undefined, deps());
+    // Six days of rows, a partition each, all well before any cut-over.
+    await q("INSERT INTO shop.sessions SELECT toDate('2026-02-01') + number % 6, number, toDateTime('2026-02-01 00:00:00') + (number % 6) * 86400 FROM numbers(1200)");
+    writeBuild("sessions-v2.json", [DB, V2]);
+
+    // Every attempt of the backfill step fails after its partition: three attempts, three partitions copied, then the run fails.
+    let copied = 0;
+    const ledger = new Ledger();
+    const failure = await runOp(config(), ledger.port, {
+      backfill: {
+        afterPartition: () => {
+          copied++;
+          throw new Error(`the backfill failed after ${copied} partition(s)`);
+        },
+      },
+    }).then(
+      () => undefined,
+      (e: unknown) => e,
+    );
+    expect(failure).toBeInstanceOf(OpRunFailure);
+    const records = (failure as OpRunFailure).result.records;
+    expect(records.find((x) => x.fn === "clickhouseRebuildBackfill")?.error).toMatch(/the backfill failed after 3 partition/);
+    // No onFailure ran: the new table, the dual-write view and the receipts are there.
+    expect(records.find((x) => x.fn === "clickhouseRebuildCompensate")).toBeUndefined();
+    expect(await exists("shop", "sessions__chant_new")).toBe(true);
+    expect(await exists("shop", "sessions__chant_dual")).toBe(true);
+    expect(await count("shop.sessions__chant_new")).toBe(600);
+    expect(await receipts()).toBe(3);
+    expect(await showCreate("shop.sessions")).toMatch(/ORDER BY \(day, user_id\)/);
+
+    // The rerun resumes: three partitions skipped by receipt, three copied, verified, swapped, with no gate.
+    const r = await runOp(config(), ledger.port);
+    expect(r.status).toBe("ok");
+    expect(ledger.port.appended).toEqual([]);
+    expect(outcome(r, "Skipped")).toBe(3);
+    expect(outcome(r, "Copied")).toBe(3);
+    expect(outcome(r, "VerifiedRows")).toBe(1200);
+    expect(r.records.map((x) => x.fn)).not.toContain("clickhouseRebuildDrop");
+    expect(await showCreate("shop.sessions")).toMatch(/ORDER BY \(user_id, day\)/);
+    expect(await count("shop.sessions")).toBe(1200);
+    // The old table is kept: dropping it is a run with the Op's own gates.
+    expect(await count("shop.sessions__chant_old")).toBe(1200);
+    expect(await exists("shop", "sessions__chant_new")).toBe(false);
+    expect(await receipts()).toBe(6);
+
+    // Run with its own gates, the Op has nothing left to swap, and its drop gate binds that old table.
+    const ownLedger = new Ledger();
+    const own = await runOp({ ...config(), gates: "own" }, ownLedger.port);
+    expect(own.status).toBe("gated");
+    expect(own.gate).toMatchObject({ gate: "approve-rebuild-sessions" });
+    expect(outcome(own, "RebuildState")).toBe("swapped");
+    ownLedger.approveLast();
+    const drop = await runOp({ ...config(), gates: "own" }, ownLedger.port);
+    expect(drop.status).toBe("gated");
+    expect(drop.gate).toMatchObject({ gate: "approve-rebuild-sessions-drop" });
+    expect(await count("shop.sessions__chant_old")).toBe(1200);
+  }, 300_000);
+
+  test('a refusal says to drop the kept table by hand', async () => {
+    // A new table made from another declaration: refused, and nothing will drop it.
+    writeBuild("sessions-v3.json", [DB, { ...V1, ddl: sessionsDdl("(ts, user_id)") }]);
+    await q("CREATE TABLE shop.sessions__chant_new AS shop.sessions");
+    await q(`ALTER TABLE shop.sessions__chant_new MODIFY COMMENT 'x [chant managed-by=chant stack=e2e env=test rebuild=shop.sessions role=new]'`);
+    const args = { table: "shop.sessions", buildPath: "sessions-v3.json", environment: "e2e", dualWrite: { mode: "app" as const }, stack: MARKER.stack, ownershipEnv: MARKER.env, cwd: dir };
+    await expect(rebuildActivities.clickhouseRebuildCreate({ ...args, keepOnFailure: true }, undefined, deps())).rejects.toThrow(
+      /keeps its new table on failure \(onFailure: "keep"\), so nothing drops it: drop shop\.sessions__chant_new/,
+    );
+    await expect(rebuildActivities.clickhouseRebuildCreate(args, undefined, deps())).rejects.not.toThrow(/onFailure: "keep"/);
+    await q("DROP TABLE shop.sessions__chant_new SYNC");
+  }, 300_000);
+});