@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/clickhouse/plan/rules.ts

@@ -0,0 +1,241 @@
+/**
+ * The change classifier's rules (#3046 question 3): every way a declared
+ * ClickHouse object can change, the class of change it is on the server, and
+ * the `ALTER` restriction behind the class, cited.
+ *
+ * Three classes:
+ *
+ * - `metadata`: the server records the change and touches no existing data
+ *   (a new column reads its default from old parts; a new index covers new
+ *   parts until `MATERIALIZE INDEX`).
+ * - `rewrite`: the change is a mutation that rewrites existing parts in the
+ *   background (`system.mutations`), with no rollback.
+ * - `rebuild`: ClickHouse cannot make the change to the table it has; the data
+ *   has to be copied into a new table. A plan refuses to make one in place.
+ *
+ * Creating and dropping whole objects are classed `create` and `drop`.
+ */
+
+import { classifierRule, type ChangeClasses, type ClassifierRule as SqlClassifierRule } from "../../core/classifier";
+
+export type ChangeClass = "metadata" | "rewrite" | "rebuild" | "create" | "drop";
+
+/** A ClickHouse rule: the shared core's rule shape over these classes, citing a clickhouse.com page. */
+export type ClassifierRule = SqlClassifierRule<ChangeClass>;
+
+/** The classes as data: report order and labels, and what each costs for `classifyDisruption()`. */
+export const CHANGE_CLASSES: ChangeClasses<ChangeClass> = {
+  order: ["create", "metadata", "rewrite", "rebuild", "drop"],
+  label: { create: "create", drop: "drop", metadata: "metadata only", rewrite: "background rewrite", rebuild: "REBUILD" },
+  disruption: { metadata: "in-place", rewrite: "rolling", rebuild: "replace", create: "in-place", drop: "destroy" },
+};
+
+const DOCS = "https://clickhouse.com/docs/sql-reference/statements";
+
+const rule = (id: string, cls: ChangeClass, title: string, restriction: string, cite: string): ClassifierRule => classifierRule(id, cls, title, restriction, cite);
+
+export const CLASSIFIER_RULES = {
+  SQLCH200: rule("SQLCH200", "create", "Create an object", "A new database, table or view is created; nothing existing changes.", `${DOCS}/create`),
+  SQLCH201: rule(
+    "SQLCH201",
+    "metadata",
+    "Add a column",
+    "ADD COLUMN only changes metadata: parts written before it read the column's default until they are merged or the column is materialized.",
+    `${DOCS}/alter/column#add-column`,
+  ),
+  SQLCH202: rule(
+    "SQLCH202",
+    "metadata",
+    "Drop a column",
+    "DROP COLUMN removes the column's data from every part; it is not undone. A column in a key cannot be dropped (SQLCH213 covers that case).",
+    `${DOCS}/alter/column#drop-column`,
+  ),
+  SQLCH203: rule("SQLCH203", "metadata", "Change a comment", "COMMENT COLUMN and MODIFY COMMENT change metadata only.", `${DOCS}/alter/comment`),
+  SQLCH204: rule(
+    "SQLCH204",
+    "metadata",
+    "Add, drop or change a skip index",
+    "ADD INDEX and DROP INDEX change metadata; a new index covers parts written after it until MATERIALIZE INDEX rebuilds it for older parts.",
+    `${DOCS}/alter/skipping-index`,
+  ),
+  SQLCH205: rule(
+    "SQLCH205",
+    "rewrite",
+    "Change a TTL",
+    "MODIFY TTL is recorded in metadata, and with materialize_ttl_after_modify (on by default) the server recalculates the TTL over existing data as a mutation.",
+    `${DOCS}/alter/ttl`,
+  ),
+  SQLCH206: rule(
+    "SQLCH206",
+    "metadata",
+    "Change a table setting",
+    "MODIFY SETTING and RESET SETTING change metadata, for settings the server lets change after creation.",
+    `${DOCS}/alter/setting`,
+  ),
+  SQLCH207: rule(
+    "SQLCH207",
+    "metadata",
+    "Change a column's default",
+    "MODIFY COLUMN with a new DEFAULT, MATERIALIZED or ALIAS expression changes metadata; values already written keep the old default until the column is materialized.",
+    `${DOCS}/alter/column#modify-column`,
+  ),
+  SQLCH208: rule(
+    "SQLCH208",
+    "metadata",
+    "Change a column's codec",
+    "MODIFY COLUMN ... CODEC applies to parts written afterwards; existing parts keep their codec until they are merged or rewritten.",
+    `${DOCS}/alter/column#modify-column`,
+  ),
+  SQLCH209: rule(
+    "SQLCH209",
+    "metadata",
+    "Move a column",
+    "MODIFY COLUMN ... FIRST | AFTER changes the column order in metadata only.",
+    `${DOCS}/alter/column#modify-column`,
+  ),
+  SQLCH210: rule(
+    "SQLCH210",
+    "rewrite",
+    "Change a column's type",
+    "Changing the type of a column outside the primary key is a mutation that rewrites the column in every part in the background, with no rollback.",
+    `${DOCS}/alter/column#modify-column`,
+  ),
+  SQLCH211: rule(
+    "SQLCH211",
+    "rebuild",
+    "Change the type of a key column",
+    "The type of a primary key column can change only when the change does not modify the data (for example adding values to an Enum); anything else needs a new table.",
+    `${DOCS}/alter/column#modify-column`,
+  ),
+  SQLCH212: rule(
+    "SQLCH212",
+    "metadata",
+    "Rename a column",
+    "RENAME COLUMN changes metadata, for a column that is not in a key.",
+    `${DOCS}/alter/column#rename-column`,
+  ),
+  SQLCH213: rule(
+    "SQLCH213",
+    "rebuild",
+    "Rename or drop a key column",
+    "A column in the sorting key, the primary key or the partition key cannot be renamed or dropped.",
+    `${DOCS}/alter/column#rename-column`,
+  ),
+  SQLCH214: rule(
+    "SQLCH214",
+    "metadata",
+    "Add, drop or change a projection",
+    "ADD PROJECTION and DROP PROJECTION change metadata; a new projection covers new parts until MATERIALIZE PROJECTION builds it for older ones.",
+    `${DOCS}/alter/projection`,
+  ),
+  SQLCH215: rule("SQLCH215", "metadata", "Add or drop a constraint", "ADD CONSTRAINT and DROP CONSTRAINT change metadata; existing rows are not checked.", `${DOCS}/alter/constraint`),
+  SQLCH216: rule(
+    "SQLCH216",
+    "metadata",
+    "Append new columns to the sorting key",
+    "MODIFY ORDER BY can only append columns added by ADD COLUMN in the same ALTER, and leaves the primary key as it was; the declaration must keep the old key as PRIMARY KEY.",
+    `${DOCS}/alter/order-by`,
+  ),
+  SQLCH217: rule(
+    "SQLCH217",
+    "metadata",
+    "Change the sampling key",
+    "MODIFY SAMPLE BY changes metadata; the new sampling expression must be part of the primary key, and the server refuses it otherwise.",
+    `${DOCS}/alter/sample-by`,
+  ),
+  SQLCH218: rule(
+    "SQLCH218",
+    "rebuild",
+    "Change a setting fixed at creation",
+    "A table setting the server marks read-only (such as index_granularity) is fixed when the table is created and needs a new table to change.",
+    `${DOCS}/alter/setting`,
+  ),
+  SQLCH220: rule(
+    "SQLCH220",
+    "rebuild",
+    "Change the sorting key",
+    "The sorting key is the on-disk order of every part (and in ReplacingMergeTree the deduplication identity); beyond appending new columns (SQLCH216) it cannot change in place.",
+    `${DOCS}/alter/order-by`,
+  ),
+  SQLCH221: rule(
+    "SQLCH221",
+    "rebuild",
+    "Change the primary key",
+    "The primary key cannot be changed by ALTER; MODIFY ORDER BY leaves it as it was.",
+    `${DOCS}/alter/order-by`,
+  ),
+  SQLCH222: rule(
+    "SQLCH222",
+    "rebuild",
+    "Change the partition key",
+    "There is no ALTER for the partition key; parts are laid out by it when they are written.",
+    `${DOCS}/alter`,
+  ),
+  SQLCH223: rule(
+    "SQLCH223",
+    "rebuild",
+    "Change the table engine or its arguments",
+    "There is no ALTER for a table's engine or its arguments (a ReplacingMergeTree version column, a Distributed sharding key); a different engine is a different table.",
+    `${DOCS}/alter`,
+  ),
+  SQLCH224: rule(
+    "SQLCH224",
+    "rebuild",
+    "Change what kind of object it is",
+    "A table cannot become a view or a view a table; the old object is dropped and the new one created.",
+    `${DOCS}/create`,
+  ),
+  SQLCH230: rule(
+    "SQLCH230",
+    "metadata",
+    "Rename or move a table or view",
+    "RENAME TABLE changes the name, or moves the object to another database, in metadata on an Atomic database.",
+    `${DOCS}/rename`,
+  ),
+  SQLCH231: rule("SQLCH231", "metadata", "Rename a database", "RENAME DATABASE is metadata only, for an Atomic database.", `${DOCS}/rename`),
+  SQLCH232: rule(
+    "SQLCH232",
+    "rebuild",
+    "Change a database's engine",
+    "A database's engine is fixed at creation; there is no ALTER for it.",
+    `${DOCS}/create/database`,
+  ),
+  SQLCH240: rule(
+    "SQLCH240",
+    "metadata",
+    "Change a view's query",
+    "A plain view stores no data; CREATE OR REPLACE VIEW replaces its query.",
+    `${DOCS}/create/view`,
+  ),
+  SQLCH241: rule(
+    "SQLCH241",
+    "metadata",
+    "Change a materialized view's query",
+    "ALTER TABLE ... MODIFY QUERY replaces the query without stopping inserts; rows already written are not recomputed.",
+    `${DOCS}/alter/view`,
+  ),
+  SQLCH242: rule(
+    "SQLCH242",
+    "rebuild",
+    "Change a materialized view's target",
+    "A materialized view's TO target is fixed at creation; the view is dropped and created again, and inserts into its source in between are not seen.",
+    `${DOCS}/create/view`,
+  ),
+  SQLCH243: rule(
+    "SQLCH243",
+    "rebuild",
+    "Change a materialized view's own storage",
+    "A materialized view without TO keeps its rows in an inner table whose engine and keys are fixed like any table's.",
+    `${DOCS}/create/view`,
+  ),
+  SQLCH244: rule(
+    "SQLCH244",
+    "metadata",
+    "Change a refreshable view's schedule",
+    "ALTER TABLE ... MODIFY REFRESH changes the schedule of a refreshable materialized view.",
+    `${DOCS}/alter/view`,
+  ),
+  SQLCH250: rule("SQLCH250", "drop", "Drop an object", "DROP removes the object and, for a table, its data; it is not undone.", `${DOCS}/drop`),
+} as const satisfies Record<string, ClassifierRule>;
+
+export type ClassifierRuleId = keyof typeof CLASSIFIER_RULES;

package/src/clickhouse/plan/schema.ts

@@ -0,0 +1,56 @@
+/**
+ * The two sides a diff compares: a build's output (the declarations), or what
+ * a live server reports.
+ */
+
+import { readFileSync } from "node:fs";
+import { canonicalObject } from "./normalize";
+import type { SchemaObject } from "./diff";
+import type { ClickHouseTarget } from "../live/bind";
+import { readLiveSchema } from "../live/catalog";
+
+interface OutputDoc {
+  dialect?: string;
+  objects?: Array<{ export: string; ddl: string }>;
+}
+
+/** The objects a `chant build` output holds, keyed by export name. */
+export function schemaFromBuildOutput(json: string, defaultDatabase = "default"): SchemaObject[] {
+  const doc = JSON.parse(json) as OutputDoc;
+  if (doc.dialect !== "clickhouse" || !Array.isArray(doc.objects)) {
+    throw new Error("not a sql lexicon build output: expected { dialect: \"clickhouse\", objects: [...] }");
+  }
+  return doc.objects.map((o) => ({ key: o.export, canonical: canonicalObject(o.ddl, defaultDatabase) }));
+}
+
+export function schemaFromBuildFile(path: string, defaultDatabase = "default"): SchemaObject[] {
+  return schemaFromBuildOutput(readFileSync(path, "utf-8"), defaultDatabase);
+}
+
+/**
+ * The declared objects re-keyed by `database.name`, the only identity a live
+ * server has. The export name is kept for the report.
+ */
+export function keyedByQualifiedName(objects: readonly SchemaObject[]): Array<SchemaObject & { exportName: string }> {
+  return objects.map((o) => ({
+    key: o.canonical.database ? `${o.canonical.database}.${o.canonical.name}` : o.canonical.name,
+    canonical: o.canonical,
+    exportName: o.key,
+  }));
+}
+
+/**
+ * What the server holds, keyed by `database.name`, limited to the databases
+ * the declarations use (and the target's scope). The `default` database is
+ * left out, as import leaves it out.
+ */
+export async function schemaFromServer(target: ClickHouseTarget, databases?: ReadonlySet<string>): Promise<SchemaObject[]> {
+  const live = await readLiveSchema(target);
+  return live
+    .filter((o) => !(o.type === "ClickHouse::Database" && o.name === "default"))
+    .filter((o) => !databases || databases.has(o.database ?? o.name))
+    .map((o) => ({
+      key: o.database ? `${o.database}.${o.name}` : o.name,
+      canonical: canonicalObject(o.statement, target.defaultDatabase),
+    }));
+}

package/src/clickhouse/plan/server-format.ts

@@ -0,0 +1,39 @@
+/**
+ * The backstop for normalization against a live server: for each changed
+ * expression the rules leave different, ask the server to format both sides
+ * (`formatQuerySingleLine`, which parses and prints and touches nothing) and
+ * drop the change when they print the same. `a+b*2` and `a + (b * 2)` are one
+ * expression to the server.
+ */
+
+import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
+import type { Change } from "./diff";
+
+const EXPRESSION_FIELDS = /^(orderBy|primaryKey|partitionBy|sampleBy|select|columns\.[^.]+\.(default|ttl)|indexes\..+|projections\..+)$/;
+
+const quote = (s: string) => `'${s.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+
+function asQuery(field: string, text: string): string {
+  if (field === "select") return text;
+  if (field.endsWith(".default")) return `SELECT ${text.replace(/^(DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL) /, "")}`;
+  return `SELECT ${text}`;
+}
+
+/** The changes that remain once the server's formatter has had its say. */
+export async function dropFormattingOnly(endpoint: ClickHouseEndpoint, changes: Change[]): Promise<Change[]> {
+  const candidates = changes.filter((c) => EXPRESSION_FIELDS.test(c.field) && c.before !== undefined && c.after !== undefined);
+  if (candidates.length === 0) return changes;
+  const same = new Set<Change>();
+  for (const c of candidates) {
+    try {
+      const [row] = await clickhouseQuery<{ a: string; b: string }>(
+        endpoint,
+        `SELECT formatQuerySingleLine(${quote(asQuery(c.field, c.before!))}) AS a, formatQuerySingleLine(${quote(asQuery(c.field, c.after!))}) AS b`,
+      );
+      if (row && row.a === row.b) same.add(c);
+    } catch {
+      // The server could not parse one side as a query: the rules' answer stands.
+    }
+  }
+  return changes.filter((c) => !same.has(c));
+}

package/src/clickhouse/rebuild/backfill.ts

@@ -0,0 +1,186 @@
+/**
+ * The backfill (#3198): copy the old table into the new one with
+ * `INSERT ... SELECT`, one partition at a time, each partition an effect
+ * with its own receipt.
+ *
+ * Per partition, the cycle core's `effect()` step runs
+ * (concepts/effect-receipts.mdx): read the receipt, skip the copy when it
+ * matches, otherwise copy and write the receipt last, on success only.
+ * `effect()` itself wraps steps declared at build time around one receipt
+ * declared at build time; the partitions are only known when the run reads
+ * the old table, so the cycle runs here, once per partition, over the same
+ * `ReceiptStore` seam (`./receipts.ts`).
+ *
+ * A receipt's expectation is a hash of what the copy is: the table, the new
+ * table's UUID, the partition, the columns copied and the cut-over. A new
+ * table made again, or a new cut-over, makes every receipt stale and the
+ * partitions are copied again into the new table.
+ *
+ * At-least-once is the receipt model's guarantee, and a partition copied
+ * twice would be in the new table twice, so a copy that may have run before
+ * is undone first. A partition with no matching receipt is a partition whose
+ * copy never finished as far as anything recorded; if the new table holds
+ * any of its rows (a run killed between the INSERT and the receipt), they are
+ * deleted, the delete is waited on in `system.mutations`, and the partition
+ * is copied once more. Each copy runs under a query id derived from the new
+ * table and the partition, and any query still running under it (the server
+ * goes on with an INSERT whose client went away) is killed before the
+ * partition is cleared. A backfill interrupted halfway therefore resumes at
+ * the first partition without a receipt, and no row is copied twice.
+ *
+ * In a Replicated database (#3249) a resumed backfill may run on another
+ * replica than the one the copy ran on. The kill is `KILL QUERY ON CLUSTER`
+ * the database's own cluster, so it reaches the replica still running the
+ * copy; before the old table's partitions are listed, and before the new
+ * table's rows are counted, this replica fetches what the others wrote
+ * (`SYSTEM SYNC REPLICA`); and the receipts replicate with the tables
+ * (`./receipts.ts`). Each copy runs with `insert_deduplicate = 0`: a
+ * replicated table remembers the blocks it was given, and a partition copied
+ * again after its rows were cleared would otherwise be dropped as a repeat.
+ *
+ * With a replica down (#3270) the kill goes on without it, and each sync
+ * either finds nothing that replica alone has and returns, or stops the step
+ * naming it (`./replicas.ts`).
+ */
+
+import { EffectReceipt, receiptExpectation } from "@intentius/chant/effect-receipt";
+import { clickhouseQuery } from "../http";
+import { ident, sqlString } from "../apply/statements";
+import { clickhouseReceiptStore, receiptAddress, type ClickHouseReceiptStore } from "./receipts";
+import { RebuildRefusal, type RebuildObservation } from "./observe";
+import { DDL_SETTINGS } from "./replicas";
+import { sourcePartitionExpression, sourcePartitions } from "./partitions";
+import { cutoverOf, observe, serverNow, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
+
+export interface BackfillResult {
+  state: RebuildObservation["state"];
+  /** Partitions of the old table. */
+  partitions: number;
+  /** Copied in this run. */
+  copied: number;
+  /** Skipped: their receipt matched. */
+  skipped: number;
+  /** Partitions this run found partly copied and cleared before copying again. */
+  cleared: number;
+}
+
+/** The effect a partition's copy is, and its receipt's address suffix. */
+export const partitionEffect = (key: string, partition: string): string => `rebuild/${key}/${partition}`;
+
+/** The receipt's expected value for one partition's copy. */
+export function partitionExpectation(o: RebuildObservation, partition: string, cutover: number | undefined): string {
+  const effect = partitionEffect(o.names.key, partition);
+  return receiptExpectation(
+    EffectReceipt(effect, {
+      effect,
+      flavor: "hash",
+      inputs: { table: o.names.key, newTable: o.newTable!.uuid, partition, copied: o.copied, cutover: cutover === undefined ? null : new Date(cutover).toISOString() },
+    }),
+  );
+}
+
+export interface BackfillDeps {
+  /** The receipt store. Default: the receipts table on the target server. */
+  receipts?: ClickHouseReceiptStore;
+  /** Called after each partition is copied and its receipt written; a test interrupts the backfill here. */
+  afterPartition?: (partition: string) => void | Promise<void>;
+}
+
+export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promise<BackfillResult> {
+  const o = await observe(run);
+  const result: BackfillResult = { state: o.state, partitions: 0, copied: 0, skipped: 0, cleared: 0 };
+  if (o.state !== "rebuild") return result;
+  if (!o.newTable) throw new RebuildRefusal(`${o.names.key}: the new table is not there yet; the Create phase makes it`);
+  const n = o.names;
+  const cutover = cutoverOf(run, o);
+
+  if (cutover !== undefined) {
+    // Rows before the cut-over are the backfill's; wait until the server's
+    // clock has passed it, so none of them is still to arrive.
+    for (;;) {
+      const now = await serverNow(run.target);
+      if (now > cutover) break;
+      run.log(`-- waiting ${Math.ceil((cutover - now) / 1000)}s for the cut-over at ${new Date(cutover).toISOString()}`);
+      await sleep(Math.min(cutover - now + 50, 10_000), run.signal);
+    }
+  }
+  await waitOn(run, n.database, n.name);
+  await syncReplica(run, o, n.database, n.name);
+
+  const cutColumn = run.dualWrite.mode === "materialized-view" ? o.copied.find((c) => c.name === (run.dualWrite as { cutoverColumn: string }).cutoverColumn) : undefined;
+  const sourceRange = cutColumn && cutover !== undefined ? ` AND ${ident(cutColumn.source)} < ${utcLiteral(cutover)}` : "";
+  const targetRange = cutColumn && cutover !== undefined ? ` AND ${ident(cutColumn.name)} < ${utcLiteral(cutover)}` : "";
+  const fromOld = sourcePartitionExpression(o.live.partitionKey, o.newTable.partitionKey, o.copied);
+  const columns = o.copied.map((c) => ident(c.name)).join(", ");
+  const select = o.copied.map((c) => ident(c.source)).join(", ");
+
+  const identity = { ...(run.marker?.stack ? { stack: run.marker.stack } : {}), ...(run.marker?.env ? { env: run.marker.env } : {}) };
+  const receipts =
+    deps.receipts ??
+    clickhouseReceiptStore(run.target.endpoint, identity, {
+      ...(run.runId ? { runId: run.runId } : {}),
+      ...(o.replicated ? { replicatedIn: n.database } : {}),
+      ...(run.replicaTimeoutMs !== undefined ? { replicaTimeoutMs: run.replicaTimeoutMs } : {}),
+    });
+  const recorded = await receipts.readAll(receiptAddress(identity, `rebuild/${n.key}/`));
+
+  const partitions = await sourcePartitions(run.target, n);
+  result.partitions = partitions.length;
+  for (const p of partitions) {
+    run.signal?.throwIfAborted();
+    const effect = partitionEffect(n.key, p);
+    const expectation = partitionExpectation(o, p, cutover);
+    if (recorded.get(receiptAddress(identity, effect)) === expectation) {
+      result.skipped++;
+      continue;
+    }
+
+    const queryId = `chant-rebuild-${o.newTable.uuid}-${p}`;
+    if (o.replicated) {
+      // On every replica that answers: the copy may be running on another one
+      // than this run's. A replica that is down is skipped once Keeper sees it
+      // inactive; nothing of its can be running a copy that still writes,
+      // and a part it wrote before going down is waited for by the sync below.
+      await clickhouseQuery(run.target.endpoint, `KILL QUERY ON CLUSTER ${sqlString(n.database)} WHERE query_id = ${sqlString(queryId)} SYNC`, {
+        settings: DDL_SETTINGS,
+      });
+      await syncReplica(run, o, n.database, n.newName);
+    } else {
+      await clickhouseQuery(run.target.endpoint, `KILL QUERY WHERE query_id = ${sqlString(queryId)} SYNC`);
+    }
+    const mine = `${fromOld} = ${sqlString(p)}${targetRange}`;
+    const [left] = await clickhouseQuery<{ n: string | number }>(run.target.endpoint, `SELECT count() AS n FROM ${n.newTable} WHERE ${mine}`);
+    if (Number(left?.n ?? 0) > 0) {
+      run.log(`-- partition ${p}: ${left!.n} row(s) from a copy that did not finish; deleting them before copying again`);
+      const sql = `ALTER TABLE ${n.newTable} DELETE WHERE ${mine}`;
+      run.log(sql);
+      await clickhouseQuery(run.target.endpoint, sql);
+      await waitOn(run, n.database, n.newName);
+      result.cleared++;
+    }
+
+    const sql = `INSERT INTO ${n.newTable} (${columns}) SELECT ${select} FROM ${n.table} WHERE _partition_id = ${sqlString(p)}${sourceRange}`;
+    run.log(sql);
+    await clickhouseQuery(run.target.endpoint, sql, { queryId, settings: { async_insert: "0", insert_deduplicate: "0" }, ...(run.signal ? { signal: run.signal } : {}) });
+    // The receipt, last, on success only.
+    await receipts.write({ name: effect, effect, flavor: "hash", inputs: {} }, expectation);
+    result.copied++;
+    await deps.afterPartition?.(p);
+  }
+  run.log(`-- backfill of ${n.key}: ${result.partitions} partition(s), ${result.copied} copied, ${result.skipped} already copied, ${result.cleared} cleared first`);
+  return result;
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const t = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(t);
+        reject(signal.reason);
+      },
+      { once: true },
+    );
+  });
+}