@intentius/chant-lexicon-sql 0.114.1 → 0.116.0

package/src/clickhouse/rebuild/shards.ts

@@ -0,0 +1,104 @@
+/**
+ * A rebuild across the shards of a cluster (#3663).
+ *
+ * On a `cluster:<name>` topology the rebuild's DDL runs `ON CLUSTER`, so
+ * every server gets the new table, the dual-write view and the swap. The
+ * rows do not move that way: each shard holds its own, and a copy run on
+ * the profile's server alone copies that server's shard and no other. So
+ * with more than one shard, the backfill and the verification work shard by
+ * shard, all through the profile's server:
+ *
+ * - a shard's rows are read with `cluster('<name>', db, t)` and
+ *   `_shard_num = <n>`, one replica per shard, as the cluster's own
+ *   configuration reaches it;
+ * - they are written back to the same shard with
+ *   `INSERT INTO FUNCTION cluster('<name>', db, t__chant_new, <slot>)`, a
+ *   constant sharding key that lands on that shard, inserted in the
+ *   foreground so the statement returns once the shard has the rows;
+ * - a shard's half-copied rows are deleted with an `ALTER ... ON CLUSTER`
+ *   whose condition holds on that shard's servers only
+ *   (`getMacro('shard')`, the macro the cluster's Keeper paths already use).
+ *
+ * Each (shard, partition) is one copy with its own receipt, so a rerun after
+ * a failure on one shard skips what every shard already copied.
+ *
+ * With one shard nothing changes: the replicas of a shard replicate what is
+ * written on any of them, which is what the rebuild has always relied on.
+ */
+
+import { clickhouseQuery } from "../http";
+import type { ClickHouseTarget } from "../live/bind";
+import { ident, sqlString } from "../apply/statements";
+import { RebuildRefusal } from "./observe";
+
+export interface Shard {
+  /** `_shard_num` / `system.clusters.shard_num`, from 1. */
+  num: number;
+  /** The shard's `{shard}` macro, the same on each of its replicas. */
+  macro: string;
+  /** The sharding key that lands on this shard: the sum of the weights of the shards before it. */
+  slot: number;
+}
+
+export interface Sharding {
+  cluster: string;
+  shards: Shard[];
+}
+
+/** Settings for a copy into one shard: in the foreground, and read from the cluster rather than pushed down to each shard. */
+export const SHARD_INSERT_SETTINGS: Record<string, string> = { distributed_foreground_insert: "1", parallel_distributed_insert_select: "0" };
+
+/** Settings for a delete on one shard: its condition is the server's own macro, the same on every replica of the shard. */
+export const SHARD_DELETE_SETTINGS: Record<string, string> = { allow_nondeterministic_mutations: "1", mutations_sync: "2" };
+
+/**
+ * The shards of the target's cluster, when its topology is `cluster:<name>`
+ * and the cluster has more than one. Undefined otherwise: the rebuild works
+ * on the profile's server as it always has.
+ */
+export async function clusterSharding(target: ClickHouseTarget): Promise<Sharding | undefined> {
+  if (target.topology?.kind !== "cluster") return undefined;
+  let cluster = target.topology.cluster;
+  const macro = /^\{(\w+)\}$/.exec(cluster);
+  if (macro) {
+    const [row] = await clickhouseQuery<{ c: string }>(target.endpoint, `SELECT getMacro(${sqlString(macro[1]!)}) AS c`);
+    cluster = row?.c ?? cluster;
+  }
+  const weights = await clickhouseQuery<{ shard: number | string; weight: number | string }>(
+    target.endpoint,
+    `SELECT shard_num AS shard, any(shard_weight) AS weight FROM system.clusters WHERE cluster = ${sqlString(cluster)} GROUP BY shard_num ORDER BY shard_num`,
+  );
+  if (weights.length <= 1) return undefined;
+  const macros = new Map(
+    (await clickhouseQuery<{ shard: number | string; macro: string }>(target.endpoint, `SELECT _shard_num AS shard, getMacro('shard') AS macro FROM cluster(${sqlString(cluster)}, system.one)`)).map(
+      (r) => [Number(r.shard), r.macro],
+    ),
+  );
+  let slot = 0;
+  const shards: Shard[] = [];
+  for (const w of weights) {
+    const num = Number(w.shard);
+    const m = macros.get(num);
+    if (m === undefined) throw new RebuildRefusal(`cluster ${cluster}: shard ${num} has no {shard} macro, which a rebuild across shards needs to delete a half-copied partition on that shard alone`);
+    shards.push({ num, macro: m, slot });
+    slot += Number(w.weight) || 1;
+  }
+  const distinct = new Set(shards.map((s) => s.macro));
+  if (distinct.size !== shards.length) {
+    throw new RebuildRefusal(`cluster ${cluster}: two shards share a {shard} macro (${shards.map((s) => `${s.num}=${s.macro}`).join(", ")}); each shard needs its own`);
+  }
+  return { cluster, shards };
+}
+
+/** One shard's rows of `database.table`, as a table expression; filter with {@link onShard}. */
+export const shardTable = (sharding: Sharding, database: string, table: string): string => `cluster(${sqlString(sharding.cluster)}, ${ident(database)}, ${ident(table)})`;
+
+/** The condition that keeps one shard's rows of a {@link shardTable}. */
+export const onShard = (shard: Shard): string => `_shard_num = ${shard.num}`;
+
+/** Where a copy into one shard of `database.table` writes. */
+export const intoShard = (sharding: Sharding, shard: Shard, database: string, table: string): string =>
+  `FUNCTION cluster(${sqlString(sharding.cluster)}, ${ident(database)}, ${ident(table)}, ${shard.slot})`;
+
+/** The condition, in an `ALTER ... ON CLUSTER` mutation, that holds on that shard's servers only. */
+export const onShardServers = (shard: Shard): string => `getMacro('shard') = ${sqlString(shard.macro)}`;

package/src/clickhouse/rebuild/steps.ts

@@ -17,6 +17,7 @@ import { waitForMutations } from "../apply/mutations";
 import { renderFor } from "../topology";
 import { DDL_SETTINGS, syncReplicaWithin } from "./replicas";
 import { assertSameRows } from "./verify";
+import type { Sharding } from "./shards";
 import {
   changesBetween,
   observeRebuild,
@@ -45,6 +46,8 @@ export interface RebuildRun {
   signal?: AbortSignal;
   /** The run's id, recorded on the receipts it writes. */
   runId?: string;
+  /** On a cluster of more than one shard, its shards: the backfill and the verification work shard by shard (`./shards.ts`). */
+  sharding?: Sharding;
 }
 
 /** The server's clock, in milliseconds: the cut-over and the retention are the server's times, not this machine's. */
@@ -76,6 +79,7 @@ const q = (run: RebuildRun, statement: string) => {
 
 export async function waitOn(run: RebuildRun, database: string, table: string): Promise<void> {
   const ids = await waitForMutations(run.target.endpoint, database, table, {
+    ...(run.sharding ? { source: `clusterAllReplicas(${sqlString(run.sharding.cluster)}, system.mutations)` } : {}),
     ...(run.mutationTimeoutMs !== undefined ? { timeoutMs: run.mutationTimeoutMs } : {}),
     ...(run.signal ? { signal: run.signal } : {}),
   });

package/src/clickhouse/rebuild/verify.ts

@@ -24,6 +24,10 @@
  * later than the cut-over delay, with a time before the cut-over) stop the
  * swap too, rather than stay behind in the old table.
  *
+ * On a cluster of more than one shard (#3663) every shard is read, through
+ * the profile's server, and each partition is compared per shard
+ * (`shard<n>/<partition>`), so a shard the copy missed is a difference.
+ *
  * Any difference fails the run, naming the partitions; the run's onFailure
  * then drops the new table. On a match the result carries the digest the
  * swap gate binds: the rebuild's plan and the counts and checksums of the
@@ -39,6 +43,7 @@ import { ident, sqlString } from "../apply/statements";
 import { rebuildPlanSubject, RebuildRefusal, type RebuildObservation } from "./observe";
 import { sourcePartitionExpression } from "./partitions";
 import { cutoverOf, observe, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
+import { shardTable } from "./shards";
 
 export interface PartitionCheck {
   partition: string;
@@ -120,11 +125,15 @@ async function compareTables(
   const newBefore = cut && cutover !== undefined ? `${ident(cut.name)} < ${utcLiteral(cutover)}` : "1";
   const fromOld = sourcePartitionExpression(o.live.partitionKey, o.newTable.partitionKey, o.copied);
 
-  const read = async (table: string, partition: string, hash: string, before: string): Promise<SideCounts> => {
+  // Across the shards of a cluster (#3663), every shard's rows, each partition counted per shard as `shard<n>/<partition>`.
+  const sharding = run.sharding;
+  const read = async (table: string, name: string, partition: string, hash: string, before: string): Promise<SideCounts> => {
+    const from = sharding ? shardTable(sharding, n.database, name) : table;
+    const key = sharding ? `concat('shard', toString(_shard_num), '/', ${partition})` : partition;
     const rows = await clickhouseQuery<{ p: string; rows: string | number; checksum: string; brows: string | number; bchecksum: string }>(
       run.target.endpoint,
-      `SELECT ${partition} AS p, count() AS rows, toString(sum(${hash})) AS checksum, countIf(${before}) AS brows, toString(sumIf(${hash}, ${before})) AS bchecksum ` +
-        `FROM ${table} GROUP BY p`,
+      `SELECT ${key} AS p, count() AS rows, toString(sum(${hash})) AS checksum, countIf(${before}) AS brows, toString(sumIf(${hash}, ${before})) AS bchecksum ` +
+        `FROM ${from} GROUP BY p`,
     );
     const side: SideCounts = { all: new Map(), before: new Map() };
     for (const r of rows) {
@@ -141,8 +150,8 @@ async function compareTables(
     await syncReplica(run, o, n.database, n.name);
     await syncReplica(run, o, n.database, n.newName);
     // The old table first: a row the view has yet to commit in the new one is read as missing there, never the other way round.
-    const before = await read(n.table, "_partition_id", oldHash, oldBefore);
-    const after = await read(n.newTable, fromOld, newHash, newBefore);
+    const before = await read(n.table, n.name, "_partition_id", oldHash, oldBefore);
+    const after = await read(n.newTable, n.newName, fromOld, newHash, newBefore);
     const mismatches = differences(before.all, after.all);
     if (mismatches.length === 0 || attempt >= attempts) return { old: before, new: after, mismatches };
     run.log(`-- ${n.key}: ${mismatches.length} partition(s) differ (${mismatches.map((m) => m.partition).slice(0, 5).join(", ")}); reading again (${attempt}/${attempts})`);

package/src/codegen/postgres-classifier-page.ts

@@ -70,7 +70,7 @@ Two rules change class across the supported majors: a STORED generated column's
 
 ## Identity and renames
 
-An object is identified by its export name between two builds, and by its schema-qualified name against a server, which has no export names. Tables, views, materialized views, sequences and indexes share one namespace per schema, types and domains another, and schemas and extensions are matched by name.
+An object is identified by its export name between two builds, and by its schema-qualified name against a server, which has no export names. Tables, views, materialized views, sequences and indexes share one namespace per schema, types and domains another, and schemas and extensions are matched by name. Functions and procedures share a namespace of their own and are told apart by their input parameter types, so an overload is another object; a trigger is matched by its name on its table.
 
 A rename is declared where it happens. Before a CREATE, \`-- previously: <old name>\` says the object had another name; on a column's line, it says the column did:
 

package/src/codegen/registry.ts

@@ -26,6 +26,9 @@ const KINDS: Record<string, string> = {
   PostgresEnum: POSTGRES_ENTITY_TYPES.enum,
   PostgresDomain: POSTGRES_ENTITY_TYPES.domain,
   PostgresExtension: POSTGRES_ENTITY_TYPES.extension,
+  PostgresFunction: POSTGRES_ENTITY_TYPES.function,
+  PostgresProcedure: POSTGRES_ENTITY_TYPES.procedure,
+  PostgresTrigger: POSTGRES_ENTITY_TYPES.trigger,
 };
 
 export function buildRegistry(): string {

package/src/config.ts

@@ -115,6 +115,13 @@ export const sqlProfileSchema = z.strictObject({
   schemas: z.array(z.string()).optional(),
   /** Postgres: the schema an unqualified declaration is created in. `public` when omitted. */
   defaultSchema: z.string().optional(),
+  /**
+   * Postgres: the schema `effect()` receipts are kept in, as
+   * `<schema>.__chant_receipts`. When omitted they are in
+   * `chant_receipts.receipts`, a schema chant creates; name an existing schema
+   * on a server where the role cannot `CREATE SCHEMA`.
+   */
+  receiptsSchema: z.string().optional(),
   /**
    * The managed service this environment's Postgres runs on, when it differs
    * from `sql.provider`. Import reads the provider's own roles, schemas and

package/src/core/receipts.ts

@@ -10,9 +10,11 @@
  *
  * A dialect's store implements this interface over its own client and keeps
  * the receipts in a table whose comment carries the ownership trailer with
- * the `receipts` key (`./ownership.ts`), so schema reads leave it out. It is
- * not registered as the global `receiptRead` / `receiptWrite` activities,
- * which would take over every other lexicon's `effect()` steps in the run.
+ * the `receipts` key (`./ownership.ts`), so schema reads leave it out. A
+ * migration's store is reached by its backfill directly. The store `effect()`
+ * steps use (`../receipts.ts`) is registered as the `receiptRead` /
+ * `receiptWrite` activities only as a fallback, so it never takes over
+ * another configured lexicon's receipt row.
  */
 
 import type { ReceiptStore } from "@intentius/chant/op/receipt-store";

package/src/generated/lexicon-sql.json

@@ -63,5 +63,20 @@
     "resourceType": "Postgres::Extension",
     "kind": "resource",
     "lexicon": "sql"
+  },
+  "PostgresFunction": {
+    "resourceType": "Postgres::Function",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "PostgresProcedure": {
+    "resourceType": "Postgres::Procedure",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "PostgresTrigger": {
+    "resourceType": "Postgres::Trigger",
+    "kind": "resource",
+    "lexicon": "sql"
   }
 }

package/src/index.ts

@@ -18,7 +18,7 @@ export { database, table, view, literal } from "./clickhouse/entities";
 // A tag folds as the function its file imports, so `table` imported from
 // `@intentius/chant-lexicon-sql/postgres` is Postgres's. Declarations import
 // them from the dialect subpath.
-export { schema, index, sequence, type, domain, extension } from "./postgres/entities";
+export { schema, index, sequence, type, domain, extension, func, procedure, trigger } from "./postgres/entities";
 
 // A schema change's statements between two builds, rendered offline for a
 // migration file (#3644); also at `@intentius/chant-lexicon-sql/migration-statements`.

package/src/lint/rules/postgres-templates.ts

@@ -17,12 +17,12 @@ export const POSTGRES_TEMPLATE_SOURCES: readonly TemplateSource<PostgresTag>[] =
   {
     dialect: "postgres",
     modules: ["@intentius/chant-lexicon-sql/postgres"],
-    tags: ["schema", "table", "index", "view", "sequence", "type", "domain", "extension"],
+    tags: ["schema", "table", "index", "view", "sequence", "type", "domain", "extension", "func", "procedure", "trigger"],
   },
   {
     dialect: "postgres",
     modules: ["@intentius/chant-lexicon-sql"],
-    tags: ["schema", "index", "sequence", "type", "domain", "extension"],
+    tags: ["schema", "index", "sequence", "type", "domain", "extension", "func", "procedure", "trigger"],
   },
 ];
 

package/src/lint/rules/sqlpg001.ts

@@ -13,6 +13,9 @@ const STATEMENT: Record<PostgresTag, string> = {
   type: "enum",
   domain: "domain",
   extension: "extension",
+  func: "function",
+  procedure: "procedure",
+  trigger: "trigger",
 };
 
 const TAG_OF: Record<string, PostgresTag> = Object.fromEntries(Object.entries(STATEMENT).map(([tag, statement]) => [statement, tag as PostgresTag]));

package/src/lsp/postgres-template.ts

@@ -149,6 +149,12 @@ export function postgresDeclarations(content: string, fileName = "file.ts"): Pos
         case "extension":
           sqlName = text(node.name.span);
           break;
+        case "function":
+        case "procedure":
+        case "trigger":
+          sqlName = text(node.name.span);
+          kind = node.statement;
+          break;
         default:
           continue;
       }

package/src/mcp/postgres.ts

@@ -81,7 +81,7 @@ export function postgresSearch(params: Record<string, unknown>): unknown {
   };
 }
 
-const TAGS = { schema: pg.schema, table: pg.table, index: pg.index, view: pg.view, sequence: pg.sequence, type: pg.type, domain: pg.domain, extension: pg.extension } as const;
+const TAGS = { schema: pg.schema, table: pg.table, index: pg.index, view: pg.view, sequence: pg.sequence, type: pg.type, domain: pg.domain, extension: pg.extension, func: pg.func, procedure: pg.procedure, trigger: pg.trigger } as const;
 export const POSTGRES_TAGS = Object.keys(TAGS);
 
 /** Parse one statement with the Postgres tag; an error comes back as SQLPG001 with its line and column. */

package/src/op/activities/clickhouse-rebuild.ts

@@ -12,9 +12,9 @@ import type { ChantConfig } from "@intentius/chant/config";
 import { currentOpRun, parseDuration } from "@intentius/chant/op";
 import { bindClickHouse } from "../../clickhouse/live/bind";
 import { declaredObjects } from "../../clickhouse/apply/apply";
-import { declaredTable, observeRebuild, rebuildPlanDigest } from "../../clickhouse/rebuild/observe";
+import { declaredTable, observeRebuild, rebuildPlanDigest, RebuildRefusal } from "../../clickhouse/rebuild/observe";
 import { backfill, type BackfillDeps, type BackfillResult } from "../../clickhouse/rebuild/backfill";
-import { verifyRebuild, type VerifyResult } from "../../clickhouse/rebuild/verify";
+import { RebuildVerificationError, verifyRebuild, type VerifyResult } from "../../clickhouse/rebuild/verify";
 import {
   compensate,
   createNewTable,
@@ -32,6 +32,7 @@ import {
 } from "../../clickhouse/rebuild/steps";
 import type { ClickHouseRebuildArgs } from "../../clickhouse/rebuild/op";
 import { CLASSIFIER_RULES } from "../../clickhouse/plan/rules";
+import { clusterSharding } from "../../clickhouse/rebuild/shards";
 import { resolveMarker } from "./clickhouse-apply";
 
 export type { ClickHouseRebuildArgs } from "../../clickhouse/rebuild/op";
@@ -61,6 +62,7 @@ export async function rebuildRun(args: ClickHouseRebuildArgs, signal: AbortSigna
   const target = await bindClickHouse({ ...(args.environment !== undefined ? { environment: args.environment } : {}), config: config ?? {}, ...(deps.env ? { env: deps.env } : {}) });
   const declared = declaredTable(declaredObjects(readFileSync(resolve(cwd, args.buildPath), "utf8"), target.defaultDatabase, target.topology), args.table);
   const runId = currentOpRun()?.runId;
+  const sharding = await clusterSharding(target);
   return {
     target,
     declared,
@@ -72,9 +74,28 @@ export async function rebuildRun(args: ClickHouseRebuildArgs, signal: AbortSigna
     log: deps.log ?? ((line: string) => console.log(line)),
     ...(signal ? { signal } : {}),
     ...(runId ? { runId } : {}),
+    ...(sharding ? { sharding } : {}),
   };
 }
 
+/**
+ * With `onFailure: "keep"` (#3658) no onFailure drops the new table, so a
+ * step that fails says what a person does instead: a refusal or a failed
+ * verification would otherwise fail every run after it the same way.
+ */
+async function noting<T>(args: ClickHouseRebuildArgs, fn: () => Promise<T>): Promise<T> {
+  try {
+    return await fn();
+  } catch (err) {
+    if (args.keepOnFailure && (err instanceof RebuildRefusal || err instanceof RebuildVerificationError)) {
+      err.message +=
+        ` This rebuild keeps its new table on failure (onFailure: "keep"), so nothing drops it: ` +
+        `drop ${args.table}__chant_new and ${args.table}__chant_dual by hand to start again from a new table.`;
+    }
+    throw err;
+  }
+}
+
 export interface RebuildPlanResult {
   state: "rebuild" | "swapped" | "done";
   table: string;
@@ -87,6 +108,10 @@ export interface RebuildPlanResult {
 
 /** Plan: classify the table's change against the server and say how far the rebuild has got. */
 export async function clickhouseRebuildPlan(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<RebuildPlanResult> {
+  return noting(args, () => plan(args, signal, deps));
+}
+
+async function plan(args: ClickHouseRebuildArgs, signal: AbortSignal | undefined, deps: ClickHouseRebuildDeps): Promise<RebuildPlanResult> {
   const run = await rebuildRun(args, signal, deps);
   const o = await observeRebuild(run.target, run.declared, run.marker);
   const changes = o.changes.map((c) => `${c.rule} ${c.field}${c.before !== undefined || c.after !== undefined ? `: ${c.before ?? "none"} -> ${c.after ?? "none"}` : ""}`);
@@ -103,23 +128,23 @@ export async function clickhouseRebuildPlan(args: ClickHouseRebuildArgs, signal?
 }
 
 export async function clickhouseRebuildCreate(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<CreateResult> {
-  return createNewTable(await rebuildRun(args, signal, deps));
+  return noting(args, async () => createNewTable(await rebuildRun(args, signal, deps)));
 }
 
 export async function clickhouseRebuildDualWrite(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<DualWriteResult> {
-  return startDualWrite(await rebuildRun(args, signal, deps));
+  return noting(args, async () => startDualWrite(await rebuildRun(args, signal, deps)));
 }
 
 export async function clickhouseRebuildBackfill(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<BackfillResult> {
-  return backfill(await rebuildRun(args, signal, deps), deps.backfill);
+  return noting(args, async () => backfill(await rebuildRun(args, signal, deps), deps.backfill));
 }
 
 export async function clickhouseRebuildVerify(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<VerifyResult> {
-  return verifyRebuild(await rebuildRun(args, signal, deps));
+  return noting(args, async () => verifyRebuild(await rebuildRun(args, signal, deps)));
 }
 
 export async function clickhouseRebuildSwap(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<SwapResult> {
-  return swapTables(await rebuildRun(args, signal, deps));
+  return noting(args, async () => swapTables(await rebuildRun(args, signal, deps)));
 }
 
 export async function clickhouseRebuildRetain(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<RetainResult> {

package/src/op/activities/index.ts

@@ -4,8 +4,12 @@
  * applier (`clickhouseApply`, #3208), the Postgres applier (`postgresApply`,
  * #3280) and their envelope projection (`toApplyResult`, one for both), which
  * core's apply activity looks up by name for the `clickhouse` and `postgres`
- * targets, the emulator lifecycles, and the steps of the two migration Ops.
+ * targets, the emulator lifecycles, the steps of the two migration Ops, and
+ * the receipt store and SQL step of an Op of `effect()` batches.
  */
+import type { ReceiptReadArgs, ReceiptStalenessArgs, ReceiptWriteArgs } from "@intentius/chant/op/receipt-store";
+import { sqlReceiptActivities } from "../../receipts";
+
 export { clickhouseApply, toApplyResult, resolveMarker } from "./clickhouse-apply";
 export type { ClickHouseApplyArgs, ClickHouseApplyDeps, ClickHouseApplyOutcome } from "./clickhouse-apply";
 export { clickhouseUp, clickhouseDown, CLICKHOUSE_EMULATOR, CLICKHOUSE_EMULATOR_SPEC } from "./clickhouse-emulator";
@@ -13,11 +17,24 @@ export { postgresApply } from "./postgres-apply";
 export type { PostgresApplyArgs, PostgresApplyDeps, PostgresApplyOutcome } from "./postgres-apply";
 export { postgresUp, postgresDown, POSTGRES_EMULATOR, POSTGRES_EMULATOR_SPEC } from "./postgres-emulator";
 
+// effect() steps (#3657): their receipts are kept on the environment's
+// database server (../../receipts.ts, `chant_receipts.receipts`), and
+// `sqlExec` runs a batch's SQL there. The receipt activities are fallbacks
+// (core's ACTIVITY_FALLBACKS): those names are global to a run, and a project
+// that also lists aws or k8s keeps its receipts in that lexicon's row.
+
+/** The activities this module provides only when no other configured lexicon does. */
+export const ACTIVITY_FALLBACKS: readonly string[] = ["receiptRead", "receiptWrite", "receiptStaleness"];
+// Bound per call: the environment (`chant run --env`) and the project are read when a step runs.
+export const receiptRead = (args: ReceiptReadArgs, signal?: AbortSignal) => sqlReceiptActivities().receiptRead(args, signal);
+export const receiptWrite = (args: ReceiptWriteArgs, signal?: AbortSignal) => sqlReceiptActivities().receiptWrite(args, signal);
+export const receiptStaleness = (args: ReceiptStalenessArgs, signal?: AbortSignal) => sqlReceiptActivities().receiptStaleness(args, signal);
+export { sqlExec } from "./sql-exec";
+export type { SqlExecArgs, SqlExecResult, SqlExecDeps } from "./sql-exec";
+
 // The rebuild migration (#3198): one activity per step of ClickHouseRebuildOp
 // (../../clickhouse/rebuild/op.ts). Its receipts are kept in ClickHouse and
-// reached by the backfill directly, so this module exports no receiptRead or
-// receiptWrite: those names are global to a run and belong to the receipt row
-// of a project's aws or k8s lexicon.
+// reached by the backfill directly, with the copy's own identity.
 export {
   clickhouseRebuildPlan,
   clickhouseRebuildCreate,

package/src/op/activities/sql-exec.ts

@@ -0,0 +1,84 @@
+/**
+ * `sqlExec` (#3657): run SQL on the environment's database server, the step
+ * an `effect()` batch of a sql project is made of.
+ *
+ *     effect(EffectReceipt("country-202601", { effect: "country/202601", flavor: "existence" }), [
+ *       activity("sqlExec", { sql: "ALTER TABLE shop.orders UPDATE country = 'NL' WHERE toYYYYMM(ts) = 202601" }, "atMostOnce"),
+ *     ])
+ *
+ * The server is the one `effect()`'s receipts are kept on
+ * (`../../receipts.ts`): `sql.profiles.<env>` for the environment
+ * `chant run --env` sets, else `CLICKHOUSE_URL` or `POSTGRES_URL`. `environment`
+ * in the arguments names another. On Postgres the SQL runs on a connection of
+ * its own with the role's default `search_path`, as one implicit transaction;
+ * on ClickHouse, `settings` are sent with the query.
+ */
+
+import type { ChantConfig } from "@intentius/chant/config";
+import { clickhouseQuery } from "../../clickhouse/http";
+import { bindClickHouse } from "../../clickhouse/live/bind";
+import { resolveBoundTarget } from "../../postgres/live/bind";
+import { connectPostgres } from "../../postgres/live/client";
+import { environmentOf, receiptDialect } from "../../receipts";
+
+export interface SqlExecArgs {
+  /** The SQL to run: one statement on ClickHouse; one or more on Postgres. */
+  sql: string;
+  /** The chant environment whose server it runs on. Default: `CHANT_ENV`, then a literal `ownership.env`. */
+  environment?: string;
+  /** ClickHouse: query settings (`mutations_sync`, `max_execution_time`). */
+  settings?: Record<string, string | number>;
+  /** The project directory chant.config.ts is read from. Default: the working directory. */
+  cwd?: string;
+}
+
+export interface SqlExecResult {
+  dialect: "clickhouse" | "postgres";
+  /** Where the server binding came from: `sql.profiles.prod` or `env CLICKHOUSE_URL`. */
+  source: string;
+  /** Rows the statement returned (a `SELECT`); 0 for one that returns none. */
+  rows: number;
+}
+
+/** What a test may inject instead of the project's config and the process env. */
+export interface SqlExecDeps {
+  config?: Pick<ChantConfig, "sql" | "ownership">;
+  env?: Record<string, string | undefined>;
+  log?: (line: string) => void;
+}
+
+async function loadConfig(cwd: string): Promise<Pick<ChantConfig, "sql" | "ownership"> | undefined> {
+  try {
+    const { loadChantConfig } = await import("@intentius/chant/config");
+    return (await loadChantConfig(cwd)).config;
+  } catch {
+    return undefined;
+  }
+}
+
+export async function sqlExec(args: SqlExecArgs, signal?: AbortSignal, deps: SqlExecDeps = {}): Promise<SqlExecResult> {
+  if (typeof args?.sql !== "string" || args.sql.trim() === "") throw new Error('sqlExec: needs { sql: "<statement>" }');
+  signal?.throwIfAborted();
+  const env = deps.env ?? process.env;
+  const config = deps.config ?? (await loadConfig(args.cwd ?? process.cwd()));
+  const environment = environmentOf(args, config, env);
+  const log = deps.log ?? ((line: string) => console.log(line));
+  const where = environment !== undefined ? { environment } : {};
+  log(args.sql);
+  if (receiptDialect(config, environment, env) === "postgres") {
+    const target = await resolveBoundTarget({ ...where, config: config ?? {}, env });
+    const client = await connectPostgres(target.endpoint, { applicationName: "chant sqlExec" });
+    try {
+      // The catalog reader's empty search_path is not what a hand-written statement expects.
+      await client.query("RESET search_path");
+      const rows = await client.query(args.sql);
+      return { dialect: "postgres", source: target.source, rows: Array.isArray(rows) ? rows.length : 0 };
+    } finally {
+      await client.end();
+    }
+  }
+  const target = await bindClickHouse({ ...where, config: config ?? {}, env });
+  const settings = args.settings ? Object.fromEntries(Object.entries(args.settings).map(([k, v]) => [k, String(v)])) : undefined;
+  const rows = await clickhouseQuery(target.endpoint, args.sql, { ...(settings ? { settings } : {}), ...(signal ? { signal } : {}) });
+  return { dialect: "clickhouse", source: target.source, rows: rows.length };
+}

package/src/op/activity-contracts.ts

@@ -34,6 +34,7 @@ const rebuildArgs = z.strictObject({
   stack: z.string().optional(),
   ownershipEnv: z.string().optional(),
   cwd: z.string().optional(),
+  keepOnFailure: z.boolean().optional(),
 });
 
 const state = z.enum(["rebuild", "swapped", "done"]);
@@ -208,3 +209,17 @@ export const postgresMigrationCompensateContract = activityContract(
   z.object({ dropped: z.array(z.string()) }),
   tableEntity,
 );
+
+// ── effect() batches (#3657) ───────────────────────────────────────────
+
+/** `SqlExecArgs` (`./activities/sql-exec.ts`): one batch's SQL on the environment's server. */
+export const sqlExecContract = activityContract(
+  "sqlExec",
+  z.strictObject({
+    sql: z.string().min(1),
+    environment: z.string().optional(),
+    settings: z.record(z.string(), z.union([z.string(), z.number()])).optional(),
+    cwd: z.string().optional(),
+  }),
+  z.object({ dialect: z.enum(["clickhouse", "postgres"]), source: z.string(), rows: z.number() }),
+);