@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/clickhouse/rebuild/receipts.ts

@@ -0,0 +1,143 @@
+/**
+ * Effect receipts kept in ClickHouse (#3198): where the rebuild migration's
+ * backfill records which partitions it has copied.
+ *
+ * A receipt is the proof that one effect ran, kept where the effect happened
+ * (concepts/effect-receipts.mdx). The backfill's effect is "this partition of
+ * the old table is in the new table", and both tables are on the server, so
+ * the receipt is a row on the same server:
+ *
+ *     SELECT * FROM chant_receipts.receipts
+ *
+ * Why not git (core's lifecycle receipt store, `chant/lifecycle`): the
+ * receipt has to die with the data it witnesses. The rebuild's `onFailure`
+ * drops the new table; a receipt kept in git would go on saying partitions
+ * were copied into a table that no longer exists, and a resumed backfill
+ * would skip them. Here every receipt's expectation is bound to the new
+ * table's UUID, so a new table made again reads every old receipt as stale,
+ * and the receipts are read and written over the connection the copy already
+ * uses, with no commit and push per partition. Readable with nothing but a
+ * ClickHouse client, which is the walk-away property the other rows have.
+ *
+ * The store implements core's `ReceiptStore` seam
+ * (`@intentius/chant/op/receipt-store`), as the sql lexicon's shared
+ * `SqlReceiptStore` (`../../core/receipts.ts`). It is not exported as the
+ * `receiptRead` / `receiptWrite` activities: those names are global to a
+ * run, and a project that also configures aws or k8s would have its
+ * `effect()` steps read ClickHouse instead of SSM or a ConfigMap. The backfill
+ * activity reaches the store directly, the way the fly lexicon's release
+ * component reaches its own.
+ *
+ * Write discipline: the backfill writes a partition's receipt after the
+ * partition's copy succeeded, last, and nothing else writes one.
+ *
+ * In a Replicated database (#3249) the receipts sit beside the tables they
+ * witness, in `<db>.__chant_receipts`, a `ReplicatedReplacingMergeTree`: the
+ * database runs its CREATE on every replica and the table replicates its
+ * rows, so a receipt written by a copy on one replica is read by a resumed
+ * backfill on another. Each read first waits for this replica to fetch what
+ * the others wrote (`SYSTEM SYNC REPLICA ... LIGHTWEIGHT`). A separate
+ * `chant_receipts` database would exist on the one replica the backfill
+ * first ran on. With a replica down (#3270) the CREATE goes on without it,
+ * and a read that cannot fetch a receipt only that replica has stops the
+ * backfill, naming it (`./replicas.ts`).
+ */
+
+import { OWNERSHIP_MANAGED_BY_VALUE } from "@intentius/chant/ownership";
+import type { EffectReceiptRef } from "@intentius/chant/op/receipt-store";
+import { receiptAddress, type SqlReceiptStore } from "../../core/receipts";
+import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
+import { ident, sqlString } from "../apply/statements";
+import { CLICKHOUSE_COMMENT_OWNERSHIP_KEYS, RECEIPTS_TRAILER_KEY } from "../ownership";
+import { DDL_SETTINGS, syncReplicaWithin } from "./replicas";
+
+/** The database the receipts live in. Its comment marks it as chant's, so schema reads leave it out. */
+export const RECEIPTS_DATABASE = "chant_receipts";
+/** The table, one row per receipt write. */
+export const RECEIPTS_TABLE = "receipts";
+
+/** The table in a Replicated database that holds the receipts of the rebuilds in it. */
+export const REPLICATED_RECEIPTS_TABLE = "__chant_receipts";
+
+/** Where the receipts are kept: `chant_receipts.receipts`, or `<db>.__chant_receipts` for a Replicated database. */
+export function receiptsTable(replicatedIn?: string): { database: string; name: string } {
+  return replicatedIn ? { database: replicatedIn, name: REPLICATED_RECEIPTS_TABLE } : { database: RECEIPTS_DATABASE, name: RECEIPTS_TABLE };
+}
+
+const COMMENT = `chant effect receipts [chant ${CLICKHOUSE_COMMENT_OWNERSHIP_KEYS.managedBy}=${OWNERSHIP_MANAGED_BY_VALUE} ${RECEIPTS_TRAILER_KEY}=effects]`;
+/** Plain synchronous inserts: an async insert can be acknowledged before it is written. */
+const SYNC = { async_insert: "0" };
+
+export { receiptAddress } from "../../core/receipts";
+
+/** The shared core's receipt store, kept on a ClickHouse server. */
+export type ClickHouseReceiptStore = SqlReceiptStore;
+
+/**
+ * A receipt store on the server at `endpoint`. Creates the receipts database
+ * and table on first write if they are not there. `identity` is the
+ * project's ownership stack and env, which every address starts with.
+ * `replicatedIn` names a Replicated database to keep them in instead, and
+ * `replicaTimeoutMs` how long a read waits for this replica to catch up there.
+ */
+export function clickhouseReceiptStore(
+  endpoint: ClickHouseEndpoint,
+  identity: { stack?: string; env?: string },
+  opts: { runId?: string; replicatedIn?: string; replicaTimeoutMs?: number } = {},
+): ClickHouseReceiptStore {
+  const where = receiptsTable(opts.replicatedIn);
+  const TABLE = `${ident(where.database)}.${ident(where.name)}`;
+  let ensured = false;
+  const ensure = async () => {
+    if (ensured) return;
+    if (!opts.replicatedIn) await clickhouseQuery(endpoint, `CREATE DATABASE IF NOT EXISTS ${ident(RECEIPTS_DATABASE)} COMMENT ${sqlString(COMMENT)}`);
+    await clickhouseQuery(
+      endpoint,
+      `CREATE TABLE IF NOT EXISTS ${TABLE} (address String, effect String, expectation String, run_id String, written_at DateTime64(3) DEFAULT now64(3)) ` +
+        `ENGINE = ${opts.replicatedIn ? "ReplicatedReplacingMergeTree" : "ReplacingMergeTree"}(written_at) ORDER BY address COMMENT ${sqlString(COMMENT)}`,
+      { settings: DDL_SETTINGS },
+    );
+    ensured = true;
+  };
+  const exists = async () => {
+    const rows = await clickhouseQuery<{ n: number | string }>(
+      endpoint,
+      `SELECT count() AS n FROM system.tables WHERE database = ${sqlString(where.database)} AND name = ${sqlString(where.name)}`,
+    );
+    return Number(rows[0]?.n ?? 0) > 0;
+  };
+  /** Whether there is a table to read; in a Replicated database, after this replica has caught up with it. */
+  const readable = async () => {
+    if (!ensured && !(await exists())) return false;
+    if (opts.replicatedIn) await syncReplicaWithin(endpoint, where.database, where.name, opts.replicaTimeoutMs !== undefined ? { timeoutMs: opts.replicaTimeoutMs } : {});
+    return true;
+  };
+  const address = (receipt: EffectReceiptRef) => receiptAddress(identity, receipt.effect);
+
+  return {
+    async read(receipt) {
+      if (!(await readable())) return undefined;
+      const rows = await clickhouseQuery<{ expectation: string }>(
+        endpoint,
+        `SELECT expectation FROM ${TABLE} WHERE address = ${sqlString(address(receipt))} ORDER BY written_at DESC LIMIT 1`,
+      );
+      return rows[0]?.expectation;
+    },
+    async readAll(prefix) {
+      if (!(await readable())) return new Map();
+      const rows = await clickhouseQuery<{ address: string; expectation: string }>(
+        endpoint,
+        `SELECT address, argMax(expectation, written_at) AS expectation FROM ${TABLE} WHERE startsWith(address, ${sqlString(prefix)}) GROUP BY address`,
+      );
+      return new Map(rows.map((r) => [r.address, r.expectation]));
+    },
+    async write(receipt, expectation) {
+      await ensure();
+      await clickhouseQuery(
+        endpoint,
+        `INSERT INTO ${TABLE} (address, effect, expectation, run_id) VALUES (${sqlString(address(receipt))}, ${sqlString(receipt.effect)}, ${sqlString(expectation)}, ${sqlString(opts.runId ?? "")})`,
+        { settings: SYNC },
+      );
+    },
+  };
+}

package/src/clickhouse/rebuild/replicas.test.ts

@@ -0,0 +1,70 @@
+/**
+ * The bounded replica wait (#3270), against a stand-in for the server's HTTP
+ * interface: `SYSTEM SYNC REPLICA` times out the way the server does, and the
+ * queue holds a part from a replica that is down. The two-replica e2e
+ * (`rebuild-replicated.e2e.test.ts`) runs it against the pinned server.
+ */
+
+import { createServer, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { afterEach, describe, expect, test } from "vitest";
+import { DDL_SETTINGS, ReplicaFetchError, syncReplicaWithin } from "./replicas";
+
+let server: Server | undefined;
+afterEach(() => new Promise<void>((resolve) => (server ? server.close(() => resolve()) : resolve())));
+
+/** A server whose sync times out until `fetchedAfter` syncs, with one part from r2 in the queue until then. */
+async function standIn(fetchedAfter = Infinity): Promise<{ url: string; syncs: Array<string | null> }> {
+  const syncs: Array<string | null> = [];
+  server = createServer((req, res) => {
+    let body = "";
+    req.on("data", (c) => (body += c));
+    req.on("end", () => {
+      const params = new URL(req.url ?? "/", "http://x").searchParams;
+      const fetched = syncs.length >= fetchedAfter;
+      if (body.startsWith("SYSTEM SYNC REPLICA")) {
+        syncs.push(params.get("receive_timeout"));
+        if (syncs.length > fetchedAfter) return void res.end("");
+        res.statusCode = 408;
+        return void res.end("Code: 159. DB::Exception: SYNC REPLICA shop.events: command timed out. See the 'receive_timeout' setting. (TIMEOUT_EXCEEDED)");
+      }
+      if (body.includes("system.replicas")) return void res.end(JSON.stringify({ replica_name: "r1", replica_is_active: { r1: 1, r2: 0 } }) + "\n");
+      if (body.includes("system.replication_queue")) {
+        return void res.end(fetched ? "" : JSON.stringify({ new_part_name: "202602_3_3_0", source_replica: "r2", last_exception: "Code: 210. Connection refused\nmore" }) + "\n");
+      }
+      res.statusCode = 400;
+      res.end(`unexpected: ${body}`);
+    });
+  });
+  await new Promise<void>((resolve) => server!.listen(0, "127.0.0.1", () => resolve()));
+  return { url: `http://127.0.0.1:${(server!.address() as AddressInfo).port}`, syncs };
+}
+
+describe("a replica that is down", () => {
+  test("DDL goes on without the inactive replicas of a Replicated database", () => {
+    expect(DDL_SETTINGS).toEqual({ distributed_ddl_output_mode: "throw_only_active" });
+  });
+
+  test("a part on the down replica alone: the wait is bounded, and the error names the replica and the part", async () => {
+    const { url, syncs } = await standIn();
+    const err = await syncReplicaWithin({ url }, "shop", "events", { timeoutMs: 1500 }).then(
+      () => undefined,
+      (e: unknown) => e,
+    );
+    expect(err).toBeInstanceOf(ReplicaFetchError);
+    expect((err as ReplicaFetchError).missing).toEqual([{ part: "202602_3_3_0", source: "r2", sourceActive: false, lastError: "Code: 210. Connection refused" }]);
+    expect((err as Error).message).toMatch(
+      /^shop\.events: waited \ds for replica r1 to fetch what the other replicas wrote, and 1 part\(s\) are still to fetch: 202602_3_3_0 from r2 \(inactive\) \(last fetch error: Code: 210\. Connection refused\)\. Those rows are on r2 alone/,
+    );
+    // Each sync waits no longer than what is left, in whole seconds.
+    expect(syncs.every((s) => s === "1" || s === "2")).toBe(true);
+  });
+
+  test("fetched within the bound: the step goes on", async () => {
+    const { url, syncs } = await standIn(1);
+    const lines: string[] = [];
+    await syncReplicaWithin({ url }, "shop", "events", { timeoutMs: 30_000, log: (l) => lines.push(l) });
+    expect(syncs).toHaveLength(1);
+    expect(lines).toEqual(["SYSTEM SYNC REPLICA `shop`.`events` LIGHTWEIGHT"]);
+  });
+});

package/src/clickhouse/rebuild/replicas.ts

@@ -0,0 +1,148 @@
+/**
+ * A replica of a Replicated database that is down (#3270): what each step of
+ * the rebuild does about it.
+ *
+ * DDL (create, dual-write view, EXCHANGE, the views' DETACH and ATTACH, the
+ * drops, the receipts table) goes through the database's log in Keeper. The
+ * replica the run talks to executes it, and the others take it from the log;
+ * one that is down runs it in order when it comes back. So DDL proceeds on
+ * the live replicas: every statement carries
+ * `distributed_ddl_output_mode = throw_only_active`, which "doesn't wait for
+ * inactive replicas of the Replicated database" and still throws when a live
+ * one fails the statement
+ * (https://clickhouse.com/docs/reference/settings/session-settings/distributed-ddl).
+ * With the default `throw` the statement would wait the whole
+ * `distributed_ddl_task_timeout` (180s) for the down replica and then fail,
+ * after the live replicas had already run it. A replica counts as inactive
+ * once its Keeper session has expired (the client's `session_timeout_ms`,
+ * 30s by default), so a statement right after a replica goes down waits that
+ * long and then goes on. `KILL QUERY ON CLUSTER` takes the same setting and
+ * behaves the same way.
+ *
+ * Reading rows is different. A part written on a replica that went down
+ * before the others fetched it is on that replica alone, and the rows in it
+ * cannot be read anywhere else. A backfill that went on without them would
+ * copy the table less those rows, the verification would compare two tables
+ * that both lack them and pass, and the swap would lose them. So before a
+ * step reads rows it waits, for a bounded time, until this replica has
+ * fetched everything the others wrote (`SYSTEM SYNC REPLICA ... LIGHTWEIGHT`,
+ * https://clickhouse.com/docs/sql-reference/statements/system#sync-replica).
+ * When the down replica holds nothing this one lacks, which is the usual case,
+ * the wait returns at once and the step goes on. Otherwise the step stops
+ * with a {@link ReplicaFetchError} naming the replica and its parts.
+ */
+
+import { clickhouseQuery, ClickHouseQueryError, type ClickHouseEndpoint } from "../http";
+import { qualifiedIdent, sqlString } from "../apply/statements";
+
+/** Settings every DDL statement of the rebuild carries: go on without the replicas that are down. Ignored in an Atomic database. */
+export const DDL_SETTINGS: Readonly<Record<string, string>> = { distributed_ddl_output_mode: "throw_only_active" };
+
+/** How long a step waits for this replica to fetch what the others wrote, by default. */
+export const DEFAULT_REPLICA_TIMEOUT_MS = 120_000;
+
+/**
+ * One `SYSTEM SYNC REPLICA` waits at most this long (its `receive_timeout`),
+ * and the wait is repeated until the deadline. It keeps each HTTP request
+ * well inside the client's own five-minute response timeout, and puts a log
+ * line between them.
+ */
+const SYNC_SLICE_S = 60;
+
+/** A part this replica still has to fetch, and the replica it is to fetch it from. */
+export interface MissingPart {
+  part: string;
+  /** The replica that wrote it, as `system.replication_queue` names it. */
+  source: string;
+  /** Whether Keeper still lists that replica as active (it does for a while after it went down, until its session expires). */
+  sourceActive: boolean;
+  /** The last fetch error, first line, if any. */
+  lastError?: string;
+}
+
+/** A step stopped because this replica could not fetch parts that only another replica has. */
+export class ReplicaFetchError extends Error {
+  constructor(
+    /** `db.t`. */
+    readonly table: string,
+    /** The replica the run talks to. */
+    readonly replica: string,
+    readonly missing: MissingPart[],
+    readonly waitedMs: number,
+  ) {
+    const sources = [...new Set(missing.map((m) => m.source))];
+    const shown = missing
+      .slice(0, 10)
+      .map((m) => m.part + " from " + m.source + (m.sourceActive ? "" : " (inactive)") + (m.lastError ? " (last fetch error: " + m.lastError + ")" : ""))
+      .join(", ");
+    super(
+      `${table}: waited ${Math.round(waitedMs / 1000)}s for replica ${replica} to fetch what the other replicas wrote, and ${missing.length} part(s) are still to fetch: ${shown}${missing.length > 10 ? ", ..." : ""}. ` +
+        `Those rows are on ${sources.join(", ")} alone, so the step stops rather than read the table without them; nothing has been swapped. ` +
+        `Bring ${sources.join(", ")} back (it rejoins through Keeper and this replica fetches the parts), then run again. ` +
+        `replicaTimeout sets how long a step waits.`,
+    );
+    this.name = "ReplicaFetchError";
+  }
+}
+
+const isTimeout = (err: unknown) => err instanceof ClickHouseQueryError && /TIMEOUT_EXCEEDED|command timed out/.test(err.serverMessage);
+
+/**
+ * Wait until this replica has fetched every part the others have written to
+ * `database.table`, for at most `timeoutMs`; then throw a
+ * {@link ReplicaFetchError} naming what it still lacks and where that is.
+ */
+export async function syncReplicaWithin(
+  endpoint: ClickHouseEndpoint,
+  database: string,
+  table: string,
+  opts: { timeoutMs?: number; log?: (line: string) => void; signal?: AbortSignal } = {},
+): Promise<void> {
+  const started = Date.now();
+  const deadline = started + (opts.timeoutMs ?? DEFAULT_REPLICA_TIMEOUT_MS);
+  const sql = `SYSTEM SYNC REPLICA ${qualifiedIdent(database, table)} LIGHTWEIGHT`;
+  opts.log?.(sql);
+  for (;;) {
+    opts.signal?.throwIfAborted();
+    const slice = Math.max(1, Math.min(SYNC_SLICE_S, Math.ceil((deadline - Date.now()) / 1000)));
+    try {
+      await clickhouseQuery(endpoint, sql, { settings: { receive_timeout: String(slice) }, ...(opts.signal ? { signal: opts.signal } : {}) });
+      return;
+    } catch (err) {
+      if (!isTimeout(err)) throw err;
+    }
+    const { replica, missing } = await missingParts(endpoint, database, table);
+    // Fetched between the timeout and the look: nothing is left to wait for.
+    if (missing.length === 0) return;
+    if (Date.now() >= deadline) throw new ReplicaFetchError(`${database}.${table}`, replica, missing, Date.now() - started);
+    opts.log?.(`-- ${database}.${table}: replica ${replica} is still to fetch ${missing.length} part(s) from ${[...new Set(missing.map((m) => m.source))].join(", ")}; waiting`);
+  }
+}
+
+/** The parts this replica's queue is still to fetch for `database.table`, with where from. */
+export async function missingParts(endpoint: ClickHouseEndpoint, database: string, table: string): Promise<{ replica: string; missing: MissingPart[] }> {
+  const where = `database = ${sqlString(database)} AND table = ${sqlString(table)}`;
+  const [replicas] = await clickhouseQuery<{ replica_name: string; replica_is_active: Record<string, number> }>(
+    endpoint,
+    `SELECT replica_name, replica_is_active FROM system.replicas WHERE ${where}`,
+  );
+  // The entries LIGHTWEIGHT waits for.
+  const queue = await clickhouseQuery<{ new_part_name: string; source_replica: string; last_exception: string }>(
+    endpoint,
+    `SELECT new_part_name, source_replica, last_exception FROM system.replication_queue WHERE ${where} ` +
+      `AND type IN ('GET_PART', 'ATTACH_PART', 'DROP_RANGE', 'REPLACE_RANGE', 'DROP_PART') ORDER BY new_part_name`,
+  );
+  const active = replicas?.replica_is_active ?? {};
+  return {
+    replica: replicas?.replica_name ?? "this replica",
+    missing: queue.map((e) => {
+      const lastError = e.last_exception.trim().split("\n")[0];
+      return {
+        part: e.new_part_name,
+        source: e.source_replica || "another replica",
+        sourceActive: Number(active[e.source_replica] ?? 0) === 1,
+        ...(lastError ? { lastError } : {}),
+      };
+    }),
+  };
+}