@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/spec/fetch.test.ts

@@ -0,0 +1,77 @@
+import { createServer, type Server } from "node:http";
+import { copyFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { fetchCatalog, SNAPSHOT_FILE } from "./fetch";
+import { CLICKHOUSE_VERSION } from "./pin";
+import { parseCatalog, stringifyCatalog } from "./catalog";
+
+/** A stand-in for a server's HTTP interface: answers `version()` and returns no rows for the rest. */
+function fakeServer(version: string): Promise<{ url: string; server: Server; queries: string[] }> {
+  const queries: string[] = [];
+  const server = createServer((req, res) => {
+    let body = "";
+    req.on("data", (c) => (body += c));
+    req.on("end", () => {
+      queries.push(body);
+      res.end(body.includes("version()") ? `${JSON.stringify({ v: version })}\n` : "");
+    });
+  });
+  return new Promise((resolve) =>
+    server.listen(0, "127.0.0.1", () => {
+      const { port } = server.address() as { port: number };
+      resolve({ url: `http://127.0.0.1:${port}`, server, queries });
+    }),
+  );
+}
+
+let dir: string;
+const quiet = () => undefined;
+
+beforeAll(() => {
+  dir = mkdtempSync(join(tmpdir(), "chant-sql-fetch-"));
+});
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+describe("where generation gets the catalog", () => {
+  test("a snapshot at the pin is read as is, and nothing is started", async () => {
+    const file = join(dir, "at-pin.json");
+    copyFileSync(SNAPSHOT_FILE, file);
+    const lines: string[] = [];
+    const catalog = await fetchCatalog({ snapshotFile: file, serverUrl: undefined, log: (l) => lines.push(l) });
+    expect(catalog.version).toBe(CLICKHOUSE_VERSION);
+    expect(lines.join("\n")).toContain("committed snapshot");
+  });
+
+  test("a server at a version other than the pin is refused, and the snapshot is left alone", async () => {
+    const file = join(dir, "moved.json");
+    const old = { ...parseCatalog(readFileSync(SNAPSHOT_FILE, "utf-8")), version: "26.3.39.7" };
+    writeFileSync(file, stringifyCatalog(old));
+    const before = readFileSync(file, "utf-8");
+
+    const fake = await fakeServer("26.9.8.3");
+    try {
+      await expect(fetchCatalog({ snapshotFile: file, serverUrl: fake.url, log: quiet })).rejects.toThrow(
+        /is ClickHouse 26\.9\.8\.3, not the pinned/,
+      );
+    } finally {
+      fake.server.close();
+    }
+    expect(readFileSync(file, "utf-8")).toBe(before);
+  });
+
+  test("a server at the pin is read and the snapshot rewritten from it", async () => {
+    const file = join(dir, "refresh.json");
+    writeFileSync(file, "{}");
+    const fake = await fakeServer(CLICKHOUSE_VERSION);
+    try {
+      const catalog = await fetchCatalog({ snapshotFile: file, serverUrl: fake.url, log: quiet });
+      expect(catalog.version).toBe(CLICKHOUSE_VERSION);
+      expect(fake.queries.some((q) => q.includes("FROM system.table_engines ORDER BY name"))).toBe(true);
+      expect(parseCatalog(readFileSync(file, "utf-8")).version).toBe(CLICKHOUSE_VERSION);
+    } finally {
+      fake.server.close();
+    }
+  });
+});

package/src/spec/fetch.ts

@@ -0,0 +1,112 @@
+/**
+ * Where generation gets the ClickHouse catalog.
+ *
+ * The committed snapshot, `clickhouse-catalog.snapshot.json`, is the source
+ * every ordinary run reads: a fresh clone, CI and `npm run prepack` generate
+ * from it with no Docker and no network. A server is read only when the
+ * snapshot cannot serve the pin:
+ *
+ * - the pin moved (`CLICKHOUSE_VERSION` differs from the snapshot's version),
+ * - `force` is set, or
+ * - `CHANT_SQL_CLICKHOUSE_URL` names a server to read instead of starting one.
+ *
+ * Reading a server means starting the pinned image in a throwaway container
+ * (or using the named one), checking that its `version()` is the pin, reading
+ * `system.*`, and rewriting the snapshot. A server at any other version is
+ * refused: a snapshot labelled with one pin and holding another's surface is
+ * the failure the pin exists to prevent.
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { CLICKHOUSE_VERSION, clickhouseImage } from "./pin";
+import { parseCatalog, readCatalog, stringifyCatalog, type ClickHouseCatalog } from "./catalog";
+import { startScratchServer } from "../clickhouse/container";
+import type { ClickHouseEndpoint } from "../clickhouse/http";
+
+export const SNAPSHOT_FILE = join(dirname(fileURLToPath(import.meta.url)), "clickhouse-catalog.snapshot.json");
+
+/** The key the single catalog document is filed under in the pipeline's schema map. */
+export const CATALOG_KEY = "ClickHouse::Catalog";
+
+export interface FetchCatalogOptions {
+  /** Read a server even when the snapshot matches the pin. */
+  force?: boolean;
+  /** A running server to read instead of starting the pinned image. Defaults to `CHANT_SQL_CLICKHOUSE_URL`. */
+  serverUrl?: string;
+  /** The snapshot to read and rewrite. Tests point it elsewhere. */
+  snapshotFile?: string;
+  /** Log a line saying which route was taken. */
+  log?: (line: string) => void;
+}
+
+function readSnapshot(file: string): ClickHouseCatalog | undefined {
+  try {
+    return parseCatalog(readFileSync(file, "utf-8"));
+  } catch {
+    return undefined;
+  }
+}
+
+/** Read the pinned catalog from a server and refuse any other version. */
+export async function catalogFromServer(endpoint: ClickHouseEndpoint, image: string): Promise<ClickHouseCatalog> {
+  const catalog = await readCatalog(endpoint, image);
+  if (catalog.version !== CLICKHOUSE_VERSION) {
+    throw new Error(
+      `the server at ${endpoint.url} is ClickHouse ${catalog.version}, not the pinned ${CLICKHOUSE_VERSION}. ` +
+        "Point CHANT_SQL_CLICKHOUSE_URL at a server running the pin, or update CLICKHOUSE_IMAGE_DIGEST in src/spec/pin.ts " +
+        "together with CLICKHOUSE_VERSION.",
+    );
+  }
+  return catalog;
+}
+
+/**
+ * The catalog for the current pin: the snapshot when it matches, otherwise a
+ * fresh read that also rewrites the snapshot.
+ */
+export async function fetchCatalog(options: FetchCatalogOptions = {}): Promise<ClickHouseCatalog> {
+  const file = options.snapshotFile ?? SNAPSHOT_FILE;
+  const log = options.log ?? ((line: string) => console.error(line));
+  const serverUrl = options.serverUrl ?? process.env.CHANT_SQL_CLICKHOUSE_URL;
+  const snapshot = readSnapshot(file);
+
+  if (snapshot && snapshot.version === CLICKHOUSE_VERSION && !options.force && !serverUrl) {
+    log(`[sql] clickhouse catalog: committed snapshot at ${snapshot.version}`);
+    return snapshot;
+  }
+
+  const reason = !snapshot
+    ? "no readable snapshot"
+    : snapshot.version !== CLICKHOUSE_VERSION
+      ? `the pin moved from ${snapshot.version} to ${CLICKHOUSE_VERSION}`
+      : serverUrl
+        ? "CHANT_SQL_CLICKHOUSE_URL is set"
+        : "force";
+
+  let catalog: ClickHouseCatalog;
+  if (serverUrl) {
+    log(`[sql] clickhouse catalog: reading ${serverUrl} (${reason})`);
+    catalog = await catalogFromServer({ url: serverUrl }, clickhouseImage());
+  } else {
+    const image = clickhouseImage();
+    log(`[sql] clickhouse catalog: starting ${image} (${reason})`);
+    const server = await startScratchServer(image, { namePrefix: "chant-sql-catalog" });
+    try {
+      catalog = await catalogFromServer(server.endpoint, image);
+    } finally {
+      await server.stop();
+    }
+  }
+
+  writeFileSync(file, stringifyCatalog(catalog));
+  log(`[sql] clickhouse catalog: wrote ${file}`);
+  return catalog;
+}
+
+/** The pipeline's `fetchSchemas`: one document, the catalog, keyed {@link CATALOG_KEY}. */
+export async function fetchSchemas(options: FetchCatalogOptions = {}): Promise<Map<string, Buffer>> {
+  const catalog = await fetchCatalog(options);
+  return new Map([[CATALOG_KEY, Buffer.from(stringifyCatalog(catalog), "utf-8")]]);
+}

package/src/spec/pin.ts

@@ -0,0 +1,47 @@
+/**
+ * The pinned ClickHouse server the ClickHouse dialect's types come from.
+ *
+ * ClickHouse publishes no schema for its DDL. The engine names, column type
+ * families, codecs, skip index types and both settings surfaces are read from a
+ * running server's `system.*` tables (chant #3195), so the pin is a server
+ * image, not a document.
+ *
+ * Pinned to an exact patch of the 26.8 LTS line. 26.3 LTS lacks the `syntax`
+ * column on `system.table_engines` and the `system.data_skipping_index_types`
+ * table the engine-argument and skip-index types are built from, and two
+ * patches of one line differ (26.8.5.13 and 26.8.15.10 differ by nine query
+ * settings), so a floating `26.8` tag would change generated output with no
+ * commit. The digest sits beside the tag so a re-pushed tag cannot change it
+ * unseen either: generation runs `image@digest` and refuses a server whose
+ * `version()` is not {@link CLICKHOUSE_VERSION}.
+ *
+ * Moving the pin: change both constants, then `chant dev generate` (or
+ * `npm run generate`). The committed snapshot no longer matches the pin, so
+ * generation starts the pinned server, rewrites
+ * `src/spec/clickhouse-catalog.snapshot.json`, and the pull request's diff of
+ * that file is the surface change. Read its `-` lines: a removed engine or
+ * function is what breaks a user's declarations.
+ */
+
+/** The pinned `clickhouse-server` release, as `SELECT version()` reports it. */
+export const CLICKHOUSE_VERSION = "26.8.15.10";
+
+/** The image digest of `clickhouse/clickhouse-server:<CLICKHOUSE_VERSION>`. Move it with the version. */
+export const CLICKHOUSE_IMAGE_DIGEST = "sha256:9716a352d25f538bd19d868b001141e7a60e4d035bc9b823658c6a071660aa61";
+
+/** The official server image. */
+export const CLICKHOUSE_IMAGE_REPOSITORY = "clickhouse/clickhouse-server";
+
+/** The full image reference generation runs: tag for the reader, digest for the runtime. */
+export function clickhouseImage(): string {
+  return `${CLICKHOUSE_IMAGE_REPOSITORY}:${CLICKHOUSE_VERSION}@${CLICKHOUSE_IMAGE_DIGEST}`;
+}
+
+/**
+ * ClickHouse tags its LTS releases `v<version>-lts` on GitHub. The upgrade
+ * tooling hands `replace` the raw tag; the constant holds the bare version,
+ * which is also the docker tag.
+ */
+export function versionFromReleaseTag(tag: string): string {
+  return tag.replace(/^v/, "").replace(/-lts$/, "");
+}