@intentius/chant-lexicon-sql 0.124.45 → 0.124.48

package/src/coverage.test.ts

@@ -3,6 +3,8 @@ import { KIND_COVERAGE, PINNED_COMMANDS, kindCoverageProblems, pinnedCommandsCur
 import { POSTGRES_PROBES } from "./postgres/live/undeclarable";
 import { CLICKHOUSE_PROBES } from "./clickhouse/live/undeclarable";
 import { SQL_DIALECTS } from "./dialects";
+import { POSTGRES_ENTITY_TYPES } from "./postgres/entity-types";
+import { CLICKHOUSE_ENTITY_TYPES } from "./clickhouse/entities";
 
 describe("the object coverage table (#3911)", () => {
   test.each(SQL_DIALECTS)("%s: every CREATE command a pinned server has is marked, and nothing else", (dialect) => {
@@ -29,10 +31,32 @@ describe("the object coverage table (#3911)", () => {
     ]);
   });
 
-  test("every refused kind has the live query that finds it, and every query a refused-or-declared row", () => {
+  test("a declared row names an entity type the dialect has, and every entity type but a grant has a declared row", () => {
+    // A grant is a GRANT statement, not a CREATE command, so no row is about it.
+    const types = {
+      postgres: { all: Object.values(POSTGRES_ENTITY_TYPES) as string[], notCreate: ["Postgres::Grant", "Postgres::DefaultPrivileges"] },
+      clickhouse: { all: Object.values(CLICKHOUSE_ENTITY_TYPES) as string[], notCreate: ["ClickHouse::Grant"] },
+    };
+    for (const dialect of SQL_DIALECTS) {
+      const entities = KIND_COVERAGE[dialect].filter((r) => r.status === "declared").map((r) => r.entity!);
+      for (const e of entities) expect(types[dialect].all, `${dialect} row entity ${e}`).toContain(e);
+      for (const e of types[dialect].all.filter((t) => !types[dialect].notCreate.includes(t))) expect(entities, `${dialect} entity type ${e} has no declared row`).toContain(e);
+    }
+  });
+
+  test("a kind that is declared, referenced, refused or out of scope is told apart: only a declared row names an entity", () => {
+    for (const dialect of SQL_DIALECTS) {
+      for (const r of KIND_COVERAGE[dialect]) {
+        if (r.status === "declared") expect(r.reason, `${dialect} ${r.kind}`).toBeUndefined();
+        else expect(r.entity, `${dialect} ${r.kind}`).toBeUndefined();
+      }
+    }
+  });
+
+  test("every refused or referenced kind has the live query that finds it, and every query a row", () => {
     const probes = { postgres: POSTGRES_PROBES, clickhouse: CLICKHOUSE_PROBES };
     for (const dialect of SQL_DIALECTS) {
-      const refused = KIND_COVERAGE[dialect].filter((r) => r.status === "refused").map((r) => r.kind);
+      const refused = KIND_COVERAGE[dialect].filter((r) => r.status === "refused" || r.status === "referenced").map((r) => r.kind);
       for (const kind of refused) expect(Object.keys(probes[dialect]), `${dialect} ${kind}`).toContain(kind);
       const kinds = KIND_COVERAGE[dialect].map((r) => r.kind);
       for (const kind of Object.keys(probes[dialect])) expect(kinds, `${dialect} probe ${kind}`).toContain(kind);
@@ -68,6 +92,7 @@ describe("a statement's kind", () => {
     ["CREATE SETTINGS PROFILE p SETTINGS max_threads = 1", "settings profile"],
     ["CREATE OR REPLACE FUNCTION f AS (x) -> x + 1", "SQL function"],
     ["CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 10", "quota"],
+    ["CREATE NAMED COLLECTION IF NOT EXISTS c AS url = 'x'", "named collection"],
   ])("clickhouse: %s is a %s", (sql, kind) => {
     expect(statementKind("clickhouse", sql)?.kind).toBe(kind);
   });

package/src/coverage.ts

@@ -4,8 +4,10 @@
  * Which kinds of object each dialect declares (#3911): every `CREATE` command
  * a pinned server has (`./coverage/<dialect>-commands.json`, written by
  * `scripts/sql-create-commands.ts` from the server's own sources) has a row
- * in `./coverage/<dialect>.json` marking it declared, refused with a reason,
- * or out of scope. A command with several forms that differ in what chant can
+ * in `./coverage/<dialect>.json` marking it declared, referenced, refused
+ * with a reason, or out of scope. A referenced kind (a ClickHouse named
+ * collection, #4395) is never declared: a declaration names one, a plan
+ * checks it exists, and import lists the server's as prerequisites. A command with several forms that differ in what chant can
  * hold (`CREATE TYPE ... AS ENUM` and `CREATE TYPE ... AS RANGE`) has a row
  * per form. Import refuses an object of a refused kind by that row's `kind`
  * (`./core/undeclarable.ts`), and the docs page is rendered from the rows
@@ -31,8 +33,16 @@ import clickhouseCommands from "./coverage/clickhouse-commands.json" with { type
 
 // ── Object kinds ───────────────────────────────────────────────────────
 
-/** `declared`: a tag holds it. `refused`: import names it and stops. `out-of-scope`: the environment's, never imported. */
-export type KindStatus = "declared" | "refused" | "out-of-scope";
+/**
+ * `declared`: a tag holds it. `referenced`: the environment's, never declared
+ * and its values never read; a declaration names it, a plan checks it exists,
+ * and import lists the server's as prerequisites (#4395). `refused`: import
+ * names it and stops. `out-of-scope`: the environment's, never imported.
+ */
+export type KindStatus = "declared" | "referenced" | "refused" | "out-of-scope";
+
+/** Every status a coverage row may have. */
+export const KIND_STATUSES: readonly KindStatus[] = ["declared", "referenced", "refused", "out-of-scope"];
 
 /** One row of a dialect's coverage table: a `CREATE` command, or one form of it. */
 export interface KindRow {
@@ -51,7 +61,7 @@ export interface KindRow {
   status: KindStatus;
   /** The entity type that declares it, when declared. */
   entity?: string;
-  /** Why it is refused or out of scope. */
+  /** Why it is referenced, refused or out of scope. */
   reason?: string;
   /** Anything else a reader should know. */
   note?: string;
@@ -80,7 +90,8 @@ export function kindRow(dialect: SqlDialect, kind: string): KindRow {
  * What is wrong with a dialect's coverage table, against its pinned servers'
  * commands: a command a pinned server has and no row covers, a row for a
  * command no pinned server has, a kind named twice, a declared row with no
- * entity, a refused or out-of-scope row with no reason. Empty when complete.
+ * entity, a referenced, refused or out-of-scope row with no reason. Empty
+ * when complete.
  */
 export function kindCoverageProblems(dialect: SqlDialect, opts: { pins?: ReadonlyArray<{ version: string; commands: readonly string[] }>; rows?: readonly KindRow[] } = {}): string[] {
   const rows = opts.rows ?? KIND_COVERAGE[dialect];
@@ -96,7 +107,7 @@ export function kindCoverageProblems(dialect: SqlDialect, opts: { pins?: Readonl
   for (const r of rows) {
     if (seen.has(r.kind)) problems.push(`the kind "${r.kind}" has two rows`);
     seen.add(r.kind);
-    if (!["declared", "refused", "out-of-scope"].includes(r.status)) problems.push(`${r.kind}: status "${r.status}" is not declared, refused or out-of-scope`);
+    if (!KIND_STATUSES.includes(r.status)) problems.push(`${r.kind}: status "${r.status}" is not ${KIND_STATUSES.slice(0, -1).join(", ")} or ${KIND_STATUSES.at(-1)}`);
     if (r.status === "declared" && !r.entity) problems.push(`${r.kind} is declared, and names no entity type`);
     if (r.status !== "declared" && !r.reason) problems.push(`${r.kind} is ${r.status}, and gives no reason`);
   }

package/src/drizzle-command.ts

@@ -0,0 +1,94 @@
+/**
+ * `chant sql drizzle [--output <path>] [--check] [--bigint number|bigint]`
+ * (#4401): build the project the way `chant build` does, folded, and write the
+ * Drizzle schema module {@link renderDrizzle} renders from its entities.
+ *
+ * The file goes to `<project>/drizzle-schema.ts` by default, outside the
+ * source directory the build reads, and starts with chant's generated-file
+ * marker either way. `-o` and `--output` put it elsewhere, which is also the
+ * flag a workspace `generated` entry's generator passes. `--check` renders the
+ * same text and compares it with the file instead of writing it: exit 1,
+ * naming the file, when they differ.
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import type { CommandGroupContext } from "@intentius/chant/cli/command-group";
+import { DrizzleDialectError, renderDrizzle } from "./drizzle";
+import { buildFolded, loadProject } from "./sql-types-command";
+
+/** The file `chant sql drizzle` writes by default, in the project directory. */
+export const DRIZZLE_SCHEMA_FILE = "drizzle-schema.ts";
+
+const USAGE = "usage: chant sql drizzle [--output <path>] [--check] [--bigint number|bigint]";
+
+interface Args {
+  output?: string;
+  check: boolean;
+  bigint?: "number" | "bigint";
+}
+
+function parseArgs(args: readonly string[]): Args | { error: string } {
+  const out: Args = { check: false };
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i]!;
+    if (a === "--output" || a === "-o") {
+      const v = args[++i];
+      if (!v) return { error: `${a} needs a path` };
+      out.output = v;
+    } else if (a.startsWith("--output=")) out.output = a.slice("--output=".length);
+    else if (a === "--check") out.check = true;
+    else if (a === "--bigint" || a.startsWith("--bigint=")) {
+      const v = a === "--bigint" ? args[++i] : a.slice("--bigint=".length);
+      if (v !== "number" && v !== "bigint") return { error: "--bigint takes number or bigint" };
+      out.bigint = v;
+    } else return { error: `unknown argument ${a}` };
+  }
+  return out;
+}
+
+export async function runDrizzle(ctx: CommandGroupContext): Promise<number> {
+  const args = parseArgs(ctx.rawArgs);
+  if ("error" in args) {
+    console.error(`${args.error}\n${USAGE}`);
+    return 1;
+  }
+  const cwd = process.cwd();
+  const project = await loadProject(cwd);
+  const output = args.output !== undefined ? resolve(cwd, args.output) : join(project.projectDir, DRIZZLE_SCHEMA_FILE);
+  const result = await buildFolded(project, "chant sql drizzle", "nothing was written");
+  if (!result) return 1;
+
+  let rendered;
+  try {
+    rendered = renderDrizzle(result.entities, { config: project.config as unknown as Record<string, unknown>, ...(args.bigint ? { bigintMode: args.bigint } : {}) });
+  } catch (err) {
+    if (!(err instanceof DrizzleDialectError)) throw err;
+    console.error(`chant sql drizzle: ${err.message}`);
+    return 1;
+  }
+  const shown = relative(cwd, output);
+  const named = shown && !shown.startsWith("..") ? shown : output;
+  if (args.check) {
+    // Compares only; the file is never written.
+    const onDisk = existsSync(output) ? readFileSync(output, "utf8") : undefined;
+    if (onDisk === rendered.text) {
+      console.log(`${named} is current`);
+      return 0;
+    }
+    console.error(
+      onDisk === undefined
+        ? `chant sql drizzle: ${named} does not exist; run \`chant sql drizzle\` to write it`
+        : `chant sql drizzle: ${named} differs from what the build declares; run \`chant sql drizzle\` to rewrite it`,
+    );
+    return 1;
+  }
+  mkdirSync(dirname(output), { recursive: true });
+  writeFileSync(output, rendered.text);
+  const { tables, views, enums, sequences } = rendered.counts;
+  const plural = (n: number, word: string) => `${n} ${word}${n === 1 ? "" : "s"}`;
+  console.log(`wrote ${named}: ${[plural(tables, "table"), plural(views, "view"), plural(enums, "enum"), plural(sequences, "sequence")].join(", ")}`);
+  for (const s of rendered.skipped) console.log(`not written: ${s.kind} ${s.names.join(", ")} (${s.reason})`);
+  for (const n of rendered.approximated) console.log(`partly written: ${n}`);
+  return 0;
+}

package/src/drizzle-kit.e2e.test.ts

@@ -0,0 +1,89 @@
+/**
+ * `drizzle-kit generate` finds nothing to migrate (#4401). The declaration of
+ * `examples/postgres-getting-started` is applied to a throwaway Postgres;
+ * `drizzle-kit pull` writes what the server holds as drizzle-kit's baseline
+ * snapshot (`generate` compares a schema with the last snapshot in its `out`
+ * directory, never with a server); then `drizzle-kit generate` runs against
+ * the schema `chant sql drizzle` wrote and must report no changes and write no
+ * new migration.
+ *
+ * Needs Docker and network access (npm installs drizzle-orm and drizzle-kit in
+ * a scratch directory). Skipped without Docker.
+ */
+
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { build } from "@intentius/chant/build";
+import { resolveProjectBuildOptions } from "@intentius/chant/cli/build-options";
+import { DRIZZLE_ORM_CHECKED, renderDrizzle } from "./drizzle";
+import { sqlPlugin } from "./plugin";
+import { dockerAvailable, startTestPostgres, type TestPostgres } from "./postgres/testing/server";
+import { serializedObjects } from "./sql-types";
+import { sqlSerializer } from "./serializer";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const DRIZZLE_KIT = "0.31.11";
+const docker = await dockerAvailable();
+
+describe.skipIf(!docker)("drizzle-kit generate against a database applied from the declaration", () => {
+  let server: TestPostgres;
+  const work = mkdtempSync(join(tmpdir(), "chant-4401-kit-"));
+
+  beforeAll(async () => {
+    server = await startTestPostgres();
+  }, 180_000);
+
+  afterAll(async () => {
+    await server?.stop();
+    rmSync(work, { recursive: true, force: true });
+  });
+
+  const kit = (...args: string[]): string => execFileSync("npx", ["--no-install", "drizzle-kit", ...args], { cwd: work, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
+
+  test("no migration", async () => {
+    const dir = join(here, "..", "examples", "postgres-getting-started");
+    const options = resolveProjectBuildOptions({ config: {}, configDir: dir, plugins: [sqlPlugin], modes: { fold: true, sandbox: false } });
+    const result = await build(join(dir, "src"), [sqlSerializer], undefined, options);
+    expect(result.errors.map(String)).toEqual([]);
+    const config = { sql: { dialect: "postgres" } };
+
+    // The declaration, applied in its own order.
+    const doc = serializedObjects(result.entities, config) as { objects: Array<{ export: string; ddl: string }>; applyOrder?: string[] };
+    const byExport = new Map(doc.objects.map((o) => [o.export, o]));
+    const client = await server.connect();
+    try {
+      for (const name of doc.applyOrder ?? doc.objects.map((o) => o.export)) await client.query(byExport.get(name)!.ddl);
+    } finally {
+      await client.end();
+    }
+
+    writeFileSync(join(work, "package.json"), JSON.stringify({ name: "kit-scratch", private: true, type: "module" }));
+    execFileSync("npm", ["install", "--silent", `drizzle-orm@${DRIZZLE_ORM_CHECKED}`, `drizzle-kit@${DRIZZLE_KIT}`, "pg"], { cwd: work, stdio: "inherit" });
+    writeFileSync(join(work, "schema.ts"), renderDrizzle(result.entities, { config }).text);
+    const { url, password } = server.endpoint();
+    writeFileSync(
+      join(work, "drizzle.config.ts"),
+      `import { defineConfig } from "drizzle-kit";
+export default defineConfig({
+  dialect: "postgresql",
+  schema: "./schema.ts",
+  out: "./out",
+  schemaFilter: ["app"],
+  dbCredentials: { url: ${JSON.stringify(url.replace("postgres@", `postgres:${password}@`))}, ssl: false },
+});
+`,
+    );
+
+    kit("pull");
+    const before = readdirSync(join(work, "out")).filter((f) => f.endsWith(".sql"));
+    expect(before).toHaveLength(1);
+
+    const out = kit("generate");
+    expect(out).toMatch(/No schema changes, nothing to migrate/);
+    expect(readdirSync(join(work, "out")).filter((f) => f.endsWith(".sql"))).toEqual(before);
+  }, 600_000);
+});

package/src/drizzle-types.e2e.test.ts

@@ -0,0 +1,67 @@
+/**
+ * The generated Drizzle schemas type-check against drizzle-orm (#4401), and a
+ * select over `examples/postgres-getting-started` has the row its declaration
+ * gives. The goldens are `test-fixtures/drizzle/*\/schema.ts`
+ * (`drizzle.test.ts` keeps them equal to what `renderDrizzle` writes);
+ * `postgres-getting-started/rows.ts` holds the row-type assertions.
+ *
+ * drizzle-orm is not a dependency of this repository, so the test runs only
+ * where it resolves from the fixtures (install `drizzle-orm@0.45.4`, the
+ * version `DRIZZLE_ORM_CHECKED` names, and `typescript`), and is skipped
+ * otherwise.
+ */
+
+import { createRequire } from "node:module";
+import { dirname, join, relative } from "node:path";
+import { fileURLToPath } from "node:url";
+import ts from "typescript";
+import { describe, expect, test } from "vitest";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const repoRoot = join(here, "..", "..", "..");
+const fixtures = join(here, "..", "test-fixtures", "drizzle");
+
+function drizzleInstalled(): boolean {
+  try {
+    createRequire(join(fixtures, "x.js")).resolve("drizzle-orm/package.json");
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Every error tsc reports in the fixture's files, as `file(line): TScode message`. */
+function tscErrors(dir: string): string[] {
+  const options: ts.CompilerOptions = {
+    target: ts.ScriptTarget.ES2022,
+    lib: ["lib.es2023.d.ts"],
+    module: ts.ModuleKind.ESNext,
+    moduleResolution: ts.ModuleResolutionKind.Bundler,
+    strict: true,
+    noEmit: true,
+    skipLibCheck: true,
+    types: ["node"],
+    typeRoots: [join(repoRoot, "node_modules", "@types")],
+  };
+  const files = ts.sys.readDirectory(dir, [".ts"]);
+  const program = ts.createProgram(files, options);
+  const out: string[] = [];
+  for (const file of files) {
+    const sf = program.getSourceFile(file)!;
+    for (const d of [...program.getSyntacticDiagnostics(sf), ...program.getSemanticDiagnostics(sf)]) {
+      const { line } = sf.getLineAndCharacterOfPosition(d.start ?? 0);
+      out.push(`${relative(dir, file)}(${line + 1}): TS${d.code} ${ts.flattenDiagnosticMessageText(d.messageText, "\n").split("\n")[0]}`);
+    }
+  }
+  return out.sort();
+}
+
+describe.skipIf(!drizzleInstalled())("the generated Drizzle schema, through tsc", () => {
+  test("examples/postgres-getting-started: the schema compiles and a select has the declared row type", () => {
+    expect(tscErrors(join(fixtures, "postgres-getting-started"))).toEqual([]);
+  }, 120_000);
+
+  test("every mapping compiles, including two tables that reference each other", () => {
+    expect(tscErrors(join(fixtures, "mappings"))).toEqual([]);
+  }, 120_000);
+});