@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/src/sqlserver/types.ts

@@ -0,0 +1,154 @@
+/**
+ * The T-SQL to ClickHouse type map.
+ *
+ * Source types and their ranges, precisions and defaults are from Microsoft's
+ * data type reference
+ * (https://learn.microsoft.com/en-us/sql/t-sql/data-types/data-types-transact-sql)
+ * and the page of each type under it: `float(n)` is `real` for n in 1..24 and
+ * the 8-byte type for 25..53 with a default of 53; `decimal(p,s)` defaults to
+ * (18,0) with p up to 38; `datetime2` and `time` take a scale of 0 to 7 with
+ * a default of 7; `money` is a 64-bit integer with four decimals and
+ * `smallmoney` a 32-bit one; `rowversion` (`timestamp`) is an 8-byte counter.
+ *
+ * Target types are ClickHouse types of the pinned server (see
+ * `../spec/pin.ts`). `types.test.ts` checks every family this file writes
+ * against `TYPE_FAMILIES` in the generated catalog and every precision against
+ * the `DateTime64`, `Time64` and `Decimal` ranges, so a pin move that drops a
+ * type fails there.
+ *
+ * A type either maps with no judgment (`note` absent) or carries a `note`
+ * naming what the map could not keep. A type this file does not know
+ * (`sql_variant`, a user-defined alias type) is refused by name.
+ */
+
+export interface MappedType {
+  ok: true;
+  /** The ClickHouse type, before `Nullable`. */
+  type: string;
+  /** The declared maximum length of a character or binary type, when it has one. */
+  maxLength?: number;
+  /** Present when the map is a judgment: what was lost or chosen. */
+  note?: { message: string; evidence: string[] };
+}
+
+export interface RefusedType {
+  ok: false;
+  reason: string;
+}
+
+export type TypeMapping = MappedType | RefusedType;
+
+type Arg = number | "max";
+
+const num = (args: readonly Arg[], i: number): number | undefined => (typeof args[i] === "number" ? (args[i] as number) : undefined);
+
+/** Map a column type. `name` is lower-case, `args` the parenthesised arguments. */
+export function mapType(name: string, args: readonly Arg[]): TypeMapping {
+  switch (name) {
+    case "tinyint":
+      return { ok: true, type: "UInt8" };
+    case "smallint":
+      return { ok: true, type: "Int16" };
+    case "int":
+    case "integer":
+      return { ok: true, type: "Int32" };
+    case "bigint":
+      return { ok: true, type: "Int64" };
+    case "bit":
+      return { ok: true, type: "Bool" };
+    case "decimal":
+    case "numeric":
+    case "dec": {
+      const p = num(args, 0) ?? 18;
+      const s = num(args, 1) ?? 0;
+      return { ok: true, type: `Decimal(${String(p)}, ${String(s)})` };
+    }
+    case "money":
+      return { ok: true, type: "Decimal(19, 4)" };
+    case "smallmoney":
+      return { ok: true, type: "Decimal(10, 4)" };
+    case "real":
+      return { ok: true, type: "Float32" };
+    case "float": {
+      const n = num(args, 0) ?? 53;
+      return { ok: true, type: n <= 24 ? "Float32" : "Float64" };
+    }
+    case "date":
+      return { ok: true, type: "Date32" };
+    case "time":
+      return { ok: true, type: `Time64(${String(num(args, 0) ?? 7)})` };
+    case "datetime2":
+      return { ok: true, type: `DateTime64(${String(num(args, 0) ?? 7)})` };
+    case "datetime":
+      return { ok: true, type: "DateTime64(3)" };
+    case "smalldatetime":
+      return { ok: true, type: "DateTime64(0)" };
+    case "datetimeoffset": {
+      const p = num(args, 0) ?? 7;
+      return {
+        ok: true,
+        type: `DateTime64(${String(p)}, 'UTC')`,
+        note: {
+          message: "datetimeoffset is read as a UTC DateTime64; the stored offset is not kept.",
+          evidence: ["datetimeoffset stores a time zone offset (-14:00 to +14:00) with each value; a ClickHouse DateTime64 holds an instant and one column time zone", "values must be converted to UTC when loaded"],
+        },
+      };
+    }
+    case "char":
+    case "varchar":
+    case "nchar":
+    case "nvarchar":
+    case "text":
+    case "ntext":
+    case "sysname": {
+      const max = name === "sysname" ? 128 : typeof args[0] === "number" ? args[0] : undefined;
+      return { ok: true, type: "String", ...(max !== undefined ? { maxLength: max } : {}) };
+    }
+    case "binary":
+    case "varbinary":
+    case "image": {
+      const max = typeof args[0] === "number" ? args[0] : undefined;
+      return { ok: true, type: "String", ...(max !== undefined ? { maxLength: max } : {}) };
+    }
+    case "uniqueidentifier":
+      return { ok: true, type: "UUID" };
+    case "json":
+      return { ok: true, type: "JSON" };
+    case "xml":
+      return {
+        ok: true,
+        type: "String",
+        note: { message: "xml is read as String; ClickHouse has no XML type.", evidence: ["the xml type validates and indexes documents; a String does neither"] },
+      };
+    case "rowversion":
+    case "timestamp":
+      return {
+        ok: true,
+        type: "String",
+        note: {
+          message: `${name} is read as String; its 8 bytes are an opaque counter.`,
+          evidence: ["rowversion is a database-wide counter that SQL Server writes on each row change, not a date or time", "a String keeps the bytes; a UInt64 version column has to be loaded from the source"],
+        },
+      };
+    case "hierarchyid":
+      return { ok: true, type: "String", note: { message: "hierarchyid is read as String; ClickHouse has no hierarchy type.", evidence: ["the binary form is kept as bytes; hierarchy methods (GetAncestor, IsDescendantOf) have no ClickHouse equivalent"] } };
+    case "geography":
+    case "geometry":
+      return { ok: true, type: "String", note: { message: `${name} is read as String; the spatial value is not converted.`, evidence: ["a spatial value needs a conversion (well-known text or binary) at load time", "ClickHouse has Point, Ring and Polygon types, which fit only some shapes"] } };
+    case "vector": {
+      const dims = num(args, 0);
+      return {
+        ok: true,
+        type: "Array(Float32)",
+        note: { message: "vector is read as Array(Float32).", evidence: [dims !== undefined ? `declared with ${String(dims)} dimensions, which an Array does not enforce` : "the dimension count is not enforced by an Array"] },
+      };
+    }
+    case "sql_variant":
+      return { ok: false, reason: "sql_variant holds values of any type, and no single ClickHouse type matches it" };
+    case "cursor":
+    case "table":
+      return { ok: false, reason: `${name} is not a type a stored column can have` };
+    default:
+      return { ok: false, reason: `the type ${name} is not a system type; user-defined alias types are not read` };
+  }
+}

package/src/sqlserver/validate.e2e.test.ts

@@ -0,0 +1,32 @@
+/**
+ * `--validate` for a SQL Server import (#4398), against the pinned ClickHouse
+ * server: the translated sample schema is created, and a statement the server
+ * refuses is reported by name.
+ *
+ * Needs Docker; skips cleanly without it. One throwaway container, removed
+ * afterwards even on failure.
+ */
+
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, test } from "vitest";
+import { dockerAvailable } from "../clickhouse/container";
+import { sqlPlugin } from "../plugin";
+import { validateOnEmulator } from "./validate";
+
+const enabled = await dockerAvailable();
+const fixture = readFileSync(join(import.meta.dirname, "..", "..", "test-fixtures", "sqlserver", "wwi-excerpt.sql"), "utf8");
+const parse = (text: string) => sqlPlugin.templateParser!({ from: "sqlserver" }).parse(text);
+
+describe.skipIf(!enabled)("the emulator accepts the translated sample", () => {
+  test("every statement of the WideWorldImporters excerpt is created", async () => {
+    const diagnostics = await validateOnEmulator(parse(fixture));
+    expect(diagnostics, diagnostics.map((d) => d.message).join("\n")).toEqual([]);
+  }, 300_000);
+
+  test("a view over a column the table does not have is rejected, by name", async () => {
+    const diagnostics = await validateOnEmulator(parse("CREATE TABLE t (a int NOT NULL);\nGO\nCREATE VIEW v AS SELECT b FROM t"));
+    expect(diagnostics).toHaveLength(1);
+    expect(diagnostics[0]).toMatchObject({ ruleId: "SQLRV012", severity: "error", entity: "v" });
+  }, 300_000);
+});

package/src/sqlserver/validate.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, test } from "vitest";
+import { ClickHouseQueryError } from "../clickhouse/http";
+import { sqlPlugin } from "../plugin";
+import { statementsOf, validateStatements } from "./validate";
+
+const ir = sqlPlugin.templateParser!({ from: "sqlserver" }).parse(`
+CREATE VIEW WebApi.V AS SELECT x FROM Sales.T;
+GO
+CREATE TABLE Sales.T (x int NOT NULL);
+`);
+
+describe("validateStatements", () => {
+  test("runs databases, then tables, then views, whatever order the file had", () => {
+    expect(statementsOf(ir).map((s) => `${s.type.replace("ClickHouse::", "")} ${s.name}`)).toEqual(["Database WebApi", "Database Sales", "Table Sales.T", "View WebApi.V"]);
+  });
+
+  test("a statement the server accepts raises nothing", async () => {
+    const seen: string[] = [];
+    const out = await validateStatements(statementsOf(ir), async (sql) => void seen.push(sql));
+    expect(out).toEqual([]);
+    expect(seen).toHaveLength(4);
+    expect(seen[2]).toContain("CREATE TABLE Sales.T");
+  });
+
+  test("a refusal is an error diagnostic with the server's first line, and the run goes on", async () => {
+    const out = await validateStatements(statementsOf(ir), async (sql) => {
+      if (sql.includes("CREATE VIEW")) throw new ClickHouseQueryError(400, "Code: 47. DB::Exception: Unknown identifier x\nstack trace", sql);
+      if (sql.includes("CREATE TABLE")) throw new Error("connection reset");
+    });
+    expect(out).toEqual([
+      { ruleId: "SQLRV012", severity: "error", entity: "Sales.T", message: "Sales.T: the emulator rejected the statement. connection reset" },
+      { ruleId: "SQLRV012", severity: "error", entity: "WebApi.V", message: "WebApi.V: the emulator rejected the statement. Code: 47. DB::Exception: Unknown identifier x" },
+    ]);
+  });
+
+  test("a refusal fails the run even without --strict", async () => {
+    const { importExitCode } = await import("@intentius/chant/cli/commands/import-report");
+    const out = await validateStatements(statementsOf(ir), async () => {
+      throw new Error("no");
+    });
+    expect(importExitCode(out, false)).toBe(1);
+  });
+});
+
+describe("validateImport", () => {
+  test("is the sql plugin's hook for --validate", () => {
+    expect(typeof sqlPlugin.validateImport).toBe("function");
+  });
+});

package/src/sqlserver/validate.ts

@@ -0,0 +1,76 @@
+/**
+ * `chant import --validate` for a translated schema (#4398): create the
+ * emitted DDL on the pinned ClickHouse server, as `chant migrate --validate`
+ * runs the emitted YAML through the target's linter.
+ *
+ * `validateOnEmulator` starts its own throwaway container of the pinned image
+ * (`../clickhouse/container.ts`) and removes it afterwards. It never reads
+ * `CLICKHOUSE_URL`, so a validation cannot reach a server that holds data.
+ * `validateStatements` is the part that needs no Docker: it runs statements
+ * through any `run` function, in the order a server needs them (databases,
+ * then tables, then views), and turns each refusal into an `error`
+ * diagnostic naming the object.
+ */
+
+import type { ImportDiagnostic, TemplateIR } from "@intentius/chant/import/parser";
+import { CLICKHOUSE_ENTITY_TYPES } from "../clickhouse/entities";
+import { ClickHouseQueryError, clickhouseQuery } from "../clickhouse/http";
+import { dockerAvailable, startScratchServer } from "../clickhouse/container";
+import { clickhouseImage } from "../spec/pin";
+
+export interface ValidationStatement {
+  /** The object, as the diagnostic names it. */
+  name: string;
+  ddl: string;
+  type: string;
+}
+
+const RANK: Record<string, number> = {
+  [CLICKHOUSE_ENTITY_TYPES.database]: 0,
+  [CLICKHOUSE_ENTITY_TYPES.table]: 1,
+  [CLICKHOUSE_ENTITY_TYPES.view]: 2,
+  [CLICKHOUSE_ENTITY_TYPES.materializedView]: 2,
+};
+
+/** The IR's ClickHouse objects as statements in creation order. */
+export function statementsOf(ir: TemplateIR): ValidationStatement[] {
+  const out = ir.resources.map((r, i) => {
+    const p = r.properties as { database?: string; name: string; ddl: string };
+    return { name: p.database ? `${p.database}.${p.name}` : p.name, ddl: p.ddl, type: r.type, i };
+  });
+  return out
+    .sort((a, b) => (RANK[a.type] ?? 3) - (RANK[b.type] ?? 3) || a.i - b.i)
+    .map(({ name, ddl, type }) => ({ name, ddl, type }));
+}
+
+/** Run each statement; a refusal is an error diagnostic, and the run goes on to the next. */
+export async function validateStatements(statements: readonly ValidationStatement[], run: (sql: string) => Promise<void>): Promise<ImportDiagnostic[]> {
+  const out: ImportDiagnostic[] = [];
+  for (const s of statements) {
+    try {
+      await run(s.ddl);
+    } catch (e) {
+      const msg = e instanceof ClickHouseQueryError ? e.serverMessage.trim().split("\n")[0]! : e instanceof Error ? e.message : String(e);
+      out.push({ ruleId: "SQLRV012", severity: "error", message: `${s.name}: the emulator rejected the statement. ${msg}`, entity: s.name });
+    }
+  }
+  return out;
+}
+
+/**
+ * Create the IR's objects on a throwaway server of the pinned image. Throws,
+ * naming the cause, when Docker is not available: the caller reports that the
+ * validation did not run, rather than reporting a pass.
+ */
+export async function validateOnEmulator(ir: TemplateIR): Promise<ImportDiagnostic[]> {
+  if (ir.resources.some((r) => r.type.startsWith("Postgres::"))) throw new Error("--validate checks ClickHouse DDL; this template holds Postgres objects");
+  if (!(await dockerAvailable())) throw new Error("Docker is not available, so the ClickHouse emulator cannot start");
+  const server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-import-validate" });
+  try {
+    return await validateStatements(statementsOf(ir), async (sql) => {
+      await clickhouseQuery(server.endpoint, sql);
+    });
+  } finally {
+    await server.stop();
+  }
+}

package/src/validate.ts

@@ -47,6 +47,16 @@ const REQUIRED_ENTITIES = [
   "PostgresCollation",
   "PostgresCast",
   "PostgresEventTrigger",
+  "PostgresOperator",
+  "PostgresOperatorClass",
+  "PostgresOperatorFamily",
+  "PostgresForeignTable",
+  "PostgresTextSearchConfiguration",
+  "PostgresTextSearchDictionary",
+  "PostgresStatistics",
+  "PostgresForeignDataWrapper",
+  "PostgresForeignServer",
+  "PostgresPublication",
   "PostgresExtension",
   "PostgresFunction",
   "PostgresProcedure",

package/src/workspace-kinds.test.ts

@@ -17,5 +17,16 @@ describeWorkspaceKindConformance({
     { name: "a chant project beside a dbt_project.yml", kind: "dbt", files: { "dbt_project.yml": project, "chant.config.ts": "" }, claims: true },
     { name: "a directory whose dbt project sits one level down", kind: "dbt", files: { "analytics/dbt_project.yml": project }, claims: false },
     { name: "models without a project file", kind: "dbt", files: { "models/sources.yml": "version: 2\n" }, claims: false },
+    // Migration directories (#4400).
+    { name: "a goose directory", kind: "goose", files: { "00001_users.sql": "-- +goose Up\nCREATE TABLE users (id int);\n-- +goose Down\nDROP TABLE users;\n" }, claims: true },
+    { name: "plain SQL files without goose sections", kind: "goose", files: { "00001_users.sql": "CREATE TABLE users (id int);\n" }, claims: false },
+    { name: "a golang-migrate directory", kind: "golang-migrate", files: { "1_users.up.sql": "CREATE TABLE users (id int);\n", "1_users.down.sql": "DROP TABLE users;\n" }, claims: true },
+    { name: "a golang-migrate directory beats clickhouse-migrations on the same file names", kind: "golang-migrate", files: { "1_users.up.sql": "select 1;\n" }, claims: true },
+    { name: "migrations with no up files", kind: "golang-migrate", files: { "1_users.sql": "select 1;\n" }, claims: false },
+    { name: "numbered SQL files", kind: "clickhouse-migrations", files: { "001_init.sql": "CREATE TABLE t (id UInt64) ENGINE = Memory;\n" }, claims: true },
+    { name: "SQL files with no version", kind: "clickhouse-migrations", files: { "init.sql": "select 1;\n" }, claims: false },
+    { name: "a Flyway project with a configuration file", kind: "flyway", files: { "flyway.toml": "[flyway]\n" }, claims: true },
+    { name: "a Flyway directory with only migrations", kind: "flyway", files: { "V1__init.sql": "select 1;\n" }, claims: true },
+    { name: "a directory with no Flyway files", kind: "flyway", files: { "schema.sql": "select 1;\n" }, claims: false },
   ],
 });

package/workspace-kinds.json

@@ -1,10 +1,10 @@
 {
-  "$comment": "Member kinds this lexicon supplies to a chant workspace (#3932). chant reads this file as data and never imports the lexicon. A dbt project reads the tables a sql member declares, so its kind ships beside them. Its reader, dbt-project, is chant's own code: it parses dbt_project.yml and the sources YAML under the model paths, never runs dbt, never renders Jinja and never reads profiles.yml, and turns each source table and column into a link to the sql member's <schema>.<table> and <schema>.<table>.<column> outputs (#3931).",
+  "$comment": "Member kinds this lexicon supplies to a chant workspace (#3932). chant reads this file as data and never imports the lexicon. A dbt project reads the tables a sql member declares, so its kind ships beside them. Its reader, dbt-project, is chant's own code: it parses dbt_project.yml and the sources YAML under the model paths, never runs dbt, never renders Jinja and never reads profiles.yml, and turns each source table and column into a link to the sql member's <schema>.<table> and <schema>.<table>.<column> outputs (#3931). The same pass exposes each model as an output, <schema>.<model> and <schema>.<model>.<column> where the model's YAML lists columns (#3935); the schema is the modelSchema field, or the custom schema dbt config names when the project's generate_schema_name makes that independent of the target. Four more kinds read existing migration directories (#4400): goose, golang-migrate, clickhouse-migrations and flyway. Each has a probe and a reader (goose-dir, golang-migrate-dir, clickhouse-migrations-dir, flyway-dir) that lists the migrations in the order the tool applies them, with their dialect, and never runs the tool or a server. The reader returns no links and no outputs: the tables and columns a directory produces come from replaying it into a server, not from reading its files.",
   "schema": 1,
   "kinds": [
     {
       "name": "dbt",
-      "description": "a dbt project, with a dbt_project.yml in its directory; its sources link to the tables and columns a sql member declares, and dbt is never run",
+      "description": "a dbt project, with a dbt_project.yml in its directory; its sources link to the tables and columns a sql member declares, its models are outputs other members link to, and dbt is never run",
       "precedence": 300,
       "probe": {
         "anyFile": [
@@ -23,6 +23,104 @@
           "description": "How source names are cased before they are matched: postgres folds an unquoted name to lowercase and keeps a quoted one as written, the way dbt-postgres quotes a source's schema and identifier unless quoting turns it off; clickhouse keeps every name as written.",
           "default": "postgres",
           "pattern": "^(postgres|clickhouse)$"
+        },
+        "modelSchema": {
+          "type": "string",
+          "description": "The schema the project's models land in. When set, each model is the output <modelSchema>.<model> and dbt's config is not read for the schema. Unset, chant reads +schema, the model YAML's config and the model's config(schema=...), and a model whose schema depends on the target (dbt's default generate_schema_name joins the target schema to a custom one) is reported as a gap (WSP099) that asks you to set this field.",
+          "pattern": "^.+$"
+        }
+      }
+    },
+    {
+      "name": "goose",
+      "description": "a goose migrations directory, with .sql files that carry -- +goose Up (and optionally -- +goose Down) sections; the migrations are listed in version order and goose is never run",
+      "precedence": 300,
+      "probe": {
+        "anyLine": {
+          "in": [
+            "*.sql"
+          ],
+          "starts": [
+            "-- +goose up"
+          ]
+        }
+      },
+      "reader": "goose-dir",
+      "fields": {
+        "dialect": {
+          "type": "string",
+          "description": "The SQL dialect the migrations are written in. The tool takes it from a driver or URL the directory does not hold, so a goose or golang-migrate member names it; unset, postgres.",
+          "pattern": "^(postgres|clickhouse)$"
+        },
+        "migrationsDir": {
+          "type": "string",
+          "description": "The directory under the member's own that holds the migration files. Only the member's own directory is probed, so a member whose files sit in a subdirectory names its kind in its entry.",
+          "default": ".",
+          "pattern": "^(?!/)(?!(.*/)?\\.\\.(/|$)).+$"
+        }
+      }
+    },
+    {
+      "name": "golang-migrate",
+      "description": "a golang-migrate directory, with <version>_<name>.up.sql and .down.sql pairs; the up files are listed in version order and migrate is never run",
+      "precedence": 300,
+      "probe": {
+        "anyFile": [
+          "*.up.sql"
+        ]
+      },
+      "reader": "golang-migrate-dir",
+      "fields": {
+        "dialect": {
+          "type": "string",
+          "description": "The SQL dialect the migrations are written in. The tool takes it from a driver or URL the directory does not hold, so a goose or golang-migrate member names it; unset, postgres.",
+          "pattern": "^(postgres|clickhouse)$"
+        },
+        "migrationsDir": {
+          "type": "string",
+          "description": "The directory under the member's own that holds the migration files. Only the member's own directory is probed, so a member whose files sit in a subdirectory names its kind in its entry.",
+          "default": ".",
+          "pattern": "^(?!/)(?!(.*/)?\\.\\.(/|$)).+$"
+        }
+      }
+    },
+    {
+      "name": "clickhouse-migrations",
+      "description": "a directory for the Node or Python clickhouse-migrations tool, with <version>_<name>.sql files; they are listed in version order and the tool is never run",
+      "precedence": 100,
+      "probe": {
+        "anyFile": [
+          "[0-9]*_*.sql"
+        ]
+      },
+      "reader": "clickhouse-migrations-dir",
+      "fields": {
+        "migrationsDir": {
+          "type": "string",
+          "description": "The directory under the member's own that holds the migration files. Only the member's own directory is probed, so a member whose files sit in a subdirectory names its kind in its entry.",
+          "default": ".",
+          "pattern": "^(?!/)(?!(.*/)?\\.\\.(/|$)).+$"
+        }
+      }
+    },
+    {
+      "name": "flyway",
+      "description": "a Flyway project, with a flyway.toml or flyway.conf, or V<version>__<description>.sql files; the migrations of its locations are listed in the order flyway migrate applies them and Flyway is never run",
+      "precedence": 300,
+      "probe": {
+        "anyFile": [
+          "flyway.toml",
+          "flyway.conf",
+          "V*__*.sql",
+          "R__*.sql"
+        ]
+      },
+      "reader": "flyway-dir",
+      "fields": {
+        "dialect": {
+          "type": "string",
+          "description": "The SQL dialect the migrations are written in. Unset, chant reads it from the JDBC URL in flyway.toml or flyway.conf, and falls back to postgres.",
+          "pattern": "^(postgres|clickhouse)$"
         }
       }
     }