@intentius/chant-lexicon-sql 0.124.46 → 0.124.48

package/dist/migrate/from-postgres/translate.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Postgres tables into the ClickHouse tables a CDC pipeline fills (#4399).
+ *
+ * The input is the Postgres declarations a build reads (or a `.sql` file read
+ * into the same declarations), the output a {@link Mirror} per table: its
+ * ClickHouse columns, its sort key and the facts the writers need. Writing
+ * the mirror as DDL or as a `CdcMirror` call is `./emit.ts`.
+ *
+ * What is mechanical makes no finding: the type map in `./types.ts`, names,
+ * column order, a `NOT NULL` column keeping its type. Everything else is
+ * decided by the shared catalog (`../../clickhouse/review-catalog.ts`): the
+ * sort key from the primary key, the `ReplacingMergeTree` engine for a
+ * mirror, `Nullable` against a default, `LowCardinality` candidates, and the
+ * constraints ClickHouse does not enforce. This file adds what only Postgres
+ * has: identity, serial and generated columns, defaults, storage options,
+ * indexes that are not carried, and the objects of the file that are not
+ * tables.
+ *
+ * A mirror's columns carry no `DEFAULT`: the pipeline writes every column of
+ * a row, so the source's default has already run. That is an `info` finding
+ * (SQLRV008), not a `warning`, since it changes nothing a reader of the
+ * mirror sees.
+ *
+ * The Postgres schema is not part of the mirror's name. A pipeline chooses
+ * the ClickHouse database it writes to, so `shop.customers` becomes
+ * `customers` in that database. Two tables with one name in different
+ * schemas keep their schema as a database qualifier (`shop.customers`).
+ */
+import type { Declarable } from "@intentius/chant/declarable";
+import type { ImportDiagnostic } from "@intentius/chant/import/parser";
+/** The names of the columns the pipeline adds, and what `CdcMirror` calls them by default. */
+export declare const MIRROR_COLUMNS: {
+    readonly version: "_version";
+    readonly deleted: "_is_deleted";
+};
+export interface MirrorColumn {
+    name: string;
+    /** The ClickHouse type, `Nullable` and `LowCardinality` decisions included. */
+    type: string;
+    comment?: string;
+}
+export interface Mirror {
+    /** The export the Postgres table is declared as, or its name when the source has none. */
+    export: string;
+    /** The ClickHouse table name, qualified with a database only when two tables would collide. */
+    name: string;
+    /** The Postgres schema and table, for the workspace link. */
+    source: {
+        schema: string;
+        table: string;
+    };
+    columns: MirrorColumn[];
+    /** The sort key; empty means `tuple()`. */
+    primaryKey: string[];
+    /** MergeTree settings the key needs. */
+    settings: Record<string, string>;
+    /** `-- ` lines for constraints ClickHouse does not enforce. */
+    constraintComments: string[];
+    /** The table's description, which `CdcMirror` has no place for. */
+    comment?: string;
+}
+export interface Translation {
+    mirrors: Mirror[];
+    diagnostics: ImportDiagnostic[];
+    /** The Postgres outputs each mirrored column reads, as a workspace link names them (`shop.customers.email`). */
+    outputs: string[];
+}
+export interface TranslateOptions {
+    /** The mirror is a `CdcMirror` call: a table description has no place in it. */
+    composite: boolean;
+}
+export declare function translatePostgres(entities: ReadonlyMap<string, Declarable>, opts: TranslateOptions): Translation;
+//# sourceMappingURL=translate.d.ts.map

package/dist/migrate/from-postgres/translate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"translate.d.ts","sourceRoot":"","sources":["../../../src/migrate/from-postgres/translate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAgBvE,8FAA8F;AAC9F,eAAO,MAAM,cAAc;;;CAA2D,CAAC;AAEvF,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,+EAA+E;IAC/E,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,MAAM;IACrB,0FAA0F;IAC1F,MAAM,EAAE,MAAM,CAAC;IACf,+FAA+F;IAC/F,IAAI,EAAE,MAAM,CAAC;IACb,6DAA6D;IAC7D,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1C,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,+DAA+D;IAC/D,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAChC,gHAAgH;IAChH,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,gFAAgF;IAChF,SAAS,EAAE,OAAO,CAAC;CACpB;AAwOD,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI,EAAE,gBAAgB,GAAG,WAAW,CA0ChH"}

package/dist/migrate/from-postgres/translate.js

@@ -0,0 +1,261 @@
+/**
+ * Postgres tables into the ClickHouse tables a CDC pipeline fills (#4399).
+ *
+ * The input is the Postgres declarations a build reads (or a `.sql` file read
+ * into the same declarations), the output a {@link Mirror} per table: its
+ * ClickHouse columns, its sort key and the facts the writers need. Writing
+ * the mirror as DDL or as a `CdcMirror` call is `./emit.ts`.
+ *
+ * What is mechanical makes no finding: the type map in `./types.ts`, names,
+ * column order, a `NOT NULL` column keeping its type. Everything else is
+ * decided by the shared catalog (`../../clickhouse/review-catalog.ts`): the
+ * sort key from the primary key, the `ReplacingMergeTree` engine for a
+ * mirror, `Nullable` against a default, `LowCardinality` candidates, and the
+ * constraints ClickHouse does not enforce. This file adds what only Postgres
+ * has: identity, serial and generated columns, defaults, storage options,
+ * indexes that are not carried, and the objects of the file that are not
+ * tables.
+ *
+ * A mirror's columns carry no `DEFAULT`: the pipeline writes every column of
+ * a row, so the source's default has already run. That is an `info` finding
+ * (SQLRV008), not a `warning`, since it changes nothing a reader of the
+ * mirror sees.
+ *
+ * The Postgres schema is not part of the mirror's name. A pipeline chooses
+ * the ClickHouse database it writes to, so `shop.customers` becomes
+ * `customers` in that database. Two tables with one name in different
+ * schemas keep their schema as a database qualifier (`shop.customers`).
+ */
+import { constraintReview, engineReview, findingText, lowCardinalityReview, nullableReview, sortKeyReview, } from "../../clickhouse/review-catalog.js";
+import { isPostgresObject, POSTGRES_ENTITY_TYPES } from "../../postgres/entities.js";
+import { mapPostgresType } from "./types.js";
+/** The names of the columns the pipeline adds, and what `CdcMirror` calls them by default. */
+export const MIRROR_COLUMNS = { version: "_version", deleted: "_is_deleted" };
+const DEFAULT_SCHEMA = "public";
+const ident = (name) => (/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) ? name : `\`${name.replace(/\\/g, "\\\\").replace(/`/g, "\\`")}\``);
+const oneLine = (s) => s.replace(/\s+/g, " ").trim();
+class Report {
+    constructor() {
+        this.diagnostics = [];
+    }
+    finding(f) {
+        this.diagnostics.push({ ruleId: f.ruleId, severity: f.severity, message: findingText(f), entity: f.object });
+    }
+    /** A finding of this file's own. `error` marks a table the translation refuses; the catalog's findings never are. */
+    note(ruleId, severity, object, message, evidence, column, suggestion) {
+        const text = findingText({ ruleId, severity: severity === "error" ? "warning" : severity, object, ...(column ? { column } : {}), message, evidence, ...(suggestion ? { suggestion } : {}) });
+        this.diagnostics.push({ ruleId, severity, message: text, entity: object });
+    }
+}
+const propsOf = (e) => e.props;
+const display = (schema, name) => (schema ? `${schema}.${name}` : name);
+/** The Postgres tables among `entities`, in the order given, with the export each is declared as. */
+function tablesOf(entities) {
+    const out = [];
+    for (const [key, e] of entities)
+        if (isPostgresObject(e) && e.entityType === POSTGRES_ENTITY_TYPES.table)
+            out.push({ key, props: propsOf(e) });
+    return out;
+}
+/** Whether an index is on this table: its table is named bare, or with the schema, as the DDL wrote it. */
+const tableMatches = (ix, t) => {
+    const target = ix.tableName.replace(/"/g, "");
+    return target === t.name || target === `${t.schema ?? DEFAULT_SCHEMA}.${t.name}`;
+};
+function translateTable(key, t, indexes, mirrorName, opts, report) {
+    const schema = t.schema ?? DEFAULT_SCHEMA;
+    const object = display(t.schema, t.name);
+    const pk = new Set(t.primaryKey?.columns ?? []);
+    for (const reserved of Object.values(MIRROR_COLUMNS)) {
+        if (t.columns.some((c) => c.name === reserved)) {
+            report.note("SQLRV011", "error", object, `The table is not carried: it already has a column named ${reserved}, which the mirror adds.`, [`column ${reserved} in the source`], reserved, "Rename the source column, or name the pipeline's columns otherwise in the CdcMirror call.");
+            return undefined;
+        }
+    }
+    if (opts.composite && !/^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?$/.test(mirrorName)) {
+        report.note("SQLRV011", "error", object, `The table is not carried: CdcMirror writes the name ${mirrorName} unquoted, and it needs quoting.`, [`the name ${mirrorName} is not a plain identifier`], undefined, "Emit without --use-composites to keep the quoted name, or rename the table.");
+        return undefined;
+    }
+    if (t.partitionOf !== undefined) {
+        report.note("SQLRV011", "info", object, "A partition is not carried; its rows reach the mirror through the parent table.", [`PARTITION OF ${typeof t.partitionOf === "string" ? t.partitionOf : "the parent"}`]);
+        return undefined;
+    }
+    const unread = [];
+    if (t.like.length > 0)
+        unread.push(...t.like.map((l) => `LIKE ${l}`));
+    if (t.inherits.length > 0)
+        unread.push(`INHERITS (${t.inherits.map((i) => (typeof i === "string" ? i : "a declared table")).join(", ")})`);
+    if (t.ofType !== undefined)
+        unread.push(`OF ${typeof t.ofType === "string" ? t.ofType : "a declared type"}`);
+    if (unread.length > 0) {
+        report.note("SQLRV011", "warning", object, "Columns that come from LIKE, INHERITS or OF are not read.", unread, undefined, "Add the columns to the CdcMirror call by hand, or declare them in the table.");
+    }
+    const columnTypes = {};
+    const columns = [];
+    const maxLengths = new Map();
+    const droppedStorage = [];
+    for (const col of t.columns) {
+        const m = mapPostgresType(col.type ?? "");
+        if (m.note)
+            report.note("SQLRV007", "warning", object, m.note.message, m.note.evidence, col.name);
+        if (m.maxLength !== undefined)
+            maxLengths.set(col.name, m.maxLength);
+        describeGenerated(col, m.serial === true, object, report);
+        if (col.default !== undefined && col.generated === undefined && !(m.serial && /nextval\(/i.test(col.default))) {
+            report.note("SQLRV008", "info", object, `The default of ${col.name} is not carried.`, [`DEFAULT ${oneLine(col.default)}`, "the pipeline writes every column of a row, so the source has already run the default"], col.name, "Add a ClickHouse DEFAULT if the table is also written by something other than the pipeline.");
+        }
+        if (col.collate) {
+            report.note("SQLRV007", "warning", object, `Collation ${col.collate} on ${col.name} is not carried.`, ["ClickHouse compares String values as bytes; a Postgres collation orders them by locale"], col.name);
+        }
+        if (col.compression)
+            droppedStorage.push(`column ${col.name} COMPRESSION ${col.compression}`);
+        if (col.storage)
+            droppedStorage.push(`column ${col.name} STORAGE ${col.storage}`);
+        // A primary key column, an identity column and a serial column are NOT NULL whatever the DDL says.
+        const forcedNotNull = pk.has(col.name) || col.generated?.kind === "identity" || m.serial === true;
+        const sourceNullable = m.array === true ? false : !(col.notNull === true || forcedNotNull);
+        const n = nullableReview({ object, column: col.name, type: m.type, sourceNullable, ...(col.default !== undefined && sourceNullable ? { defaultExpr: oneLine(col.default) } : {}) });
+        if (n.finding)
+            report.finding(n.finding);
+        columnTypes[col.name] = n.type;
+        const lc = lowCardinalityReview({ object, column: col.name, type: n.type, ...(m.maxLength !== undefined ? { maxLength: m.maxLength } : {}) });
+        if (lc)
+            report.finding(lc);
+        columns.push({ name: col.name, type: n.type, ...(col.comment ? { comment: col.comment } : {}) });
+    }
+    if (columns.length === 0) {
+        report.note("SQLRV011", "error", object, "The table is not carried: it declares no column.", ["the CREATE TABLE lists no column"]);
+        return undefined;
+    }
+    // Sort key: the primary key, else the first unique key.
+    const uniqueIndex = indexes.find((i) => i.unique === true && i.elements.every((e) => e.column !== undefined));
+    const candidates = [];
+    if (t.primaryKey && t.primaryKey.columns.length > 0) {
+        candidates.push({ kind: "primary-key", ...(t.primaryKey.name ? { name: t.primaryKey.name } : {}), columns: t.primaryKey.columns, source: `PRIMARY KEY${t.primaryKey.name ? ` ${t.primaryKey.name}` : ""}` });
+    }
+    const firstUnique = t.uniques[0];
+    if (firstUnique)
+        candidates.push({ kind: "unique-key", ...(firstUnique.name ? { name: firstUnique.name } : {}), columns: firstUnique.columns, source: `UNIQUE${firstUnique.name ? ` ${firstUnique.name}` : ""}` });
+    else if (uniqueIndex)
+        candidates.push({ kind: "unique-key", name: uniqueIndex.name, columns: uniqueIndex.elements.map((e) => e.column), source: `CREATE UNIQUE INDEX ${uniqueIndex.name}` });
+    const sortKey = sortKeyReview({ object, candidates: candidates.filter((c) => c.columns.every((c2) => c2 in columnTypes)), columnTypes });
+    report.finding(sortKey.finding);
+    if (opts.composite && Object.keys(sortKey.settings).length > 0) {
+        report.note("SQLRV001", "warning", object, `The CdcMirror call has no SETTINGS, so ${Object.keys(sortKey.settings).join(", ")} is not set.`, [`the sort key has a Nullable column and needs ${Object.entries(sortKey.settings).map(([k, v]) => `${k} = ${v}`).join(", ")}`], undefined, "Emit without --use-composites to keep the setting, or make the key columns NOT NULL in Postgres.");
+    }
+    const engine = engineReview({
+        object,
+        mode: "cdc",
+        cdcColumns: { version: MIRROR_COLUMNS.version, deleted: MIRROR_COLUMNS.deleted },
+        ...(sortKey.orderBy.length > 0 ? { uniqueKey: sortKey.orderBy } : {}),
+        facts: [`the table is declared in Postgres${t.persistence ? ` as ${t.persistence}` : ""}`],
+    });
+    report.finding(engine.finding);
+    // Constraints ClickHouse does not enforce.
+    const facts = [];
+    for (const fk of t.foreignKeys) {
+        facts.push({ kind: "foreign-key", ...(fk.name ? { name: fk.name } : {}), columns: fk.columns, references: { table: fk.refTable, columns: fk.refColumns } });
+    }
+    for (const u of t.uniques)
+        facts.push({ kind: "unique", ...(u.name ? { name: u.name } : {}), columns: u.columns });
+    for (const ix of indexes) {
+        if (ix.unique === true && ix.elements.every((e) => e.column !== undefined))
+            facts.push({ kind: "unique", name: ix.name, columns: ix.elements.map((e) => e.column) });
+    }
+    for (const c of t.checks)
+        facts.push({ kind: "check", ...(c.name ? { name: c.name } : {}), expression: c.expr });
+    for (const x of t.exclusions)
+        facts.push({ kind: "exclusion", ...(x.name ? { name: x.name } : {}), expression: `${x.using ? `USING ${x.using} ` : ""}(${x.elements})` });
+    const constraints = constraintReview({ object, constraints: facts });
+    for (const f of constraints.findings)
+        report.finding(f);
+    // Indexes that are not carried.
+    for (const ix of indexes) {
+        if (ix.unique === true && ix.elements.every((e) => e.column !== undefined))
+            continue;
+        const on = ix.elements.map((e) => e.column ?? oneLine(e.expr)).join(", ");
+        report.note("SQLRV009", "warning", object, `${ix.name}${ix.unique ? ", a unique index," : ", an index,"} on (${on}) is not carried.`, [`CREATE ${ix.unique ? "UNIQUE " : ""}INDEX ${ix.name}${ix.method ? ` USING ${ix.method}` : ""}${ix.where ? ` WHERE ${oneLine(ix.where)}` : ""}`], undefined, "The sort key is the only index a MergeTree table has by default. A skip index (minmax, set, bloom_filter) on the column can prune granules.");
+    }
+    // Storage the source chose.
+    const dropped = [...droppedStorage];
+    if (t.partitionBy)
+        dropped.push(`PARTITION BY ${oneLine(t.partitionBy)}`);
+    if (t.with)
+        dropped.push(`WITH ${oneLine(t.with)}`);
+    if (t.tablespace)
+        dropped.push(`TABLESPACE ${t.tablespace}`);
+    if (t.using)
+        dropped.push(`USING ${t.using}`);
+    if (t.persistence)
+        dropped.push(t.persistence.toUpperCase());
+    if (t.rowSecurity === true)
+        dropped.push("ROW LEVEL SECURITY");
+    if (dropped.length > 0) {
+        report.note("SQLRV009", "warning", object, "Storage options of the source table are not carried.", dropped, undefined, "Choose PARTITION BY for the mirror from how it is queried; a source partition is by the write path, a ClickHouse one by the read path.");
+    }
+    if (t.comment && opts.composite) {
+        report.note("SQLRV011", "info", object, "The table's description is not carried: the CdcMirror call has no place for it.", [`COMMENT ON TABLE: ${oneLine(t.comment)}`], undefined, "Emit without --use-composites to keep it as a COMMENT clause.");
+    }
+    return {
+        export: key,
+        name: mirrorName,
+        source: { schema, table: t.name },
+        columns,
+        primaryKey: sortKey.orderBy,
+        settings: sortKey.settings,
+        constraintComments: constraints.comments,
+        ...(t.comment && !opts.composite ? { comment: t.comment } : {}),
+    };
+}
+/** Identity, serial and generated columns: plain columns here, said once each. */
+function describeGenerated(col, serial, object, report) {
+    if (col.generated?.kind === "identity") {
+        report.note("SQLRV006", "info", object, `Identity column ${col.name} is a plain column here.`, [`GENERATED ${col.generated.always ? "ALWAYS" : "BY DEFAULT"} AS IDENTITY${col.generated.options ? ` (${oneLine(col.generated.options)})` : ""}`, "ClickHouse has no auto-increment column, and the pipeline loads the value the source made"], col.name);
+    }
+    else if (col.generated !== undefined) {
+        report.note("SQLRV006", "warning", object, `Generated column ${col.name} is a plain column here.`, [`GENERATED ALWAYS AS (${oneLine(col.generated.expr ?? "")}) ${col.generated.kind.toUpperCase()}`, "the pipeline loads the stored value; a virtual column is not sent by every pipeline"], col.name, col.generated.kind === "virtual" ? "Add the column as ALIAS or MATERIALIZED with the expression rewritten in ClickHouse SQL, if the pipeline does not send it." : undefined);
+    }
+    else if (serial) {
+        report.note("SQLRV006", "info", object, `Serial column ${col.name} is a plain integer column here.`, ["serial draws from a sequence, which ClickHouse does not have, and the pipeline loads the value the source made"], col.name);
+    }
+}
+/** What an object of the source that is not a table is called in a finding. */
+function describeObject(e) {
+    const props = propsOf(e);
+    return `${e.entityType.replace(/^Postgres::/, "").toLowerCase()} ${display(props.schema, props.name ?? "?")}`;
+}
+export function translatePostgres(entities, opts) {
+    const report = new Report();
+    const tables = tablesOf(entities);
+    const indexes = [...entities.values()].filter((e) => isPostgresObject(e) && e.entityType === POSTGRES_ENTITY_TYPES.index).map((e) => propsOf(e));
+    // Names: a table keeps its bare name unless another schema has a table of that name.
+    const counts = new Map();
+    for (const { props } of tables)
+        counts.set(props.name, (counts.get(props.name) ?? 0) + 1);
+    const schemas = new Set(tables.map(({ props }) => props.schema).filter((s) => s !== undefined && s !== DEFAULT_SCHEMA));
+    const mirrors = [];
+    for (const { key, props } of tables) {
+        const collides = (counts.get(props.name) ?? 0) > 1;
+        const name = collides ? display(props.schema ?? DEFAULT_SCHEMA, props.name) : props.name;
+        if (collides) {
+            report.note("SQLRV011", "info", display(props.schema, props.name), `The table keeps its schema as a database, ${name}, because two schemas declare a table of this name.`, ["the Postgres schema is otherwise not part of the mirror's name"]);
+        }
+        const mine = indexes.filter((ix) => tableMatches(ix, props));
+        const m = translateTable(key, props, mine, name, opts, report);
+        if (m)
+            mirrors.push(m);
+    }
+    for (const s of [...schemas].sort()) {
+        report.note("SQLRV011", "info", s, `Schema ${s} is not part of the mirror's name.`, ["a pipeline writes to one ClickHouse database, which the profile or the pipeline names"], undefined, `Create the tables in the database your pipeline targets (${s} is a common choice).`);
+    }
+    // Everything else the source declares: named once, with no decision to make.
+    for (const e of entities.values()) {
+        if (!isPostgresObject(e))
+            continue;
+        if (e.entityType === POSTGRES_ENTITY_TYPES.table || e.entityType === POSTGRES_ENTITY_TYPES.index || e.entityType === POSTGRES_ENTITY_TYPES.schema)
+            continue;
+        report.note("SQLRV011", "info", describeObject(e), `Not carried: ${describeObject(e)}.`, ["a mirror is made of tables; this object is not one"]);
+    }
+    const outputs = mirrors.flatMap((m) => m.columns.map((c) => `${m.source.schema}.${m.source.table}.${c.name}`));
+    return { mirrors, diagnostics: report.diagnostics, outputs };
+}

package/dist/migrate/from-postgres/types.d.ts

@@ -0,0 +1,51 @@
+/**
+ * The Postgres to ClickHouse type map (#4399).
+ *
+ * A type either maps with no judgment (`note` absent) or carries a `note`
+ * naming what the map could not keep. Only the first kind is mechanical: the
+ * translator writes no diagnostic for it. A type this file does not know (an
+ * enum, a domain, an extension type) maps to `String` with a note, because a
+ * mirror still has to hold the value.
+ *
+ * Source types and defaults are from the Postgres data types chapter
+ * (https://www.postgresql.org/docs/current/datatype.html): `timestamp` and
+ * `time` take a precision of 0 to 6 and default to microseconds, `numeric`
+ * with no precision holds any value, `money` is a fixed-point type with a
+ * locale-set scale, `float(n)` is `real` for n up to 24 and `double
+ * precision` above.
+ *
+ * Target types are ClickHouse types of the pinned server (`../../spec/pin.ts`);
+ * `types.test.ts` checks every family this file writes against the generated
+ * catalog.
+ */
+export interface MappedType {
+    ok: true;
+    /** The ClickHouse type, before `Nullable`. */
+    type: string;
+    /** The declared maximum length of a character type, when it has one. */
+    maxLength?: number;
+    /** The column is `serial`, `bigserial` or `smallserial`: an integer with a sequence behind it. */
+    serial?: boolean;
+    /** The column is an array. ClickHouse cannot make an `Array` `Nullable`. */
+    array?: boolean;
+    /** Present when the map is a judgment: what was lost or chosen. */
+    note?: {
+        message: string;
+        evidence: string[];
+    };
+}
+export type TypeMapping = MappedType;
+interface ParsedType {
+    /** Lower case words of the type, schema prefix and quotes removed: `timestamp with time zone`. */
+    base: string;
+    args: number[];
+    /** Array dimensions. */
+    dims: number;
+    raw: string;
+}
+/** Split a column type as written into its words, its parenthesised arguments and its array dimensions. */
+export declare function parseType(text: string): ParsedType;
+/** Map a column type as the declaration wrote it. */
+export declare function mapPostgresType(text: string): TypeMapping;
+export {};
+//# sourceMappingURL=types.d.ts.map

package/dist/migrate/from-postgres/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/migrate/from-postgres/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,IAAI,CAAC;IACT,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kGAAkG;IAClG,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,4EAA4E;IAC5E,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,mEAAmE;IACnE,IAAI,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CAChD;AAED,MAAM,MAAM,WAAW,GAAG,UAAU,CAAC;AAErC,UAAU,UAAU;IAClB,kGAAkG;IAClG,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACb;AAED,2GAA2G;AAC3G,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CA6BlD;AA0ID,qDAAqD;AACrD,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAoBzD"}

package/dist/migrate/from-postgres/types.js

@@ -0,0 +1,211 @@
+/**
+ * The Postgres to ClickHouse type map (#4399).
+ *
+ * A type either maps with no judgment (`note` absent) or carries a `note`
+ * naming what the map could not keep. Only the first kind is mechanical: the
+ * translator writes no diagnostic for it. A type this file does not know (an
+ * enum, a domain, an extension type) maps to `String` with a note, because a
+ * mirror still has to hold the value.
+ *
+ * Source types and defaults are from the Postgres data types chapter
+ * (https://www.postgresql.org/docs/current/datatype.html): `timestamp` and
+ * `time` take a precision of 0 to 6 and default to microseconds, `numeric`
+ * with no precision holds any value, `money` is a fixed-point type with a
+ * locale-set scale, `float(n)` is `real` for n up to 24 and `double
+ * precision` above.
+ *
+ * Target types are ClickHouse types of the pinned server (`../../spec/pin.ts`);
+ * `types.test.ts` checks every family this file writes against the generated
+ * catalog.
+ */
+/** Split a column type as written into its words, its parenthesised arguments and its array dimensions. */
+export function parseType(text) {
+    let rest = text.trim();
+    let dims = 0;
+    for (;;) {
+        const m = /\s*\[\s*\d*\s*\]\s*$/.exec(rest);
+        if (!m)
+            break;
+        rest = rest.slice(0, m.index);
+        dims++;
+    }
+    const arrayWord = /\s+array(\s*\[\s*\d*\s*\])?$/i.exec(rest);
+    if (arrayWord) {
+        rest = rest.slice(0, arrayWord.index);
+        dims++;
+    }
+    const args = [];
+    rest = rest.replace(/\(([^)]*)\)/g, (_all, inner) => {
+        for (const part of inner.split(",")) {
+            const n = Number(part.trim());
+            if (Number.isFinite(n))
+                args.push(n);
+        }
+        return " ";
+    });
+    const base = rest
+        .replace(/"/g, "")
+        .replace(/^pg_catalog\./i, "")
+        .replace(/\s+/g, " ")
+        .trim()
+        .toLowerCase();
+    return { base, args, dims, raw: text.trim() };
+}
+const STRING_NOTE = (what, evidence) => ({ message: `${what} is read as String.`, evidence });
+function mapScalar(t) {
+    const { base, args } = t;
+    switch (base) {
+        case "smallint":
+        case "int2":
+            return { ok: true, type: "Int16" };
+        case "integer":
+        case "int":
+        case "int4":
+            return { ok: true, type: "Int32" };
+        case "bigint":
+        case "int8":
+            return { ok: true, type: "Int64" };
+        case "smallserial":
+        case "serial2":
+            return { ok: true, type: "Int16", serial: true };
+        case "serial":
+        case "serial4":
+            return { ok: true, type: "Int32", serial: true };
+        case "bigserial":
+        case "serial8":
+            return { ok: true, type: "Int64", serial: true };
+        case "real":
+        case "float4":
+            return { ok: true, type: "Float32" };
+        case "double precision":
+        case "float8":
+            return { ok: true, type: "Float64" };
+        case "float":
+            return { ok: true, type: (args[0] ?? 53) <= 24 ? "Float32" : "Float64" };
+        case "numeric":
+        case "decimal": {
+            if (args.length === 0) {
+                return {
+                    ok: true,
+                    type: "Decimal(38, 18)",
+                    note: {
+                        message: "numeric with no precision is read as Decimal(38, 18).",
+                        evidence: ["Postgres stores up to 131072 digits before the point and 16383 after; a ClickHouse Decimal has a fixed precision and scale", "a value that does not fit fails the insert"],
+                    },
+                };
+            }
+            const p = args[0];
+            const s = args[1] ?? 0;
+            if (p > 76 || s < 0 || s > p) {
+                return {
+                    ok: true,
+                    type: "Decimal(76, 38)",
+                    note: { message: `numeric(${String(p)}, ${String(s)}) is read as Decimal(76, 38).`, evidence: ["ClickHouse Decimal holds at most 76 digits"] },
+                };
+            }
+            return { ok: true, type: `Decimal(${String(p)}, ${String(s)})` };
+        }
+        case "money":
+            return {
+                ok: true,
+                type: "Decimal(19, 2)",
+                note: { message: "money is read as Decimal(19, 2).", evidence: ["money takes its scale from the lc_monetary setting of the source server, and two decimals is the common one"] },
+            };
+        case "boolean":
+        case "bool":
+            return { ok: true, type: "Bool" };
+        case "text":
+        case "varchar":
+        case "character varying":
+        case "name":
+        case "citext":
+            return { ok: true, type: "String", ...(args[0] !== undefined ? { maxLength: args[0] } : {}) };
+        case "char":
+        case "character":
+        case "bpchar":
+            return {
+                ok: true,
+                type: "String",
+                ...(args[0] !== undefined ? { maxLength: args[0] } : {}),
+                ...(args[0] !== undefined && args[0] > 1 ? { note: STRING_NOTE("char(n)", ["Postgres pads a char(n) value with spaces to n characters; a String keeps the value as it is loaded"]) } : {}),
+            };
+        case "bytea":
+            return { ok: true, type: "String" };
+        case "uuid":
+            return { ok: true, type: "UUID" };
+        case "date":
+            return { ok: true, type: "Date32" };
+        case "time":
+        case "time without time zone":
+            return { ok: true, type: `Time64(${String(args[0] ?? 6)})` };
+        case "timetz":
+        case "time with time zone":
+            return { ok: true, type: "String", note: STRING_NOTE("timetz", ["a time of day with an offset has no ClickHouse type; the offset is kept in the text"]) };
+        case "timestamp":
+        case "timestamp without time zone":
+            return { ok: true, type: `DateTime64(${String(args[0] ?? 6)})` };
+        case "timestamptz":
+        case "timestamp with time zone":
+            return { ok: true, type: `DateTime64(${String(args[0] ?? 6)}, 'UTC')` };
+        case "interval":
+            return { ok: true, type: "String", note: STRING_NOTE("interval", ["a ClickHouse Interval type cannot be stored in a column", "the value is kept as Postgres prints it"]) };
+        case "json":
+        case "jsonb":
+            return { ok: true, type: "JSON" };
+        case "oid":
+            return { ok: true, type: "UInt32" };
+        case "inet":
+        case "cidr":
+            return {
+                ok: true,
+                type: "IPv6",
+                note: { message: `${base} is read as IPv6.`, evidence: ["an IPv6 column holds an IPv4 address in its mapped form", "the prefix length of a network is not kept"] },
+            };
+        case "xml":
+        case "macaddr":
+        case "macaddr8":
+        case "tsvector":
+        case "tsquery":
+        case "bit":
+        case "bit varying":
+        case "varbit":
+        case "point":
+        case "line":
+        case "lseg":
+        case "box":
+        case "path":
+        case "polygon":
+        case "circle":
+            return { ok: true, type: "String", note: STRING_NOTE(base, [`ClickHouse has no ${base} type`, "the value is kept as Postgres prints it"]) };
+        default:
+            return {
+                ok: true,
+                type: "String",
+                note: STRING_NOTE(t.raw, [`${t.raw} is not a built-in type chant maps (an enum, a domain, a composite or an extension type)`, "the value is kept as Postgres prints it; an enum could be an Enum8 or Enum16 once its labels are settled"]),
+            };
+    }
+}
+/** Map a column type as the declaration wrote it. */
+export function mapPostgresType(text) {
+    const t = parseType(text);
+    const inner = mapScalar(t);
+    if (t.dims === 0)
+        return inner;
+    let type = inner.type;
+    for (let i = 0; i < t.dims; i++)
+        type = `Array(${type})`;
+    return {
+        ok: true,
+        type,
+        array: true,
+        ...(inner.serial ? { serial: true } : {}),
+        note: {
+            message: `${t.raw} is read as ${type}.`,
+            evidence: [
+                ...(inner.note ? [inner.note.message] : []),
+                "ClickHouse cannot make an Array Nullable, so a NULL array is stored as an empty one",
+                "Postgres does not enforce the declared dimensions or lengths of an array",
+            ],
+        },
+    };
+}