@intentius/chant-lexicon-sql 0.124.31 → 0.124.33

package/dist/composites/catalog.js

@@ -0,0 +1,524 @@
+export const compositeCatalog = [
+    {
+        "name": "AuditLogTable",
+        "lexicon": "sql",
+        "description": "An append-only audit log partitioned by month on its timestamp, with a default partition and an index by actor.",
+        "bundles": [
+            "PostgresIndex",
+            "PostgresTable"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The table's name, unqualified."
+            },
+            {
+                "name": "schema",
+                "type": "PostgresSchema | string",
+                "required": false,
+                "description": "The schema the table and its partitions live in: the entity, or its name (default `public`)."
+            },
+            {
+                "name": "timestamp",
+                "type": "string",
+                "required": false,
+                "description": "The event time column, the partition key (default `occurred_at`)."
+            },
+            {
+                "name": "actor",
+                "type": "string",
+                "required": false,
+                "description": "The column naming who acted (default `actor`)."
+            }
+        ]
+    },
+    {
+        "name": "CdcMirror",
+        "lexicon": "sql",
+        "description": "A ReplacingMergeTree mirror of a CDC-fed source table, with a view of the rows not deleted.",
+        "bundles": [
+            "ClickHouseTable",
+            "ClickHouseView"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The mirror table's name."
+            },
+            {
+                "name": "columns",
+                "type": "string",
+                "required": true,
+                "description": "The source table's columns, as SQL, without the version and deleted columns."
+            },
+            {
+                "name": "primaryKey",
+                "type": "string",
+                "required": true,
+                "description": "The source table's primary key, the mirror's sort key: `id` or `(tenant_id, id)`."
+            },
+            {
+                "name": "version",
+                "type": "string",
+                "required": false,
+                "description": "The version column the pipeline writes (default `_version`, UInt64)."
+            },
+            {
+                "name": "deleted",
+                "type": "string",
+                "required": false,
+                "description": "The deleted flag the pipeline writes (default `_is_deleted`, UInt8, 1 for a delete)."
+            },
+            {
+                "name": "partitionBy",
+                "type": "string",
+                "required": false,
+                "description": "A `PARTITION BY` expression."
+            }
+        ]
+    },
+    {
+        "name": "EventsTable",
+        "lexicon": "sql",
+        "description": "An append-only MergeTree events table, partitioned by its timestamp and expired by TTL.",
+        "bundles": [
+            "ClickHouseTable"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The table's name, `name` or `database.name`."
+            },
+            {
+                "name": "columns",
+                "type": "string",
+                "required": true,
+                "description": "Every column except the timestamp, as SQL: `user_id UUID, kind LowCardinality(String)`."
+            },
+            {
+                "name": "orderBy",
+                "type": "string",
+                "required": true,
+                "description": "The sort key."
+            },
+            {
+                "name": "timestamp",
+                "type": "string",
+                "required": false,
+                "description": "The timestamp column (default `ts`)."
+            },
+            {
+                "name": "timestampType",
+                "type": "string",
+                "required": false,
+                "description": "The timestamp's type (default `DateTime`): a Date, DateTime or DateTime64."
+            },
+            {
+                "name": "ttlDays",
+                "type": "number",
+                "required": false,
+                "description": "Days a row is kept (default 90)."
+            },
+            {
+                "name": "partitionBy",
+                "type": "string",
+                "required": false,
+                "description": "The partition key (default `toYYYYMM(<timestamp>)`)."
+            }
+        ]
+    },
+    {
+        "name": "JoinTable",
+        "lexicon": "sql",
+        "description": "The table of a many-to-many relation: two foreign keys, a composite primary key and the reverse index.",
+        "bundles": [
+            "PostgresIndex",
+            "PostgresTable"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The table's name, unqualified."
+            },
+            {
+                "name": "schema",
+                "type": "PostgresSchema | string",
+                "required": false,
+                "description": "The schema the table lives in: the entity, or its name (default `public`)."
+            },
+            {
+                "name": "left",
+                "type": "PostgresTable",
+                "required": true,
+                "description": "The table on the left of the relation, as the entity."
+            },
+            {
+                "name": "leftKey",
+                "type": "string",
+                "required": false,
+                "description": "The column on the left table the key references (default `id`)."
+            },
+            {
+                "name": "leftColumn",
+                "type": "string",
+                "required": true,
+                "description": "The join table's column holding the left key: `user_id`."
+            },
+            {
+                "name": "leftType",
+                "type": "string",
+                "required": false,
+                "description": "The left column's type (default `bigint`)."
+            },
+            {
+                "name": "right",
+                "type": "PostgresTable",
+                "required": true,
+                "description": "The table on the right of the relation, as the entity."
+            },
+            {
+                "name": "rightKey",
+                "type": "string",
+                "required": false,
+                "description": "The column on the right table the key references (default `id`)."
+            },
+            {
+                "name": "rightColumn",
+                "type": "string",
+                "required": true,
+                "description": "The join table's column holding the right key: `team_id`."
+            },
+            {
+                "name": "rightType",
+                "type": "string",
+                "required": false,
+                "description": "The right column's type (default `bigint`)."
+            },
+            {
+                "name": "onDelete",
+                "type": "string",
+                "required": false,
+                "description": "What happens to a link when the row it references is deleted (default `CASCADE`)."
+            }
+        ]
+    },
+    {
+        "name": "RefreshedView",
+        "lexicon": "sql",
+        "description": "A materialized view over a source, with the unique index that allows REFRESH ...",
+        "bundles": [
+            "PostgresIndex",
+            "PostgresView"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The view's name, unqualified."
+            },
+            {
+                "name": "schema",
+                "type": "PostgresSchema | string",
+                "required": false,
+                "description": "The schema the view lives in: the entity, or its name (default `public`)."
+            },
+            {
+                "name": "source",
+                "type": "PostgresRelation",
+                "required": true,
+                "description": "The table or view the materialized view reads, as the entity."
+            },
+            {
+                "name": "select",
+                "type": "string",
+                "required": true,
+                "description": "The select list, each computed item aliased: `region, count(*) AS orders, sum(total) AS revenue`."
+            },
+            {
+                "name": "groupBy",
+                "type": "string",
+                "required": true,
+                "description": "The grouping key: `region`."
+            },
+            {
+                "name": "uniqueOn",
+                "type": "string",
+                "required": false,
+                "description": "The columns of the unique index, which must identify one row of the result (default `groupBy`)."
+            }
+        ]
+    },
+    {
+        "name": "ReplacingTable",
+        "lexicon": "sql",
+        "description": "A ReplacingMergeTree table that keeps the highest version of each row, with its version column.",
+        "bundles": [
+            "ClickHouseTable"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The table's name, `name` or `database.name`."
+            },
+            {
+                "name": "columns",
+                "type": "string",
+                "required": true,
+                "description": "Every column except the version column, as SQL: `id UInt64, email String`."
+            },
+            {
+                "name": "orderBy",
+                "type": "string",
+                "required": true,
+                "description": "The sort key, which is also the key rows are deduplicated by: `id` or `(tenant_id, id)`."
+            },
+            {
+                "name": "version",
+                "type": "string",
+                "required": false,
+                "description": "The version column (default `version`)."
+            },
+            {
+                "name": "versionType",
+                "type": "string",
+                "required": false,
+                "description": "The version column's type (default `UInt64`): a UInt, Date, DateTime or DateTime64."
+            },
+            {
+                "name": "partitionBy",
+                "type": "string",
+                "required": false,
+                "description": "A `PARTITION BY` expression."
+            }
+        ]
+    },
+    {
+        "name": "RollupView",
+        "lexicon": "sql",
+        "description": "A materialized view that rolls a source table up into a target table it declares beside it.",
+        "bundles": [
+            "ClickHouseTable",
+            "ClickHouseView"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The target table's name."
+            },
+            {
+                "name": "source",
+                "type": "ClickHouseRelation",
+                "required": true,
+                "description": "The table or view the rollup reads, as the entity: `events`."
+            },
+            {
+                "name": "columns",
+                "type": "string",
+                "required": true,
+                "description": "The target's columns, as SQL: `day Date, kind LowCardinality(String), n UInt64`."
+            },
+            {
+                "name": "select",
+                "type": "string",
+                "required": true,
+                "description": "The select list, each item aliased to a target column: `toDate(ts) AS day, kind, count() AS n`."
+            },
+            {
+                "name": "groupBy",
+                "type": "string",
+                "required": true,
+                "description": "The grouping key: `day, kind`."
+            },
+            {
+                "name": "orderBy",
+                "type": "string",
+                "required": false,
+                "description": "The target's sort key (default `groupBy` in parentheses)."
+            },
+            {
+                "name": "engine",
+                "type": "string",
+                "required": false,
+                "description": "The target's engine (default `SummingMergeTree`)."
+            },
+            {
+                "name": "partitionBy",
+                "type": "string",
+                "required": false,
+                "description": "The target's partition key."
+            }
+        ]
+    },
+    {
+        "name": "ShardedTable",
+        "lexicon": "sql",
+        "description": "A ReplicatedMergeTree table on every node of a cluster and the Distributed table over it.",
+        "bundles": [
+            "ClickHouseTable"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The Distributed table's name, unqualified."
+            },
+            {
+                "name": "cluster",
+                "type": "string",
+                "required": true,
+                "description": "The cluster both tables are created on, as the server's `remote_servers` names it."
+            },
+            {
+                "name": "columns",
+                "type": "string",
+                "required": true,
+                "description": "The columns, as SQL."
+            },
+            {
+                "name": "orderBy",
+                "type": "string",
+                "required": true,
+                "description": "The local table's sort key."
+            },
+            {
+                "name": "shardingKey",
+                "type": "string",
+                "required": false,
+                "description": "The expression that picks a shard for an inserted row (default `rand()`)."
+            },
+            {
+                "name": "partitionBy",
+                "type": "string",
+                "required": false,
+                "description": "The local table's partition key."
+            }
+        ]
+    },
+    {
+        "name": "SoftDeleteTable",
+        "lexicon": "sql",
+        "description": "A table with created and updated timestamps and a soft-delete column, a view of its live rows and an index over them.",
+        "bundles": [
+            "PostgresIndex",
+            "PostgresTable",
+            "PostgresView"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The table's name, unqualified."
+            },
+            {
+                "name": "schema",
+                "type": "PostgresSchema | string",
+                "required": false,
+                "description": "The schema the table, its index and its view live in: the entity, or its name (default `public`)."
+            },
+            {
+                "name": "columns",
+                "type": "string",
+                "required": true,
+                "description": "Every column except the three the composite adds, as SQL, with the primary key: `id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, title text NOT NULL`."
+            },
+            {
+                "name": "liveKey",
+                "type": "string",
+                "required": false,
+                "description": "The column the live-rows index covers: the one the app looks rows up by, as SQL (default `id`)."
+            },
+            {
+                "name": "comment",
+                "type": "string",
+                "required": false,
+                "description": "The table's comment, which also covers its columns for SQLPG112 (default: says rows are soft-deleted)."
+            },
+            {
+                "name": "createdAt",
+                "type": "string",
+                "required": false,
+                "description": "The creation timestamp column (default `created_at`)."
+            },
+            {
+                "name": "updatedAt",
+                "type": "string",
+                "required": false,
+                "description": "The update timestamp column (default `updated_at`)."
+            },
+            {
+                "name": "deletedAt",
+                "type": "string",
+                "required": false,
+                "description": "The deletion timestamp column (default `deleted_at`)."
+            }
+        ]
+    },
+    {
+        "name": "TenantTable",
+        "lexicon": "sql",
+        "description": "A multi-tenant table with the tenant key first in its primary key and its index.",
+        "bundles": [
+            "PostgresIndex",
+            "PostgresTable"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The table's name, unqualified."
+            },
+            {
+                "name": "schema",
+                "type": "PostgresSchema | string",
+                "required": false,
+                "description": "The schema the table lives in: the entity, or its name (default `public`)."
+            },
+            {
+                "name": "columns",
+                "type": "string",
+                "required": true,
+                "description": "Every column except the tenant's, as SQL: `id bigint GENERATED ALWAYS AS IDENTITY, title text NOT NULL`."
+            },
+            {
+                "name": "primaryKey",
+                "type": "string",
+                "required": true,
+                "description": "The columns that, after the tenant, make a row unique: `id`."
+            },
+            {
+                "name": "indexOn",
+                "type": "string",
+                "required": true,
+                "description": "The columns the secondary index covers after the tenant: `created_at DESC`."
+            },
+            {
+                "name": "tenant",
+                "type": "string",
+                "required": false,
+                "description": "The tenant column (default `tenant_id`)."
+            },
+            {
+                "name": "tenantType",
+                "type": "string",
+                "required": false,
+                "description": "The tenant column's type (default `uuid`)."
+            }
+        ]
+    }
+];

package/dist/composites/cdc-mirror.js

@@ -0,0 +1,32 @@
+/**
+ * `CdcMirror`: a ReplacingMergeTree mirror of a source database table fed by change data capture, with a view of its live rows.
+ *
+ * A CDC pipeline (ClickPipes, Debezium through Kafka, PeerDB) writes every
+ * change to a source row as a new row here, with an increasing version and a
+ * deleted flag. ReplacingMergeTree with both arguments keeps the latest
+ * version per primary key and, on merge, drops rows whose latest version is a
+ * delete. The `current` view reads the table with `FINAL` and without deleted
+ * rows, which is what a query of the source table would have returned.
+ */
+import { Composite } from "@intentius/chant/composite";
+import { table, view } from "../clickhouse/entities.js";
+/** A ReplacingMergeTree mirror of a CDC-fed source table, with a view of the rows not deleted. */
+export const CdcMirror = Composite((props) => {
+    const version = props.version ?? "_version";
+    const deleted = props.deleted ?? "_is_deleted";
+    const mirror = table `
+    CREATE TABLE ${props.name} (
+      ${props.columns},
+      ${version} UInt64,
+      ${deleted} UInt8
+    )
+    ENGINE = ReplacingMergeTree(${version}, ${deleted})
+    ${props.partitionBy ? `PARTITION BY ${props.partitionBy}` : ""}
+    ORDER BY ${props.primaryKey}`;
+    const current = view `
+    CREATE VIEW ${`${props.name}_current`} AS
+    SELECT * EXCEPT (${version}, ${deleted})
+    FROM ${mirror} FINAL
+    WHERE ${deleted} = 0`;
+    return { table: mirror, current };
+}, "CdcMirror");

package/dist/composites/clickhouse.js

@@ -0,0 +1,6 @@
+/** The ClickHouse composites, exported at `@intentius/chant-lexicon-sql/clickhouse`. */
+export { ReplacingTable } from "./replacing-table.js";
+export { EventsTable } from "./events-table.js";
+export { RollupView } from "./rollup-view.js";
+export { CdcMirror } from "./cdc-mirror.js";
+export { ShardedTable } from "./sharded-table.js";

package/dist/composites/events-table.js

@@ -0,0 +1,26 @@
+/**
+ * `EventsTable`: an append-only MergeTree events table, partitioned by month and expired by TTL.
+ *
+ * The table is partitioned by its timestamp (by month unless `partitionBy`
+ * says otherwise) and drops rows older than `ttlDays`. It sets
+ * `ttl_only_drop_parts`, so expiry drops whole parts once every row in them
+ * has expired instead of rewriting parts to remove single rows, which is cheap
+ * when the partition and the TTL follow the same column.
+ */
+import { Composite } from "@intentius/chant/composite";
+import { table } from "../clickhouse/entities.js";
+/** An append-only MergeTree events table, partitioned by its timestamp and expired by TTL. */
+export const EventsTable = Composite((props) => {
+    const ts = props.timestamp ?? "ts";
+    const events = table `
+    CREATE TABLE ${props.name} (
+      ${props.columns},
+      ${ts} ${props.timestampType ?? "DateTime"}
+    )
+    ENGINE = MergeTree
+    PARTITION BY ${props.partitionBy ?? `toYYYYMM(${ts})`}
+    ORDER BY ${props.orderBy}
+    TTL ${ts} + INTERVAL ${props.ttlDays ?? 90} DAY
+    SETTINGS ttl_only_drop_parts = 1`;
+    return { table: events };
+}, "EventsTable");

package/dist/composites/index.js

@@ -0,0 +1,8 @@
+/**
+ * The sql lexicon's composites: tables and views that are usually declared
+ * together, built from each dialect's `table`, `index` and `view` tags so every
+ * field keeps its provenance. `./clickhouse` and `./postgres` are what the two
+ * dialect subpaths export.
+ */
+export * from "./clickhouse.js";
+export * from "./postgres.js";

package/dist/composites/join-table.js

@@ -0,0 +1,25 @@
+/**
+ * `JoinTable`: the table of a many-to-many relation, with both foreign keys indexed.
+ *
+ * The primary key is `(<leftColumn>, <rightColumn>)`, which also serves
+ * lookups from the left side. The reverse index on `(<rightColumn>,
+ * <leftColumn>)` serves lookups from the right and keeps deletes of a
+ * referenced row off a table scan (SQLPG102). Each foreign key cascades by
+ * default, so deleting a row on either side removes its links.
+ */
+import { Composite } from "@intentius/chant/composite";
+import { index, table } from "../postgres/entities.js";
+/** The table of a many-to-many relation: two foreign keys, a composite primary key and the reverse index. */
+export const JoinTable = Composite((props) => {
+    const schema = props.schema ?? "public";
+    const links = table `
+    CREATE TABLE ${schema}.${props.name} (
+      ${props.leftColumn} ${props.leftType ?? "bigint"} NOT NULL REFERENCES ${props.left} (${props.leftKey ?? "id"}) ON DELETE ${props.onDelete ?? "CASCADE"},
+      ${props.rightColumn} ${props.rightType ?? "bigint"} NOT NULL REFERENCES ${props.right} (${props.rightKey ?? "id"}) ON DELETE ${props.onDelete ?? "CASCADE"},
+      created_at timestamptz NOT NULL DEFAULT now(),
+      PRIMARY KEY (${props.leftColumn}, ${props.rightColumn})
+    )`;
+    const reverse = index `
+    CREATE INDEX ${`${props.name}_reverse_idx`} ON ${links} (${props.rightColumn}, ${props.leftColumn})`;
+    return { table: links, reverseIndex: reverse };
+}, "JoinTable");

package/dist/composites/postgres.js

@@ -0,0 +1,6 @@
+/** The Postgres composites, exported at `@intentius/chant-lexicon-sql/postgres`. */
+export { SoftDeleteTable } from "./soft-delete-table.js";
+export { AuditLogTable } from "./audit-log-table.js";
+export { JoinTable } from "./join-table.js";
+export { TenantTable } from "./tenant-table.js";
+export { RefreshedView } from "./refreshed-view.js";

package/dist/composites/refreshed-view.js

@@ -0,0 +1,24 @@
+/**
+ * `RefreshedView`: a materialized view with the unique index that lets it refresh concurrently.
+ *
+ * `REFRESH MATERIALIZED VIEW CONCURRENTLY` needs at least one unique index
+ * over plain columns, and without it every refresh takes an ACCESS EXCLUSIVE
+ * lock that blocks readers (SQLPG111). The composite declares the view and
+ * that index together, so a view cannot be added without one. The view reads
+ * `source` by reference, so the build orders them and the view's lineage
+ * names the source's columns.
+ */
+import { Composite } from "@intentius/chant/composite";
+import { index, view } from "../postgres/entities.js";
+/** A materialized view over a source, with the unique index that allows REFRESH ... CONCURRENTLY. */
+export const RefreshedView = Composite((props) => {
+    const schema = props.schema ?? "public";
+    const summary = view `
+    CREATE MATERIALIZED VIEW ${schema}.${props.name} AS
+    SELECT ${props.select}
+    FROM ${props.source}
+    GROUP BY ${props.groupBy}`;
+    const key = index `
+    CREATE UNIQUE INDEX ${`${props.name}_key`} ON ${summary} (${props.uniqueOn ?? props.groupBy})`;
+    return { view: summary, uniqueIndex: key };
+}, "RefreshedView");