@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/composites/catalog.ts

@@ -0,0 +1,529 @@
+// Generated by scripts/generate-composite-catalogs.ts (#2662) from this
+// lexicon's composite exports. Do not edit by hand: run
+// `npm run generate:composite-catalogs`. catalog.test.ts holds it to the exports.
+import type { CompositeEntry } from "@intentius/chant/lexicon";
+
+export const compositeCatalog: CompositeEntry[] = [
+  {
+    "name": "AuditLogTable",
+    "lexicon": "sql",
+    "description": "An append-only audit log partitioned by month on its timestamp, with a default partition and an index by actor.",
+    "bundles": [
+      "PostgresIndex",
+      "PostgresTable"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The table's name, unqualified."
+      },
+      {
+        "name": "schema",
+        "type": "PostgresSchema | string",
+        "required": false,
+        "description": "The schema the table and its partitions live in: the entity, or its name (default `public`)."
+      },
+      {
+        "name": "timestamp",
+        "type": "string",
+        "required": false,
+        "description": "The event time column, the partition key (default `occurred_at`)."
+      },
+      {
+        "name": "actor",
+        "type": "string",
+        "required": false,
+        "description": "The column naming who acted (default `actor`)."
+      }
+    ]
+  },
+  {
+    "name": "CdcMirror",
+    "lexicon": "sql",
+    "description": "A ReplacingMergeTree mirror of a CDC-fed source table, with a view of the rows not deleted.",
+    "bundles": [
+      "ClickHouseTable",
+      "ClickHouseView"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The mirror table's name."
+      },
+      {
+        "name": "columns",
+        "type": "string",
+        "required": true,
+        "description": "The source table's columns, as SQL, without the version and deleted columns."
+      },
+      {
+        "name": "primaryKey",
+        "type": "string",
+        "required": true,
+        "description": "The source table's primary key, the mirror's sort key: `id` or `(tenant_id, id)`."
+      },
+      {
+        "name": "version",
+        "type": "string",
+        "required": false,
+        "description": "The version column the pipeline writes (default `_version`, UInt64)."
+      },
+      {
+        "name": "deleted",
+        "type": "string",
+        "required": false,
+        "description": "The deleted flag the pipeline writes (default `_is_deleted`, UInt8, 1 for a delete)."
+      },
+      {
+        "name": "partitionBy",
+        "type": "string",
+        "required": false,
+        "description": "A `PARTITION BY` expression."
+      }
+    ]
+  },
+  {
+    "name": "EventsTable",
+    "lexicon": "sql",
+    "description": "An append-only MergeTree events table, partitioned by its timestamp and expired by TTL.",
+    "bundles": [
+      "ClickHouseTable"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The table's name, `name` or `database.name`."
+      },
+      {
+        "name": "columns",
+        "type": "string",
+        "required": true,
+        "description": "Every column except the timestamp, as SQL: `user_id UUID, kind LowCardinality(String)`."
+      },
+      {
+        "name": "orderBy",
+        "type": "string",
+        "required": true,
+        "description": "The sort key."
+      },
+      {
+        "name": "timestamp",
+        "type": "string",
+        "required": false,
+        "description": "The timestamp column (default `ts`)."
+      },
+      {
+        "name": "timestampType",
+        "type": "string",
+        "required": false,
+        "description": "The timestamp's type (default `DateTime`): a Date, DateTime or DateTime64."
+      },
+      {
+        "name": "ttlDays",
+        "type": "number",
+        "required": false,
+        "description": "Days a row is kept (default 90)."
+      },
+      {
+        "name": "partitionBy",
+        "type": "string",
+        "required": false,
+        "description": "The partition key (default `toYYYYMM(<timestamp>)`)."
+      }
+    ]
+  },
+  {
+    "name": "JoinTable",
+    "lexicon": "sql",
+    "description": "The table of a many-to-many relation: two foreign keys, a composite primary key and the reverse index.",
+    "bundles": [
+      "PostgresIndex",
+      "PostgresTable"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The table's name, unqualified."
+      },
+      {
+        "name": "schema",
+        "type": "PostgresSchema | string",
+        "required": false,
+        "description": "The schema the table lives in: the entity, or its name (default `public`)."
+      },
+      {
+        "name": "left",
+        "type": "PostgresTable",
+        "required": true,
+        "description": "The table on the left of the relation, as the entity."
+      },
+      {
+        "name": "leftKey",
+        "type": "string",
+        "required": false,
+        "description": "The column on the left table the key references (default `id`)."
+      },
+      {
+        "name": "leftColumn",
+        "type": "string",
+        "required": true,
+        "description": "The join table's column holding the left key: `user_id`."
+      },
+      {
+        "name": "leftType",
+        "type": "string",
+        "required": false,
+        "description": "The left column's type (default `bigint`)."
+      },
+      {
+        "name": "right",
+        "type": "PostgresTable",
+        "required": true,
+        "description": "The table on the right of the relation, as the entity."
+      },
+      {
+        "name": "rightKey",
+        "type": "string",
+        "required": false,
+        "description": "The column on the right table the key references (default `id`)."
+      },
+      {
+        "name": "rightColumn",
+        "type": "string",
+        "required": true,
+        "description": "The join table's column holding the right key: `team_id`."
+      },
+      {
+        "name": "rightType",
+        "type": "string",
+        "required": false,
+        "description": "The right column's type (default `bigint`)."
+      },
+      {
+        "name": "onDelete",
+        "type": "string",
+        "required": false,
+        "description": "What happens to a link when the row it references is deleted (default `CASCADE`)."
+      }
+    ]
+  },
+  {
+    "name": "RefreshedView",
+    "lexicon": "sql",
+    "description": "A materialized view over a source, with the unique index that allows REFRESH ...",
+    "bundles": [
+      "PostgresIndex",
+      "PostgresView"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The view's name, unqualified."
+      },
+      {
+        "name": "schema",
+        "type": "PostgresSchema | string",
+        "required": false,
+        "description": "The schema the view lives in: the entity, or its name (default `public`)."
+      },
+      {
+        "name": "source",
+        "type": "PostgresRelation",
+        "required": true,
+        "description": "The table or view the materialized view reads, as the entity."
+      },
+      {
+        "name": "select",
+        "type": "string",
+        "required": true,
+        "description": "The select list, each computed item aliased: `region, count(*) AS orders, sum(total) AS revenue`."
+      },
+      {
+        "name": "groupBy",
+        "type": "string",
+        "required": true,
+        "description": "The grouping key: `region`."
+      },
+      {
+        "name": "uniqueOn",
+        "type": "string",
+        "required": false,
+        "description": "The columns of the unique index, which must identify one row of the result (default `groupBy`)."
+      }
+    ]
+  },
+  {
+    "name": "ReplacingTable",
+    "lexicon": "sql",
+    "description": "A ReplacingMergeTree table that keeps the highest version of each row, with its version column.",
+    "bundles": [
+      "ClickHouseTable"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The table's name, `name` or `database.name`."
+      },
+      {
+        "name": "columns",
+        "type": "string",
+        "required": true,
+        "description": "Every column except the version column, as SQL: `id UInt64, email String`."
+      },
+      {
+        "name": "orderBy",
+        "type": "string",
+        "required": true,
+        "description": "The sort key, which is also the key rows are deduplicated by: `id` or `(tenant_id, id)`."
+      },
+      {
+        "name": "version",
+        "type": "string",
+        "required": false,
+        "description": "The version column (default `version`)."
+      },
+      {
+        "name": "versionType",
+        "type": "string",
+        "required": false,
+        "description": "The version column's type (default `UInt64`): a UInt, Date, DateTime or DateTime64."
+      },
+      {
+        "name": "partitionBy",
+        "type": "string",
+        "required": false,
+        "description": "A `PARTITION BY` expression."
+      }
+    ]
+  },
+  {
+    "name": "RollupView",
+    "lexicon": "sql",
+    "description": "A materialized view that rolls a source table up into a target table it declares beside it.",
+    "bundles": [
+      "ClickHouseTable",
+      "ClickHouseView"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The target table's name."
+      },
+      {
+        "name": "source",
+        "type": "ClickHouseRelation",
+        "required": true,
+        "description": "The table or view the rollup reads, as the entity: `events`."
+      },
+      {
+        "name": "columns",
+        "type": "string",
+        "required": true,
+        "description": "The target's columns, as SQL: `day Date, kind LowCardinality(String), n UInt64`."
+      },
+      {
+        "name": "select",
+        "type": "string",
+        "required": true,
+        "description": "The select list, each item aliased to a target column: `toDate(ts) AS day, kind, count() AS n`."
+      },
+      {
+        "name": "groupBy",
+        "type": "string",
+        "required": true,
+        "description": "The grouping key: `day, kind`."
+      },
+      {
+        "name": "orderBy",
+        "type": "string",
+        "required": false,
+        "description": "The target's sort key (default `groupBy` in parentheses)."
+      },
+      {
+        "name": "engine",
+        "type": "string",
+        "required": false,
+        "description": "The target's engine (default `SummingMergeTree`)."
+      },
+      {
+        "name": "partitionBy",
+        "type": "string",
+        "required": false,
+        "description": "The target's partition key."
+      }
+    ]
+  },
+  {
+    "name": "ShardedTable",
+    "lexicon": "sql",
+    "description": "A ReplicatedMergeTree table on every node of a cluster and the Distributed table over it.",
+    "bundles": [
+      "ClickHouseTable"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The Distributed table's name, unqualified."
+      },
+      {
+        "name": "cluster",
+        "type": "string",
+        "required": true,
+        "description": "The cluster both tables are created on, as the server's `remote_servers` names it."
+      },
+      {
+        "name": "columns",
+        "type": "string",
+        "required": true,
+        "description": "The columns, as SQL."
+      },
+      {
+        "name": "orderBy",
+        "type": "string",
+        "required": true,
+        "description": "The local table's sort key."
+      },
+      {
+        "name": "shardingKey",
+        "type": "string",
+        "required": false,
+        "description": "The expression that picks a shard for an inserted row (default `rand()`)."
+      },
+      {
+        "name": "partitionBy",
+        "type": "string",
+        "required": false,
+        "description": "The local table's partition key."
+      }
+    ]
+  },
+  {
+    "name": "SoftDeleteTable",
+    "lexicon": "sql",
+    "description": "A table with created and updated timestamps and a soft-delete column, a view of its live rows and an index over them.",
+    "bundles": [
+      "PostgresIndex",
+      "PostgresTable",
+      "PostgresView"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The table's name, unqualified."
+      },
+      {
+        "name": "schema",
+        "type": "PostgresSchema | string",
+        "required": false,
+        "description": "The schema the table, its index and its view live in: the entity, or its name (default `public`)."
+      },
+      {
+        "name": "columns",
+        "type": "string",
+        "required": true,
+        "description": "Every column except the three the composite adds, as SQL, with the primary key: `id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, title text NOT NULL`."
+      },
+      {
+        "name": "liveKey",
+        "type": "string",
+        "required": false,
+        "description": "The column the live-rows index covers: the one the app looks rows up by, as SQL (default `id`)."
+      },
+      {
+        "name": "comment",
+        "type": "string",
+        "required": false,
+        "description": "The table's comment, which also covers its columns for SQLPG112 (default: says rows are soft-deleted)."
+      },
+      {
+        "name": "createdAt",
+        "type": "string",
+        "required": false,
+        "description": "The creation timestamp column (default `created_at`)."
+      },
+      {
+        "name": "updatedAt",
+        "type": "string",
+        "required": false,
+        "description": "The update timestamp column (default `updated_at`)."
+      },
+      {
+        "name": "deletedAt",
+        "type": "string",
+        "required": false,
+        "description": "The deletion timestamp column (default `deleted_at`)."
+      }
+    ]
+  },
+  {
+    "name": "TenantTable",
+    "lexicon": "sql",
+    "description": "A multi-tenant table with the tenant key first in its primary key and its index.",
+    "bundles": [
+      "PostgresIndex",
+      "PostgresTable"
+    ],
+    "params": [
+      {
+        "name": "name",
+        "type": "string",
+        "required": true,
+        "description": "The table's name, unqualified."
+      },
+      {
+        "name": "schema",
+        "type": "PostgresSchema | string",
+        "required": false,
+        "description": "The schema the table lives in: the entity, or its name (default `public`)."
+      },
+      {
+        "name": "columns",
+        "type": "string",
+        "required": true,
+        "description": "Every column except the tenant's, as SQL: `id bigint GENERATED ALWAYS AS IDENTITY, title text NOT NULL`."
+      },
+      {
+        "name": "primaryKey",
+        "type": "string",
+        "required": true,
+        "description": "The columns that, after the tenant, make a row unique: `id`."
+      },
+      {
+        "name": "indexOn",
+        "type": "string",
+        "required": true,
+        "description": "The columns the secondary index covers after the tenant: `created_at DESC`."
+      },
+      {
+        "name": "tenant",
+        "type": "string",
+        "required": false,
+        "description": "The tenant column (default `tenant_id`)."
+      },
+      {
+        "name": "tenantType",
+        "type": "string",
+        "required": false,
+        "description": "The tenant column's type (default `uuid`)."
+      }
+    ]
+  }
+];

package/src/composites/cdc-mirror.ts

@@ -0,0 +1,54 @@
+/**
+ * `CdcMirror`: a ReplacingMergeTree mirror of a source database table fed by change data capture, with a view of its live rows.
+ *
+ * A CDC pipeline (ClickPipes, Debezium through Kafka, PeerDB) writes every
+ * change to a source row as a new row here, with an increasing version and a
+ * deleted flag. ReplacingMergeTree with both arguments keeps the latest
+ * version per primary key and, on merge, drops rows whose latest version is a
+ * delete. The `current` view reads the table with `FINAL` and without deleted
+ * rows, which is what a query of the source table would have returned.
+ */
+
+import { Composite } from "@intentius/chant/composite";
+import { table, view, type ClickHouseTable, type ClickHouseView } from "../clickhouse/entities";
+
+export interface CdcMirrorProps {
+  /** The mirror table's name. The view of live rows is `<name>_current`. */
+  name: string;
+  /** The source table's columns, as SQL, without the version and deleted columns. */
+  columns: string;
+  /** The source table's primary key, the mirror's sort key: `id` or `(tenant_id, id)`. */
+  primaryKey: string;
+  /** The version column the pipeline writes (default `_version`, UInt64). */
+  version?: string;
+  /** The deleted flag the pipeline writes (default `_is_deleted`, UInt8, 1 for a delete). */
+  deleted?: string;
+  /** A `PARTITION BY` expression. Left out by default. */
+  partitionBy?: string;
+}
+
+export type CdcMirrorMembers = {
+  table: ClickHouseTable;
+  current: ClickHouseView;
+};
+
+/** A ReplacingMergeTree mirror of a CDC-fed source table, with a view of the rows not deleted. */
+export const CdcMirror = Composite<CdcMirrorProps, CdcMirrorMembers>((props) => {
+  const version = props.version ?? "_version";
+  const deleted = props.deleted ?? "_is_deleted";
+  const mirror = table`
+    CREATE TABLE ${props.name} (
+      ${props.columns},
+      ${version} UInt64,
+      ${deleted} UInt8
+    )
+    ENGINE = ReplacingMergeTree(${version}, ${deleted})
+    ${props.partitionBy ? `PARTITION BY ${props.partitionBy}` : ""}
+    ORDER BY ${props.primaryKey}`;
+  const current = view`
+    CREATE VIEW ${`${props.name}_current`} AS
+    SELECT * EXCEPT (${version}, ${deleted})
+    FROM ${mirror} FINAL
+    WHERE ${deleted} = 0`;
+  return { table: mirror, current };
+}, "CdcMirror");

package/src/composites/clickhouse.ts

@@ -0,0 +1,7 @@
+/** The ClickHouse composites, exported at `@intentius/chant-lexicon-sql/clickhouse`. */
+
+export { ReplacingTable, type ReplacingTableProps, type ReplacingTableMembers } from "./replacing-table";
+export { EventsTable, type EventsTableProps, type EventsTableMembers } from "./events-table";
+export { RollupView, type RollupViewProps, type RollupViewMembers } from "./rollup-view";
+export { CdcMirror, type CdcMirrorProps, type CdcMirrorMembers } from "./cdc-mirror";
+export { ShardedTable, type ShardedTableProps, type ShardedTableMembers } from "./sharded-table";