@mkven/samples-generation 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Damir Manapov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,484 @@
+# samples-generation
+
+Generate sample data for multiple databases with a unified interface.
+
+## Objective
+
+We often need to prefill tables during tests, checks, and measurements. These generators support filling tables with random data and applying transformations (template-based construction of values from other columns, lookups in another table, data corruptions, etc). May be used for simple prefill and for controlled corruptions for further testing entity resolution.
+
+## Supported Databases
+
+- **PostgreSQL** - via `postgres` package
+- **ClickHouse** - via `@clickhouse/client`
+- **SQLite** - via `better-sqlite3`
+- **Trino** - via `trino-client` (writes to Iceberg tables)
+
+## Installation
+
+```bash
+pnpm install
+```
+
+## Measurements of simple generations
+
+Environment: local databases, simple setup, 1 billion rows, 5 columns (id, 10-char string, 0 - 1000 float, string choice out of 3 variants, datetime)
+
+_ClickHouse:_ Generated in 11m 2s (generation: 6m 8s, optimisation: 4m 54s), table size: 23.81 GB
+
+_Trino:_ Generated in 5m 35s (generation: 5m 34s, optimisation: 120ms), table size: 17.41 GB
+
+_PostgreSQL:_ Generated in 54m 26s (generation: 35m 55s, optimisation: 18m 30s), table size: 63.60 GB
+
+Same setup but 10 billion rows:
+
+## Quick Start
+
+### Using the Generator API
+
+```typescript
+import {
+  PostgresDataGenerator,
+  ClickHouseDataGenerator,
+  SQLiteDataGenerator,
+  TrinoDataGenerator,
+  type TableConfig,
+} from "./src/generator/index.js";
+
+const table: TableConfig = {
+  name: "users",
+  columns: [
+    { name: "id", type: "integer", generator: { kind: "sequence", start: 1 } },
+    {
+      name: "name",
+      type: "string",
+      generator: { kind: "randomString", length: 10 },
+    },
+    {
+      name: "score",
+      type: "float",
+      generator: { kind: "randomFloat", min: 0, max: 100 },
+    },
+    {
+      name: "status",
+      type: "string",
+      generator: { kind: "choice", values: ["active", "inactive"] },
+    },
+    { name: "created_at", type: "datetime", generator: { kind: "datetime" } },
+  ],
+};
+
+// All generators have the same interface
+const generator = new PostgresDataGenerator({
+  host: "localhost",
+  port: 5432,
+  database: "appdb",
+  username: "postgres",
+  password: "postgres",
+});
+
+await generator.connect();
+const result = await generator.generate({
+  table,
+  rowCount: 1000,
+  truncateFirst: true,
+  resumeSequences: true, // Continue sequence from last max value
+});
+console.log(
+  `Inserted ${result.rowsInserted} rows in ${result.generateMs}ms (optimize: ${result.optimizeMs}ms)`
+);
+await generator.disconnect();
+```
+
+### Database Configurations
+
+All databases use a consistent `host`/`port` configuration:
+
+```typescript
+// PostgreSQL
+new PostgresDataGenerator({
+  host: "localhost",
+  port: 5432,
+  database: "appdb",
+  username: "postgres",
+  password: "postgres",
+});
+
+// ClickHouse
+new ClickHouseDataGenerator({
+  host: "localhost",
+  port: 8123,
+  database: "default",
+  username: "default",
+  password: "clickhouse",
+});
+
+// SQLite
+new SQLiteDataGenerator({
+  path: "data/samples.db",
+});
+
+// Trino/Iceberg
+new TrinoDataGenerator({
+  host: "localhost",
+  port: 8080,
+  catalog: "iceberg",
+  schema: "warehouse",
+  user: "trino",
+});
+```
+
+### Column Types
+
+| Type       | PostgreSQL       | ClickHouse | SQLite  | Trino     |
+| ---------- | ---------------- | ---------- | ------- | --------- |
+| `integer`  | INTEGER          | Int32      | INTEGER | INTEGER   |
+| `bigint`   | BIGINT           | Int64      | INTEGER | BIGINT    |
+| `float`    | DOUBLE PRECISION | Float64    | REAL    | DOUBLE    |
+| `string`   | TEXT             | String     | TEXT    | VARCHAR   |
+| `boolean`  | BOOLEAN          | Bool       | INTEGER | BOOLEAN   |
+| `datetime` | TIMESTAMP        | DateTime   | TEXT    | TIMESTAMP |
+| `date`     | DATE             | Date       | TEXT    | DATE      |
+
+### Column Options
+
+Each column can have additional options:
+
+| Option            | Type      | Default | Description                                      |
+| ----------------- | --------- | ------- | ------------------------------------------------ |
+| `nullable`        | `boolean` | `false` | If `true`, omits `NOT NULL` constraint on column |
+| `nullProbability` | `number`  | `0`     | Probability of NULL values (0-1)                 |
+
+Example with nullable column:
+
+```typescript
+{
+  name: "middle_name",
+  type: "string",
+  generator: { kind: "randomString", length: 10 },
+  nullable: true,
+  nullProbability: 0.3  // 30% of rows will have NULL
+}
+```
+
+### Value Generators
+
+| Generator        | Kind             | Options                                |
+| ---------------- | ---------------- | -------------------------------------- |
+| `sequence`       | Auto-increment   | `start`, `step`                        |
+| `randomInt`      | Random integer   | `min`, `max`                           |
+| `randomFloat`    | Random float     | `min`, `max`, `precision` (default: 2) |
+| `randomString`   | Random string    | `length`                               |
+| `choice`         | Pick from list   | `values`                               |
+| `choiceByLookup` | Optimized choice | `values` (large arrays)                |
+| `constant`       | Fixed value      | `value`                                |
+| `datetime`       | Random datetime  | `from`, `to`                           |
+| `uuid`           | UUID v4          | -                                      |
+
+#### `choiceByLookup` Generator
+
+Use `choiceByLookup` instead of `choice` when selecting from thousands of values. It uses CTEs with arrays for O(1) random selection, making it efficient for billions of rows:
+
+```typescript
+{
+  name: "last_name",
+  type: "string",
+  generator: {
+    kind: "choiceByLookup",
+    values: ["Smith", "Johnson", "Williams", ...] // thousands of values
+  }
+}
+```
+
+- PostgreSQL: CTE with `ARRAY[]` and `array_length()` indexing
+- ClickHouse: `WITH` clause with array variable
+- SQLite: CTE with JSON array and `json_extract()`
+- Trino: CTE with `ARRAY[]` and `element_at()`
+
+### Generate Options
+
+```typescript
+interface GenerateOptions {
+  table: TableConfig;
+  rowCount: number;
+  createTable?: boolean; // Default: true
+  dropFirst?: boolean; // Default: false - drop table before generating
+  truncateFirst?: boolean; // Default: false
+  resumeSequences?: boolean; // Default: true - continue from max value
+  optimize?: boolean; // Default: true - run VACUUM/OPTIMIZE after insert
+}
+```
+
+### Transformations
+
+Apply transformations to existing tables. Useful for creating derived columns (like email from first/last name) or introducing realistic data quality issues.
+
+```typescript
+// Generate data first
+await generator.generate({ table: usersTable, rowCount: 10000 });
+
+// Then apply transformations
+await generator.transform("users", [
+  {
+    description: "Generate email addresses",
+    transformations: [
+      {
+        kind: "template",
+        column: "email",
+        template: "{first_name}.{last_name}@example.com",
+      },
+    ],
+  },
+]);
+```
+
+```typescript
+interface TransformResult {
+  durationMs: number;
+  batchesApplied: number;
+}
+```
+
+#### Transformation Types
+
+**Template Transformation** - Build column values from other columns:
+
+```typescript
+{
+  kind: "template",
+  column: "email",
+  template: "{first_name}.{last_name}@example.com",
+  lowercase: true  // Optional: convert result to lowercase
+}
+```
+
+**Mutate Transformation** - Introduce random character mutations:
+
+```typescript
+{
+  kind: "mutate",
+  column: "name",
+  probability: 0.1,  // 10% of rows get mutated
+  operations: ["replace", "delete", "insert"]  // Random operation selected per row
+}
+```
+
+**Lookup Transformation** - Assign values from another table via join:
+
+```typescript
+{
+  kind: "lookup",
+  column: "category_name",      // Column to update
+  fromTable: "categories",      // Source table
+  fromColumn: "name",           // Column to copy value from
+  joinOn: {
+    targetColumn: "category_id", // Column in target table
+    lookupColumn: "id"           // Column in source table to match
+  }
+}
+```
+
+> **Note:** For ClickHouse, lookup transformation uses a table swap approach (CREATE → INSERT SELECT with JOIN → RENAME) since ClickHouse doesn't support correlated subqueries in `ALTER TABLE UPDATE`. This means lookups execute **before** other transformations in the same batch. If order matters, place lookups in a separate batch.
+
+**Swap Transformation** - Swap values between two columns with probability:
+
+```typescript
+{
+  kind: "swap",
+  column1: "first_name",
+  column2: "last_name",
+  probability: 0.1  // 10% of rows get swapped
+}
+```
+
+> **Note:** Both columns use the same random decision per row, ensuring atomic swaps (if column1 gets column2's value, column2 always gets column1's value). For ClickHouse, swap also uses the table swap approach (like lookup) since ClickHouse evaluates each `rand()` call separately. Multiple swaps in the same batch are combined into a single table swap operation for efficiency.
+
+> **Design Note:** PostgreSQL, SQLite, and Trino execute each swap as a separate `UPDATE ... WHERE random() < probability` statement. This is intentionally not batched because UPDATE is a lightweight operation on these databases. ClickHouse batches swaps because each swap would otherwise require a full table copy (CREATE → INSERT → RENAME → DROP), making the overhead significant.
+
+#### Batching Transformations
+
+Transformations are organized in batches for efficiency:
+
+- Each batch becomes a separate UPDATE statement (executed sequentially)
+- Transformations within a batch are combined into a single UPDATE
+- Batches support optional descriptions for logging and debugging
+
+```typescript
+await generator.transform("users", [
+  {
+    description: "Generate email addresses",
+    transformations: [
+      {
+        kind: "template",
+        column: "email",
+        template: "{first_name}.{last_name}@example.com",
+      },
+    ],
+  },
+  {
+    description: "Introduce data quality issues",
+    transformations: [
+      {
+        kind: "mutate",
+        column: "email",
+        probability: 0.1,
+        operations: ["replace"],
+      },
+    ],
+  },
+]);
+```
+
+With descriptions, you'll see helpful logs:
+
+```
+[postgres] Applying transformations: Generate email addresses (1 transformation(s))
+[postgres] Applying transformations: Introduce data quality issues (1 transformation(s))
+```
+
+### Escape Utilities
+
+For custom queries, use the exported escape functions:
+
+```typescript
+import {
+  escapePostgresIdentifier,
+  escapeClickHouseIdentifier,
+  escapeTrinoIdentifier,
+} from "./src/generator/index.js";
+
+escapePostgresIdentifier("my-table"); // "my-table"
+escapePostgresIdentifier('table"name'); // "table""name"
+
+escapeClickHouseIdentifier("my-table"); // `my-table`
+escapeClickHouseIdentifier("table`name"); // `table``name`
+
+escapeTrinoIdentifier("samples"); // "samples"
+escapeTrinoIdentifier("samples$files"); // "samples$files" (for metadata tables)
+```
+
+### Table Size
+
+Get the size of a table (including indexes):
+
+```typescript
+// Get size in bytes
+const bytes = await generator.getTableSize("users");
+// 1234567
+
+// Get human-readable size
+const size = await generator.getTableSizeForHuman("users");
+// "1.18 MB"
+```
+
+You can also use the `formatBytes` utility directly:
+
+```typescript
+import { formatBytes } from "./src/generator/index.js";
+
+formatBytes(1024); // "1.00 KB"
+formatBytes(1048576); // "1.00 MB"
+```
+
+### Optimization
+
+By default, `generate()` runs database-specific optimization after inserting rows:
+
+| Database   | Optimization                                                                |
+| ---------- | --------------------------------------------------------------------------- |
+| PostgreSQL | `VACUUM ANALYZE` - reclaims storage and updates statistics                  |
+| ClickHouse | `OPTIMIZE TABLE FINAL` - merges all parts for MergeTree engines             |
+| SQLite     | `VACUUM` + `ANALYZE` - rebuilds file and gathers statistics                 |
+| Trino      | `rewrite_data_files` + `expire_snapshots` + `remove_orphan_files` - Iceberg |
+
+Disable for quick tests:
+
+```typescript
+await generator.generate({
+  table,
+  rowCount: 1000,
+  optimize: false, // Skip VACUUM/OPTIMIZE
+});
+```
+
+Or call manually:
+
+```typescript
+await generator.optimize("users");
+```
+
+## Scripts
+
+### Generate Data
+
+```bash
+# Generate 1000 rows in all databases (requires docker-compose up)
+npx tsx scripts/generate-all.ts
+
+# Specify row count
+npx tsx scripts/generate-all.ts --rows 1000
+npx tsx scripts/generate-all.ts -r 1_000_000
+
+# Generate for specific databases only
+npx tsx scripts/generate-all.ts --sqlite
+npx tsx scripts/generate-all.ts --postgres
+npx tsx scripts/generate-all.ts --clickhouse
+npx tsx scripts/generate-all.ts --trino
+
+# Combine options
+npx tsx scripts/generate-all.ts -r 10000 --postgres --clickhouse
+
+# Show help
+npx tsx scripts/generate-all.ts --help
+```
+
+## Docker Compose
+
+Start all databases:
+
+```bash
+pnpm compose:up
+```
+
+Services available:
+
+| Service    | Port(s)                    | Credentials           |
+| ---------- | -------------------------- | --------------------- |
+| PostgreSQL | 5432                       | postgres:postgres     |
+| ClickHouse | 8123 (HTTP), 9009 (native) | default:clickhouse    |
+| Trino      | 8080                       | trino (no password)   |
+| MinIO      | 9000 (S3), 9001 (console)  | minioadmin:minioadmin |
+| Nessie     | 19120                      | -                     |
+
+## Testing
+
+```bash
+# Run tests (SQLite only by default)
+pnpm test
+
+# Run tests with specific databases
+TEST_POSTGRES=1 pnpm test
+TEST_CLICKHOUSE=1 pnpm test
+TEST_TRINO=1 pnpm test
+
+# Run all database tests
+TEST_POSTGRES=1 TEST_CLICKHOUSE=1 TEST_TRINO=1 pnpm test
+
+# Or use the shortcut script
+./test-all-dbs.sh
+```
+
+## Quality Checks
+
+```bash
+# Run formatting, linting, type checking, and tests
+./check.sh
+
+# Check for security vulnerabilities and outdated dependencies
+./health.sh
+
+# Check for dependency updates (requires npx renovate)
+./renovate-check.sh
+
+# Run all checks
+./all-checks.sh
+```

package/dist/generator/base-generator.d.ts

@@ -0,0 +1,39 @@
+import type { DataGenerator, TableConfig, GenerateOptions, GenerateResult, GeneratedRow, Transformation, TransformationBatch, TransformResult } from "./types.js";
+export declare abstract class BaseDataGenerator implements DataGenerator {
+    abstract readonly name: string;
+    abstract connect(): Promise<void>;
+    abstract disconnect(): Promise<void>;
+    abstract createTable(table: TableConfig): Promise<void>;
+    abstract truncateTable(tableName: string): Promise<void>;
+    abstract dropTable(tableName: string): Promise<void>;
+    abstract queryRows(tableName: string, limit?: number): Promise<GeneratedRow[]>;
+    abstract countRows(tableName: string): Promise<number>;
+    /**
+     * Get the maximum value of a column (for resuming sequences)
+     */
+    abstract getMaxValue(tableName: string, columnName: string): Promise<number | null>;
+    /**
+     * Get the total size of a table in bytes (including indexes if applicable)
+     */
+    abstract getTableSize(tableName: string): Promise<number | null>;
+    /**
+     * Run database-specific optimization after large inserts
+     */
+    abstract optimize(tableName: string): Promise<void>;
+    /**
+     * Apply a batch of transformations via UPDATE statement
+     */
+    protected abstract applyTransformations(tableName: string, transformations: Transformation[]): Promise<void>;
+    /**
+     * Generate rows using database-native SQL functions.
+     * This is much faster than JavaScript-based generation.
+     */
+    protected abstract generateNative(table: TableConfig, rowCount: number, startSequence: number): Promise<void>;
+    /**
+     * Get the total size of a table as a human-readable string
+     */
+    getTableSizeForHuman(tableName: string): Promise<string | null>;
+    generate(options: GenerateOptions): Promise<GenerateResult>;
+    transform(tableName: string, batches: TransformationBatch[]): Promise<TransformResult>;
+}
+//# sourceMappingURL=base-generator.d.ts.map

package/dist/generator/base-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-generator.d.ts","sourceRoot":"","sources":["../../src/generator/base-generator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,eAAe,EACf,cAAc,EACd,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,eAAe,EAChB,MAAM,YAAY,CAAC;AAGpB,8BAAsB,iBAAkB,YAAW,aAAa;IAC9D,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAE/B,QAAQ,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IACjC,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IACpC,QAAQ,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IACvD,QAAQ,CAAC,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IACxD,QAAQ,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IACpD,QAAQ,CAAC,SAAS,CAChB,SAAS,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,YAAY,EAAE,CAAC;IAC1B,QAAQ,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAEtD;;OAEG;IACH,QAAQ,CAAC,WAAW,CAClB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAEzB;;OAEG;IACH,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAEhE;;OAEG;IACH,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,oBAAoB,CACrC,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,cAAc,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC;IAEhB;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,cAAc,CAC/B,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC;IAEhB;;OAEG;IACG,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAK/D,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;IA+D3D,SAAS,CACb,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,mBAAmB,EAAE,GAC7B,OAAO,CAAC,eAAe,CAAC;CAqB5B"}

package/dist/generator/base-generator.js

@@ -0,0 +1,72 @@
+import { formatBytes } from "./utils.js";
+export class BaseDataGenerator {
+    /**
+     * Get the total size of a table as a human-readable string
+     */
+    async getTableSizeForHuman(tableName) {
+        const size = await this.getTableSize(tableName);
+        return size === null ? null : formatBytes(size);
+    }
+    async generate(options) {
+        const { table, rowCount, createTable = true, dropFirst = false, truncateFirst = false, resumeSequences = true, optimize = true, } = options;
+        const startTime = Date.now();
+        const tableLabel = table.description
+            ? `${table.name} (${table.description})`
+            : table.name;
+        console.log(`[${this.name}] Generating: ${tableLabel} - ${rowCount.toLocaleString()} rows`);
+        if (dropFirst) {
+            await this.dropTable(table.name);
+        }
+        if (createTable) {
+            await this.createTable(table);
+        }
+        if (truncateFirst && !dropFirst) {
+            await this.truncateTable(table.name);
+        }
+        let startSequence = 1;
+        if (resumeSequences) {
+            for (const column of table.columns) {
+                if (column.generator.kind === "sequence") {
+                    const maxVal = await this.getMaxValue(table.name, column.name);
+                    if (maxVal !== null) {
+                        const step = column.generator.step ?? 1;
+                        startSequence = maxVal + step;
+                    }
+                    break; // Only check first sequence column
+                }
+            }
+        }
+        await this.generateNative(table, rowCount, startSequence);
+        const generateMs = Date.now() - startTime;
+        let optimizeMs = 0;
+        if (optimize) {
+            const optimizeStart = Date.now();
+            await this.optimize(table.name);
+            optimizeMs = Date.now() - optimizeStart;
+        }
+        return {
+            rowsInserted: rowCount,
+            durationMs: Date.now() - startTime,
+            generateMs,
+            optimizeMs,
+        };
+    }
+    async transform(tableName, batches) {
+        const startTime = Date.now();
+        let batchesApplied = 0;
+        for (let i = 0; i < batches.length; i++) {
+            const batch = batches[i]; // eslint-disable-line @typescript-eslint/no-non-null-assertion
+            if (batch.transformations.length > 0) {
+                const batchLabel = batch.description ?? `batch ${String(i + 1)}`;
+                console.log(`[${this.name}] Applying transformations: ${batchLabel} (${String(batch.transformations.length)} transformation(s))`);
+                await this.applyTransformations(tableName, batch.transformations);
+                batchesApplied++;
+            }
+        }
+        return {
+            durationMs: Date.now() - startTime,
+            batchesApplied,
+        };
+    }
+}
+//# sourceMappingURL=base-generator.js.map

package/dist/generator/base-generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-generator.js","sourceRoot":"","sources":["../../src/generator/base-generator.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,OAAgB,iBAAiB;IAkDrC;;OAEG;IACH,KAAK,CAAC,oBAAoB,CAAC,SAAiB;QAC1C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAChD,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,MAAM,EACJ,KAAK,EACL,QAAQ,EACR,WAAW,GAAG,IAAI,EAClB,SAAS,GAAG,KAAK,EACjB,aAAa,GAAG,KAAK,EACrB,eAAe,GAAG,IAAI,EACtB,QAAQ,GAAG,IAAI,GAChB,GAAG,OAAO,CAAC;QACZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW;YAClC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,WAAW,GAAG;YACxC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QACf,OAAO,CAAC,GAAG,CACT,IAAI,IAAI,CAAC,IAAI,iBAAiB,UAAU,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAC/E,CAAC;QAEF,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QAED,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,aAAa,IAAI,CAAC,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,eAAe,EAAE,CAAC;YACpB,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnC,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;oBACzC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC/D,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;wBACpB,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC;wBACxC,aAAa,GAAG,MAAM,GAAG,IAAI,CAAC;oBAChC,CAAC;oBACD,MAAM,CAAC,mCAAmC;gBAC5C,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAE1C,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC;QAC1C,CAAC;QAED,OAAO;YACL,YAAY,EAAE,QAAQ;YACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,UAAU;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CACb,SAAiB,EACjB,OAA8B;QAE9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC,+DAA+D;YAC1F,IAAI,KAAK,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,IAAI,SAAS,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBACjE,OAAO,CAAC,GAAG,CACT,IAAI,IAAI,CAAC,IAAI,+BAA+B,UAAU,KAAK,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,qBAAqB,CACrH,CAAC;gBACF,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;gBAClE,cAAc,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;QAED,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,cAAc;SACf,CAAC;IACJ,CAAC;CACF"}

package/dist/generator/clickhouse-generator.d.ts

@@ -0,0 +1,44 @@
+import type { TableConfig, GeneratedRow, GeneratorConfig, Transformation } from "./types.js";
+import { BaseDataGenerator } from "./base-generator.js";
+export interface ClickHouseConfig {
+    host: string;
+    port: number;
+    username: string;
+    password: string;
+    database: string;
+}
+/**
+ * Convert a generator config to a ClickHouse SQL expression
+ */
+export declare function generatorToClickHouseExpr(gen: GeneratorConfig, seqExpr: string): string;
+export declare class ClickHouseDataGenerator extends BaseDataGenerator {
+    private config;
+    readonly name = "clickhouse";
+    private client;
+    constructor(config: ClickHouseConfig);
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    private getClient;
+    createTable(table: TableConfig): Promise<void>;
+    truncateTable(tableName: string): Promise<void>;
+    dropTable(tableName: string): Promise<void>;
+    protected generateNative(table: TableConfig, rowCount: number, startSequence: number): Promise<void>;
+    queryRows(tableName: string, limit?: number): Promise<GeneratedRow[]>;
+    countRows(tableName: string): Promise<number>;
+    getMaxValue(tableName: string, columnName: string): Promise<number | null>;
+    getTableSize(tableName: string): Promise<number | null>;
+    optimize(tableName: string): Promise<void>;
+    protected applyTransformations(tableName: string, transformations: Transformation[]): Promise<void>;
+    /**
+     * Apply multiple swap transformations in a single table swap operation.
+     * Each swap gets its own random value to ensure independent swap decisions.
+     */
+    private applySwapTransformations;
+    /**
+     * Apply a lookup transformation using ClickHouse's table swap approach.
+     * Since ClickHouse doesn't support correlated subqueries in ALTER TABLE UPDATE,
+     * we use: CREATE TABLE new -> INSERT SELECT with JOIN -> RENAME swap
+     */
+    private applyLookupTransformation;
+}
+//# sourceMappingURL=clickhouse-generator.d.ts.map

package/dist/generator/clickhouse-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"clickhouse-generator.d.ts","sourceRoot":"","sources":["../../src/generator/clickhouse-generator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EAEZ,eAAe,EACf,cAAc,EAIf,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAIxD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAkCD;;GAEG;AACH,wBAAgB,yBAAyB,CACvC,GAAG,EAAE,eAAe,EACpB,OAAO,EAAE,MAAM,GACd,MAAM,CA4CR;AAED,qBAAa,uBAAwB,SAAQ,iBAAiB;IAIhD,OAAO,CAAC,MAAM;IAH1B,QAAQ,CAAC,IAAI,gBAAgB;IAC7B,OAAO,CAAC,MAAM,CAAiC;gBAE3B,MAAM,EAAE,gBAAgB;IAI5C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAYlB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAOjC,OAAO,CAAC,SAAS;IAOX,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAsB9C,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/C,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;cAOjC,cAAc,CAC5B,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC;IA4CV,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,SAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAUlE,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAc7C,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAcnB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAevD,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;cAWhC,oBAAoB,CAClC,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,cAAc,EAAE,GAChC,OAAO,CAAC,IAAI,CAAC;IAsHhB;;;OAGG;YACW,wBAAwB;IA0FtC;;;;OAIG;YACW,yBAAyB;CAoDxC"}