@kaelio/ktx 0.5.0 → 0.6.0

package/dist/connectors/bigquery/connector.test.js

@@ -1,363 +0,0 @@
-import { describe, expect, it, vi } from 'vitest';
-import { bigQueryConnectionConfigFromConfig, isKtxBigQueryConnectionConfig, KtxBigQueryScanConnector } from '../../connectors/bigquery/connector.js';
-import { createBigQueryLiveDatabaseIntrospection } from '../../connectors/bigquery/live-database-introspection.js';
-import { tableRefSet } from '../../context/scan/table-ref.js';
-function fakeClientFactory() {
-    const queryResults = vi.fn(async () => [
-        [{ id: 1, status: 'paid' }],
-        undefined,
-        { schema: { fields: [{ name: 'id', type: 'INT64' }, { name: 'status', type: 'STRING' }] } },
-    ]);
-    const createQueryJob = vi.fn(async (input) => {
-        if (input.query.includes('INFORMATION_SCHEMA.TABLE_CONSTRAINTS')) {
-            return [
-                {
-                    getQueryResults: async () => [
-                        [{ table_name: 'orders', column_name: 'id' }],
-                        undefined,
-                        { schema: { fields: [{ name: 'table_name', type: 'STRING' }, { name: 'column_name', type: 'STRING' }] } },
-                    ],
-                },
-            ];
-        }
-        if (input.query.includes('APPROX_COUNT_DISTINCT')) {
-            return [
-                {
-                    getQueryResults: async () => [
-                        [{ cardinality: 2 }],
-                        undefined,
-                        { schema: { fields: [{ name: 'cardinality', type: 'INT64' }] } },
-                    ],
-                },
-            ];
-        }
-        if (input.query.includes('SELECT DISTINCT CAST')) {
-            return [
-                {
-                    getQueryResults: async () => [
-                        [{ val: 'open' }, { val: 'paid' }],
-                        undefined,
-                        { schema: { fields: [{ name: 'val', type: 'STRING' }] } },
-                    ],
-                },
-            ];
-        }
-        if (input.query.includes('SELECT `status`')) {
-            return [
-                {
-                    getQueryResults: async () => [
-                        [{ status: 'paid' }],
-                        undefined,
-                        { schema: { fields: [{ name: 'status', type: 'STRING' }] } },
-                    ],
-                },
-            ];
-        }
-        return [{ getQueryResults: queryResults }];
-    });
-    const getTable = vi.fn(async () => [
-        {
-            metadata: {
-                type: 'TABLE',
-                numRows: '12',
-                description: 'Orders table',
-                schema: {
-                    fields: [
-                        { name: 'id', type: 'INT64', mode: 'REQUIRED', description: 'Order id' },
-                        { name: 'status', type: 'STRING', mode: 'NULLABLE' },
-                        { name: 'payload', type: 'RECORD', mode: 'NULLABLE' },
-                    ],
-                },
-            },
-        },
-    ]);
-    const tableRef = { id: 'orders', get: getTable };
-    return {
-        createClient: vi.fn(() => ({
-            getDatasets: vi.fn(async () => [[{ id: 'analytics' }, { id: 'staging' }]]),
-            dataset: vi.fn((datasetId) => ({
-                get: vi.fn(async () => [{ id: datasetId }]),
-                getTables: vi.fn(async () => [[tableRef]]),
-            })),
-            createQueryJob,
-        })),
-    };
-}
-const connection = {
-    driver: 'bigquery',
-    dataset_id: 'analytics',
-    credentials_json: JSON.stringify({ project_id: 'project-1', client_email: 'reader@example.test' }),
-    location: 'US',
-};
-describe('KtxBigQueryScanConnector', () => {
-    it('resolves configuration safely', () => {
-        expect(isKtxBigQueryConnectionConfig(connection)).toBe(true);
-        expect(isKtxBigQueryConnectionConfig({ driver: 'mysql' })).toBe(false);
-        expect(bigQueryConnectionConfigFromConfig({ connectionId: 'warehouse', connection })).toMatchObject({
-            projectId: 'project-1',
-            datasetIds: ['analytics'],
-            location: 'US',
-        });
-    });
-    it('introspects datasets, table metadata, primary keys, and normalized types', async () => {
-        const connector = new KtxBigQueryScanConnector({
-            connectionId: 'warehouse',
-            connection,
-            clientFactory: fakeClientFactory(),
-            now: () => new Date('2026-04-29T17:00:00.000Z'),
-        });
-        const snapshot = await connector.introspect({ connectionId: 'warehouse', driver: 'bigquery' }, { runId: 'scan-run-1' });
-        expect(snapshot).toMatchObject({
-            connectionId: 'warehouse',
-            driver: 'bigquery',
-            extractedAt: '2026-04-29T17:00:00.000Z',
-            scope: { catalogs: ['project-1'], datasets: ['analytics'] },
-            metadata: {
-                project_id: 'project-1',
-                datasets: ['analytics'],
-                table_count: 1,
-                total_columns: 3,
-            },
-        });
-        expect(snapshot.tables[0]).toMatchObject({
-            catalog: 'project-1',
-            db: 'analytics',
-            name: 'orders',
-            kind: 'table',
-            comment: 'Orders table',
-            estimatedRows: 12,
-            foreignKeys: [],
-        });
-        expect(snapshot.tables[0]?.columns).toEqual([
-            {
-                name: 'id',
-                nativeType: 'INT64',
-                normalizedType: 'BIGINT',
-                dimensionType: 'number',
-                nullable: false,
-                primaryKey: true,
-                comment: 'Order id',
-            },
-            {
-                name: 'status',
-                nativeType: 'STRING',
-                normalizedType: 'VARCHAR',
-                dimensionType: 'string',
-                nullable: true,
-                primaryKey: false,
-                comment: null,
-            },
-            {
-                name: 'payload',
-                nativeType: 'RECORD',
-                normalizedType: 'JSON',
-                dimensionType: 'string',
-                nullable: true,
-                primaryKey: false,
-                comment: null,
-            },
-        ]);
-    });
-    it('runs samples, read-only SQL, distinct values, dataset listing, row counts, and cleanup', async () => {
-        const connector = new KtxBigQueryScanConnector({
-            connectionId: 'warehouse',
-            connection,
-            clientFactory: fakeClientFactory(),
-        });
-        await expect(connector.sampleTable({
-            connectionId: 'warehouse',
-            table: { catalog: 'project-1', db: 'analytics', name: 'orders' },
-            columns: ['id', 'status'],
-            limit: 1,
-        }, { runId: 'scan-run-1' })).resolves.toEqual({
-            headers: ['id', 'status'],
-            headerTypes: ['INT64', 'STRING'],
-            rows: [[1, 'paid']],
-            totalRows: 1,
-        });
-        await expect(connector.sampleColumn({
-            connectionId: 'warehouse',
-            table: { catalog: 'project-1', db: 'analytics', name: 'orders' },
-            column: 'status',
-            limit: 5,
-        }, { runId: 'scan-run-1' })).resolves.toMatchObject({ values: ['paid'], nullCount: null, distinctCount: null });
-        await expect(connector.executeReadOnly({ connectionId: 'warehouse', sql: 'select id, status from `project-1`.`analytics`.`orders`', maxRows: 1 }, { runId: 'scan-run-1' })).resolves.toMatchObject({ headers: ['id', 'status'], rows: [[1, 'paid']], totalRows: 1, rowCount: 1 });
-        await expect(connector.executeReadOnly({ connectionId: 'warehouse', sql: 'delete from orders' }, { runId: 'scan-run-1' })).rejects.toThrow('Only read-only SELECT/WITH queries can be executed locally');
-        await expect(connector.getColumnDistinctValues({ catalog: 'project-1', db: 'analytics', name: 'orders' }, 'status', { maxCardinality: 5, limit: 10, sampleSize: 100 })).resolves.toEqual({ values: ['open', 'paid'], cardinality: 2 });
-        await expect(connector.getTableRowCount('orders')).resolves.toBe(12);
-        await expect(connector.listDatasets()).resolves.toEqual(['analytics', 'staging']);
-        await expect(connector.columnStats({ connectionId: 'warehouse', table: { catalog: 'project-1', db: 'analytics', name: 'orders' }, column: 'status' }, { runId: 'scan-run-1' })).resolves.toBeNull();
-        await connector.cleanup();
-    });
-    it('limits introspection to tables in tableScope', async () => {
-        const ordersGet = vi.fn(async () => [
-            {
-                metadata: {
-                    type: 'TABLE',
-                    numRows: '12',
-                    schema: { fields: [{ name: 'id', type: 'INT64', mode: 'REQUIRED' }] },
-                },
-            },
-        ]);
-        const skippedGet = vi.fn(async () => [
-            { metadata: { type: 'TABLE', numRows: '1', schema: { fields: [] } } },
-        ]);
-        const clientFactory = {
-            createClient: vi.fn(() => ({
-                getDatasets: vi.fn(async () => [[{ id: 'analytics' }]]),
-                dataset: vi.fn(() => ({
-                    get: vi.fn(async () => [{ id: 'analytics' }]),
-                    getTables: vi.fn(async () => [
-                        [
-                            { id: 'orders', get: ordersGet },
-                            { id: 'customers', get: skippedGet },
-                        ],
-                    ]),
-                })),
-                createQueryJob: vi.fn(async () => [
-                    {
-                        getQueryResults: async () => [
-                            [],
-                            undefined,
-                            { schema: { fields: [{ name: 'table_name', type: 'STRING' }, { name: 'column_name', type: 'STRING' }] } },
-                        ],
-                    },
-                ]),
-            })),
-        };
-        const connector = new KtxBigQueryScanConnector({
-            connectionId: 'warehouse',
-            connection,
-            clientFactory,
-        });
-        const scope = tableRefSet([{ catalog: 'project-1', db: 'analytics', name: 'orders' }]);
-        const snapshot = await connector.introspect({ connectionId: 'warehouse', driver: 'bigquery', tableScope: scope }, { runId: 'scope-test' });
-        expect(snapshot.tables.map((table) => table.name)).toEqual(['orders']);
-        expect(ordersGet).toHaveBeenCalledTimes(1);
-        expect(skippedGet).not.toHaveBeenCalled();
-    });
-    it('constructs for discovery without dataset scope and lists tables through one region information schema query', async () => {
-        const createQueryJob = vi.fn(async (input) => [
-            {
-                getQueryResults: async () => [
-                    [
-                        { table_schema: 'analytics', table_name: 'orders', table_type: 'BASE TABLE' },
-                        { table_schema: 'analytics', table_name: 'order_clone', table_type: 'CLONE' },
-                        { table_schema: 'mart', table_name: 'orders_mv', table_type: 'MATERIALIZED VIEW' },
-                    ],
-                    undefined,
-                    {
-                        schema: {
-                            fields: [
-                                { name: 'table_schema', type: 'STRING' },
-                                { name: 'table_name', type: 'STRING' },
-                                { name: 'table_type', type: 'STRING' },
-                            ],
-                        },
-                    },
-                ],
-            },
-        ]);
-        const clientFactory = {
-            createClient: vi.fn(() => ({
-                getDatasets: vi.fn(async () => [[{ id: 'analytics' }, { id: 'mart' }]]),
-                dataset: vi.fn((datasetId) => ({
-                    get: vi.fn(async () => [{ id: datasetId }]),
-                    getTables: vi.fn(async () => [[]]),
-                })),
-                createQueryJob,
-            })),
-        };
-        const connector = new KtxBigQueryScanConnector({
-            connectionId: 'warehouse',
-            connection: {
-                driver: 'bigquery',
-                credentials_json: JSON.stringify({ project_id: 'project-1' }),
-                location: 'US',
-            },
-            clientFactory,
-        });
-        await expect(connector.listTables(['analytics', 'mart'])).resolves.toEqual([
-            { schema: 'analytics', name: 'orders', kind: 'table' },
-            { schema: 'analytics', name: 'order_clone', kind: 'table' },
-            { schema: 'mart', name: 'orders_mv', kind: 'view' },
-        ]);
-        expect(createQueryJob).toHaveBeenCalledTimes(1);
-        expect(createQueryJob).toHaveBeenCalledWith(expect.objectContaining({
-            location: 'US',
-            params: { dataset_ids: ['analytics', 'mart'] },
-        }));
-        expect(createQueryJob.mock.calls[0]?.[0].query).toContain('`project-1`.`region-us`.INFORMATION_SCHEMA.TABLES');
-        expect(createQueryJob.mock.calls[0]?.[0].query).toContain("'CLONE'");
-        expect(createQueryJob.mock.calls[0]?.[0].query).toContain("'SNAPSHOT'");
-    });
-    it('keeps scan paths requiring dataset scope', async () => {
-        const connector = new KtxBigQueryScanConnector({
-            connectionId: 'warehouse',
-            connection: {
-                driver: 'bigquery',
-                credentials_json: JSON.stringify({ project_id: 'project-1' }),
-                location: 'US',
-            },
-            clientFactory: fakeClientFactory(),
-        });
-        await expect(connector.introspect({ connectionId: 'warehouse', driver: 'bigquery' }, { runId: 'scan-run-1' })).rejects.toThrow('Native BigQuery scan requires connections.warehouse.dataset_ids or dataset_id');
-    });
-    it('applies maximumBytesBilled to read-only queries when configured', async () => {
-        const clientFactory = fakeClientFactory();
-        const connector = new KtxBigQueryScanConnector({
-            connectionId: 'warehouse',
-            connection,
-            clientFactory,
-            maxBytesBilled: 123456789,
-        });
-        await expect(connector.executeReadOnly({ connectionId: 'warehouse', sql: 'select id, status from `project-1`.`analytics`.`orders`', maxRows: 1 }, { runId: 'scan-run-1' })).resolves.toMatchObject({ rows: [[1, 'paid']], rowCount: 1 });
-        const client = vi.mocked(clientFactory.createClient).mock.results[0]?.value;
-        expect(client.createQueryJob).toHaveBeenLastCalledWith(expect.objectContaining({
-            maximumBytesBilled: '123456789',
-        }));
-    });
-    it('applies canonical BigQuery YAML scan limits to query jobs', async () => {
-        const clientFactory = fakeClientFactory();
-        const connector = new KtxBigQueryScanConnector({
-            connectionId: 'warehouse',
-            connection: { ...connection, max_bytes_billed: '987654321', job_timeout_ms: 30_000 },
-            clientFactory,
-        });
-        await expect(connector.executeReadOnly({ connectionId: 'warehouse', sql: 'select id, status from `project-1`.`analytics`.`orders`', maxRows: 1 }, { runId: 'scan-run-1' })).resolves.toMatchObject({ rows: [[1, 'paid']], rowCount: 1 });
-        const client = vi.mocked(clientFactory.createClient).mock.results[0]?.value;
-        expect(client.createQueryJob).toHaveBeenLastCalledWith(expect.objectContaining({
-            maximumBytesBilled: '987654321',
-            jobTimeoutMs: 30_000,
-        }));
-    });
-    it('adapts native snapshots to live-database introspection snapshots', async () => {
-        const introspection = createBigQueryLiveDatabaseIntrospection({
-            connections: { warehouse: connection },
-            clientFactory: fakeClientFactory(),
-            now: () => new Date('2026-04-29T17:00:00.000Z'),
-        });
-        await expect(introspection.extractSchema('warehouse')).resolves.toMatchObject({
-            connectionId: 'warehouse',
-            metadata: { project_id: 'project-1' },
-            tables: expect.arrayContaining([
-                expect.objectContaining({
-                    catalog: 'project-1',
-                    db: 'analytics',
-                    name: 'orders',
-                    columns: expect.arrayContaining([
-                        {
-                            name: 'id',
-                            nativeType: 'INT64',
-                            normalizedType: 'BIGINT',
-                            dimensionType: 'number',
-                            nullable: false,
-                            primaryKey: true,
-                            comment: 'Order id',
-                        },
-                    ]),
-                }),
-            ]),
-        });
-    });
-});

package/dist/connectors/bigquery/dialect.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/connectors/bigquery/dialect.test.js

@@ -1,36 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { KtxBigQueryDialect } from './dialect.js';
-describe('KtxBigQueryDialect', () => {
-    const dialect = new KtxBigQueryDialect();
-    it('quotes identifiers and formats project.dataset.table names', () => {
-        expect(dialect.quoteIdentifier('order`items')).toBe('`order\\`items`');
-        expect(dialect.formatTableName({ catalog: 'project-1', db: 'analytics', name: 'orders' })).toBe('`project-1`.`analytics`.`orders`');
-        expect(dialect.formatTableName({ db: 'analytics', name: 'orders' })).toBe('`analytics`.`orders`');
-        expect(dialect.formatTableName({ name: 'orders' })).toBe('`orders`');
-    });
-    it('maps native BigQuery types to normalized types and scan dimensions', () => {
-        expect(dialect.mapDataType('INT64')).toBe('BIGINT');
-        expect(dialect.mapDataType('STRUCT')).toBe('JSON');
-        expect(dialect.mapDataType('GEOGRAPHY')).toBe('GEOGRAPHY');
-        expect(dialect.mapToDimensionType('TIMESTAMP')).toBe('time');
-        expect(dialect.mapToDimensionType('NUMERIC')).toBe('number');
-        expect(dialect.mapToDimensionType('BOOL')).toBe('boolean');
-        expect(dialect.mapToDimensionType('JSON')).toBe('string');
-    });
-    it('generates sampling, cardinality, and distinct-value SQL', () => {
-        expect(dialect.generateSampleQuery('`p`.`d`.`orders`', 5, ['id', 'status'])).toBe('SELECT `id`, `status` FROM `p`.`d`.`orders` ORDER BY RAND() LIMIT 5');
-        expect(dialect.generateColumnSampleQuery('`p`.`d`.`orders`', 'status', 10)).toBe("SELECT `status` FROM `p`.`d`.`orders` WHERE `status` IS NOT NULL AND TRIM(CAST(`status` AS STRING)) != '' ORDER BY RAND() LIMIT 10");
-        expect(dialect.generateCardinalitySampleQuery('`p`.`d`.`orders`', '`status`', 100)).toContain('SELECT APPROX_COUNT_DISTINCT(val) AS cardinality');
-        expect(dialect.generateDistinctValuesQuery('`p`.`d`.`orders`', '`status`', 20)).toContain('SELECT DISTINCT CAST(`status` AS STRING) AS val');
-    });
-    it('rewrites colon parameters to BigQuery named parameters', () => {
-        expect(dialect.prepareQuery('SELECT * FROM orders WHERE id = :id AND id_2 = :id_2', { id: 1, id_2: 2 })).toEqual({
-            sql: 'SELECT * FROM orders WHERE id = @id AND id_2 = @id_2',
-            params: { id: 1, id_2: 2 },
-        });
-        expect(dialect.prepareQuery('SELECT * FROM orders')).toEqual({ sql: 'SELECT * FROM orders', params: undefined });
-    });
-    it('keeps unsupported statistics explicit', () => {
-        expect(dialect.generateColumnStatisticsQuery('analytics', 'orders')).toBeNull();
-    });
-});

package/dist/connectors/clickhouse/connector.test.d.ts

@@ -1 +0,0 @@
-export {};