@kaelio/ktx 0.4.0 → 0.5.0

package/node_modules/@ktx/connector-bigquery/dist/live-database-introspection.js

@@ -1,20 +0,0 @@
-import { KtxBigQueryScanConnector, } from './connector.js';
-export function createBigQueryLiveDatabaseIntrospection(options) {
-    return {
-        async extractSchema(connectionId) {
-            const connection = options.connections[connectionId];
-            const connector = new KtxBigQueryScanConnector({
-                connectionId,
-                connection,
-                clientFactory: options.clientFactory,
-                now: options.now,
-            });
-            try {
-                return await connector.introspect({ connectionId, driver: 'bigquery' }, { runId: `bigquery-${connectionId}` });
-            }
-            finally {
-                await connector.cleanup();
-            }
-        },
-    };
-}

package/node_modules/@ktx/connector-bigquery/dist/package-exports.test.js

@@ -1,10 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import * as connector from './index.js';
-describe('@ktx/connector-bigquery exports', () => {
-    it('exports public connector, dialect, and introspection APIs', () => {
-        expect(connector.KtxBigQueryDialect).toBeTypeOf('function');
-        expect(connector.KtxBigQueryScanConnector).toBeTypeOf('function');
-        expect(connector.bigQueryConnectionConfigFromConfig).toBeTypeOf('function');
-        expect(connector.createBigQueryLiveDatabaseIntrospection).toBeTypeOf('function');
-    });
-});

package/node_modules/@ktx/connector-bigquery/package.json

@@ -1,20 +0,0 @@
-{
-  "name": "@ktx/connector-bigquery",
-  "version": "0.0.0-private",
-  "private": true,
-  "type": "module",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js",
-      "default": "./dist/index.js"
-    },
-    "./package.json": "./package.json"
-  },
-  "files": [
-    "dist"
-  ],
-  "license": "Apache-2.0"
-}

package/node_modules/@ktx/connector-clickhouse/dist/connector.d.ts

@@ -1,113 +0,0 @@
-import { createClient } from '@clickhouse/client';
-import { type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaSnapshot, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '@ktx/context/scan';
-export interface KtxClickHouseConnectionConfig {
-    driver?: string;
-    host?: string;
-    port?: number;
-    database?: string;
-    username?: string;
-    user?: string;
-    password?: string;
-    url?: string;
-    ssl?: boolean;
-    [key: string]: unknown;
-}
-export interface KtxClickHouseResolvedClientConfig {
-    host: string;
-    port: number;
-    database: string;
-    username: string;
-    password?: string;
-    ssl: boolean;
-}
-interface ClickHouseQueryInput {
-    query: string;
-    format: 'JSONCompact' | 'JSONEachRow';
-    query_params?: Record<string, unknown>;
-}
-interface ClickHouseResultSet {
-    json(): Promise<unknown>;
-}
-export interface KtxClickHouseClient {
-    query(input: ClickHouseQueryInput): Promise<ClickHouseResultSet>;
-    close(): Promise<void>;
-}
-export interface KtxClickHouseClientFactory {
-    createClient(config: Parameters<typeof createClient>[0]): KtxClickHouseClient;
-}
-interface KtxClickHouseResolvedEndpoint {
-    host: string;
-    port: number;
-    close?: () => Promise<void>;
-}
-export interface KtxClickHouseEndpointResolver {
-    resolve(input: {
-        host: string;
-        port: number;
-        connection: KtxClickHouseConnectionConfig;
-    }): Promise<KtxClickHouseResolvedEndpoint>;
-}
-export interface KtxClickHouseScanConnectorOptions {
-    connectionId: string;
-    connection: KtxClickHouseConnectionConfig | undefined;
-    clientFactory?: KtxClickHouseClientFactory;
-    endpointResolver?: KtxClickHouseEndpointResolver;
-    env?: NodeJS.ProcessEnv;
-    now?: () => Date;
-}
-export interface KtxClickHouseReadOnlyQueryInput extends KtxReadOnlyQueryInput {
-    params?: Record<string, unknown>;
-}
-export interface KtxClickHouseColumnDistinctValuesOptions {
-    maxCardinality: number;
-    limit: number;
-    sampleSize?: number;
-}
-export interface KtxClickHouseColumnDistinctValuesResult {
-    values: string[] | null;
-    cardinality: number;
-}
-export declare function isKtxClickHouseConnectionConfig(connection: KtxClickHouseConnectionConfig | undefined): connection is KtxClickHouseConnectionConfig;
-export declare function clickHouseClientConfigFromConfig(input: {
-    connectionId: string;
-    connection: KtxClickHouseConnectionConfig | undefined;
-    env?: NodeJS.ProcessEnv;
-}): KtxClickHouseResolvedClientConfig;
-export declare class KtxClickHouseScanConnector implements KtxScanConnector {
-    readonly id: string;
-    readonly driver: "clickhouse";
-    readonly capabilities: import("@ktx/context/scan").KtxConnectorCapabilities;
-    private readonly connectionId;
-    private readonly connection;
-    private readonly clientConfig;
-    private readonly clientFactory;
-    private readonly endpointResolver?;
-    private readonly now;
-    private readonly dialect;
-    private client;
-    private resolvedEndpoint;
-    constructor(options: KtxClickHouseScanConnectorOptions);
-    testConnection(): Promise<{
-        success: boolean;
-        error?: string;
-    }>;
-    introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot>;
-    sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise<KtxTableSampleResult>;
-    sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise<KtxColumnSampleResult>;
-    columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise<KtxColumnStatsResult | null>;
-    executeReadOnly(input: KtxClickHouseReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult>;
-    getColumnDistinctValues(table: KtxTableRef, columnName: string, options: KtxClickHouseColumnDistinctValuesOptions): Promise<KtxClickHouseColumnDistinctValuesResult | null>;
-    getTableRowCount(tableName: string): Promise<number>;
-    qTableName(table: Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>): string;
-    quoteIdentifier(identifier: string): string;
-    listSchemas(): Promise<string[]>;
-    listTables(schemas?: string[]): Promise<KtxTableListEntry[]>;
-    cleanup(): Promise<void>;
-    private toSchemaTable;
-    private toSchemaColumn;
-    private clientForQuery;
-    private queryEachRow;
-    private query;
-    private assertConnection;
-}
-export {};

package/node_modules/@ktx/connector-clickhouse/dist/connector.js

@@ -1,345 +0,0 @@
-import { createClient } from '@clickhouse/client';
-import { assertReadOnlySql, limitSqlForExecution } from '@ktx/context/connections';
-import { createKtxConnectorCapabilities, } from '@ktx/context/scan';
-import { readFileSync } from 'node:fs';
-import { Agent as HttpsAgent } from 'node:https';
-import { homedir } from 'node:os';
-import { resolve } from 'node:path';
-import { KtxClickHouseDialect } from './dialect.js';
-class DefaultClickHouseClientFactory {
-    createClient(config) {
-        return createClient(config);
-    }
-}
-function stringConfigValue(connection, key, env) {
-    const value = connection?.[key];
-    return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
-}
-function resolveStringReference(value, env) {
-    if (value.startsWith('env:')) {
-        const envName = value.slice('env:'.length);
-        return env[envName] ?? '';
-    }
-    if (value.startsWith('file:')) {
-        const rawPath = value.slice('file:'.length);
-        const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(1)) : rawPath;
-        return readFileSync(path, 'utf-8').trim();
-    }
-    return value;
-}
-function maybeNumber(value) {
-    return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
-}
-function parseClickHouseUrl(url) {
-    const parsed = new URL(url);
-    return {
-        host: parsed.hostname,
-        port: parsed.port ? Number(parsed.port) : undefined,
-        database: parsed.pathname.replace(/^\/+/, '') || undefined,
-        username: parsed.username ? decodeURIComponent(parsed.username) : undefined,
-        password: parsed.password ? decodeURIComponent(parsed.password) : undefined,
-        ssl: parsed.protocol === 'https:' || parsed.searchParams.get('ssl') === 'true',
-    };
-}
-function tableKind(engine) {
-    return engine === 'View' || engine === 'MaterializedView' ? 'view' : 'table';
-}
-function isNullableClickHouseType(type) {
-    return type.startsWith('Nullable(') || type.startsWith('LowCardinality(Nullable(');
-}
-export function isKtxClickHouseConnectionConfig(connection) {
-    return String(connection?.driver ?? '').toLowerCase() === 'clickhouse';
-}
-export function clickHouseClientConfigFromConfig(input) {
-    const inputDriver = input.connection?.driver ?? 'unknown';
-    if (!isKtxClickHouseConnectionConfig(input.connection)) {
-        throw new Error(`Native ClickHouse connector cannot run driver "${inputDriver}"`);
-    }
-    const env = input.env ?? process.env;
-    const referencedUrl = stringConfigValue(input.connection, 'url', env);
-    const urlConfig = referencedUrl ? parseClickHouseUrl(referencedUrl) : {};
-    const merged = { ...urlConfig, ...input.connection };
-    const host = stringConfigValue(merged, 'host', env);
-    const database = stringConfigValue(merged, 'database', env) ?? 'default';
-    const username = stringConfigValue(merged, 'username', env) ?? stringConfigValue(merged, 'user', env) ?? 'default';
-    if (!host) {
-        throw new Error(`Native ClickHouse connector requires connections.${input.connectionId}.host or url`);
-    }
-    return {
-        host,
-        port: maybeNumber(merged.port) ?? 8123,
-        database,
-        username,
-        password: stringConfigValue(merged, 'password', env),
-        ssl: merged.ssl === true,
-    };
-}
-export class KtxClickHouseScanConnector {
-    id;
-    driver = 'clickhouse';
-    capabilities = createKtxConnectorCapabilities({
-        tableSampling: true,
-        columnSampling: true,
-        columnStats: false,
-        readOnlySql: true,
-        nestedAnalysis: true,
-        formalForeignKeys: false,
-        estimatedRowCounts: true,
-    });
-    connectionId;
-    connection;
-    clientConfig;
-    clientFactory;
-    endpointResolver;
-    now;
-    dialect = new KtxClickHouseDialect();
-    client = null;
-    resolvedEndpoint = null;
-    constructor(options) {
-        this.connectionId = options.connectionId;
-        this.connection = options.connection ?? {};
-        this.clientConfig = clickHouseClientConfigFromConfig({
-            connectionId: options.connectionId,
-            connection: options.connection,
-            env: options.env,
-        });
-        this.clientFactory = options.clientFactory ?? new DefaultClickHouseClientFactory();
-        this.endpointResolver = options.endpointResolver;
-        this.now = options.now ?? (() => new Date());
-        this.id = `clickhouse:${options.connectionId}`;
-    }
-    async testConnection() {
-        try {
-            await this.query('SELECT 1');
-            return { success: true };
-        }
-        catch (error) {
-            return { success: false, error: error instanceof Error ? error.message : String(error) };
-        }
-    }
-    async introspect(input, _ctx) {
-        this.assertConnection(input.connectionId);
-        const database = this.clientConfig.database;
-        const tables = await this.queryEachRow(`
-      SELECT name, engine, comment
-      FROM system.tables
-      WHERE database = {database:String}
-        AND engine NOT IN ('Dictionary')
-      ORDER BY name
-      `, { database });
-        const columns = await this.queryEachRow(`
-      SELECT table, name, type, comment, is_in_primary_key
-      FROM system.columns
-      WHERE database = {database:String}
-      ORDER BY table, position
-      `, { database });
-        const rowCounts = await this.queryEachRow(`
-      SELECT table, sum(rows) AS row_count
-      FROM system.parts
-      WHERE database = {database:String}
-        AND active = 1
-      GROUP BY table
-      `, { database });
-        const columnsByTable = new Map();
-        for (const column of columns) {
-            columnsByTable.set(column.table, [...(columnsByTable.get(column.table) ?? []), column]);
-        }
-        const rowCountByTable = new Map(rowCounts.map((row) => [String(row.table), Number(row.row_count ?? 0)]));
-        const schemaTables = tables.map((table) => this.toSchemaTable(table, columnsByTable.get(table.name) ?? [], rowCountByTable.get(table.name) ?? 0));
-        return {
-            connectionId: this.connectionId,
-            driver: 'clickhouse',
-            extractedAt: this.now().toISOString(),
-            scope: { schemas: [database] },
-            metadata: {
-                database,
-                host: this.clientConfig.host,
-                table_count: schemaTables.length,
-                total_columns: schemaTables.reduce((sum, table) => sum + table.columns.length, 0),
-            },
-            tables: schemaTables,
-        };
-    }
-    async sampleTable(input, _ctx) {
-        this.assertConnection(input.connectionId);
-        const result = await this.query(this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns));
-        return { headers: result.headers, rows: result.rows, totalRows: result.totalRows };
-    }
-    async sampleColumn(input, _ctx) {
-        this.assertConnection(input.connectionId);
-        const result = await this.query(this.dialect.generateColumnSampleQuery(this.qTableName(input.table), input.column, input.limit));
-        const values = result.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => row[0]);
-        return { values, nullCount: null, distinctCount: null };
-    }
-    async columnStats(_input, _ctx) {
-        return null;
-    }
-    async executeReadOnly(input, _ctx) {
-        this.assertConnection(input.connectionId);
-        const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows);
-        const prepared = this.dialect.prepareQuery(limitedSql, input.params);
-        const result = await this.query(prepared.sql, prepared.params);
-        return { ...result, rowCount: result.rows.length };
-    }
-    async getColumnDistinctValues(table, columnName, options) {
-        const sampleSize = options.sampleSize ?? 10000;
-        const tableName = this.qTableName(table);
-        const quotedColumn = this.dialect.quoteIdentifier(columnName);
-        const cardinalityResult = await this.query(this.dialect.generateCardinalitySampleQuery(tableName, quotedColumn, sampleSize));
-        const cardinality = Number(cardinalityResult.rows[0]?.[0]);
-        if (Number.isNaN(cardinality)) {
-            return null;
-        }
-        if (cardinality === 0) {
-            return { values: [], cardinality: 0 };
-        }
-        if (cardinality > options.maxCardinality) {
-            return { values: null, cardinality };
-        }
-        const valuesResult = await this.query(this.dialect.generateDistinctValuesQuery(tableName, quotedColumn, options.limit));
-        return {
-            values: valuesResult.rows.filter((row) => row[0] !== null).map((row) => String(row[0])),
-            cardinality,
-        };
-    }
-    async getTableRowCount(tableName) {
-        const result = await this.query(`
-      SELECT sum(rows) AS count
-      FROM system.parts
-      WHERE database = {database:String}
-        AND table = {table:String}
-        AND active = 1
-      `, { database: this.clientConfig.database, table: tableName });
-        return Number(result.rows[0]?.[0] ?? 0);
-    }
-    qTableName(table) {
-        return this.dialect.formatTableName(table);
-    }
-    quoteIdentifier(identifier) {
-        return this.dialect.quoteIdentifier(identifier);
-    }
-    async listSchemas() {
-        const rows = await this.queryEachRow(`
-      SELECT name
-      FROM system.databases
-      WHERE name NOT IN ('system', 'INFORMATION_SCHEMA', 'information_schema')
-      ORDER BY name
-      `);
-        return rows.map((row) => row.name);
-    }
-    async listTables(schemas) {
-        const filterSchemas = schemas ?? (await this.listSchemas());
-        if (filterSchemas.length === 0)
-            return [];
-        const rows = await this.queryEachRow(`
-      SELECT database, name, engine
-      FROM system.tables
-      WHERE database IN ({schemas:Array(String)})
-      ORDER BY database, name
-      `, { schemas: filterSchemas });
-        return rows.map((row) => ({
-            schema: row.database,
-            name: row.name,
-            kind: row.engine === 'View' || row.engine === 'MaterializedView' ? 'view' : 'table',
-        }));
-    }
-    async cleanup() {
-        if (this.client) {
-            await this.client.close();
-            this.client = null;
-        }
-        if (this.resolvedEndpoint?.close) {
-            await this.resolvedEndpoint.close();
-            this.resolvedEndpoint = null;
-        }
-    }
-    toSchemaTable(table, columns, estimatedRows) {
-        const kind = tableKind(table.engine);
-        return {
-            catalog: null,
-            db: this.clientConfig.database,
-            name: table.name,
-            kind,
-            comment: table.comment || null,
-            estimatedRows: kind === 'view' ? null : estimatedRows,
-            columns: columns.map((column) => this.toSchemaColumn(column)),
-            foreignKeys: [],
-        };
-    }
-    toSchemaColumn(column) {
-        return {
-            name: column.name,
-            nativeType: column.type,
-            normalizedType: this.dialect.mapDataType(column.type),
-            dimensionType: this.dialect.mapToDimensionType(column.type),
-            nullable: isNullableClickHouseType(column.type),
-            primaryKey: column.is_in_primary_key === 1,
-            comment: column.comment || null,
-        };
-    }
-    async clientForQuery() {
-        if (!this.client) {
-            const config = { ...this.clientConfig };
-            if (this.endpointResolver) {
-                this.resolvedEndpoint = await this.endpointResolver.resolve({
-                    host: config.host,
-                    port: config.port,
-                    connection: this.connection,
-                });
-                config.host = this.resolvedEndpoint.host;
-                config.port = this.resolvedEndpoint.port;
-            }
-            const protocol = config.ssl ? 'https' : 'http';
-            const isProxied = config.host !== this.clientConfig.host;
-            this.client = this.clientFactory.createClient({
-                url: `${protocol}://${config.host}:${config.port}`,
-                username: config.username,
-                password: config.password ?? '',
-                database: config.database,
-                request_timeout: 30_000,
-                clickhouse_settings: {
-                    output_format_json_quote_64bit_integers: 1,
-                },
-                ...(isProxied && config.ssl
-                    ? {
-                        http_agent: new HttpsAgent({
-                            servername: this.clientConfig.host,
-                            keepAlive: true,
-                        }),
-                    }
-                    : {}),
-            });
-        }
-        return this.client;
-    }
-    async queryEachRow(sql, params) {
-        const client = await this.clientForQuery();
-        const resultSet = await client.query({
-            query: assertReadOnlySql(sql),
-            format: 'JSONEachRow',
-            ...(params ? { query_params: params } : {}),
-        });
-        return (await resultSet.json());
-    }
-    async query(sql, params) {
-        const client = await this.clientForQuery();
-        const resultSet = await client.query({
-            query: assertReadOnlySql(sql),
-            format: 'JSONCompact',
-            ...(params ? { query_params: params } : {}),
-        });
-        const response = (await resultSet.json());
-        const meta = response.meta ?? [];
-        return {
-            headers: meta.map((field) => field.name),
-            headerTypes: meta.map((field) => field.type),
-            rows: response.data ?? [],
-            totalRows: response.rows ?? response.data?.length ?? 0,
-        };
-    }
-    assertConnection(connectionId) {
-        if (connectionId !== this.connectionId) {
-            throw new Error(`KTX ClickHouse connector ${this.id} cannot serve connection ${connectionId}`);
-        }
-    }
-}