@kaelio/ktx 0.15.0 → 0.16.0

package/dist/connectors/athena/dialect.js

@@ -0,0 +1,146 @@
+import { columnDisplayPartCount, formatDialectDisplayRef, formatDialectTableName, parseDialectDisplayRef, } from '../../context/connections/dialect-helpers.js';
+/** @internal */
+export class KtxAthenaDialect {
+    type = 'athena';
+    dimensionTypeMappings = {
+        timestamp: 'time',
+        date: 'time',
+        bigint: 'number',
+        int: 'number',
+        integer: 'number',
+        tinyint: 'number',
+        smallint: 'number',
+        double: 'number',
+        float: 'number',
+        real: 'number',
+        boolean: 'boolean',
+    };
+    quoteIdentifier(identifier) {
+        return `"${identifier.replace(/"/g, '""')}"`;
+    }
+    formatTableName(table) {
+        return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'ansi');
+    }
+    formatDisplayRef(table) {
+        return formatDialectDisplayRef(table, 'ansi');
+    }
+    parseDisplayRef(display) {
+        return parseDialectDisplayRef(display, 'ansi');
+    }
+    columnDisplayTablePartCount() {
+        return columnDisplayPartCount('ansi');
+    }
+    mapDataType(nativeType) {
+        const base = nativeType.toLowerCase().trim().split('<')[0].split('(')[0].trim();
+        const typeMap = {
+            string: 'VARCHAR',
+            varchar: 'VARCHAR',
+            char: 'CHAR',
+            binary: 'VARBINARY',
+            bigint: 'BIGINT',
+            int: 'INTEGER',
+            integer: 'INTEGER',
+            tinyint: 'TINYINT',
+            smallint: 'SMALLINT',
+            double: 'DOUBLE',
+            float: 'FLOAT',
+            real: 'REAL',
+            decimal: 'DECIMAL',
+            boolean: 'BOOLEAN',
+            timestamp: 'TIMESTAMP',
+            date: 'DATE',
+            array: 'ARRAY',
+            map: 'MAP',
+            struct: 'STRUCT',
+            uniontype: 'UNION',
+        };
+        return typeMap[base] ?? nativeType.toUpperCase();
+    }
+    mapToDimensionType(nativeType) {
+        const base = nativeType.toLowerCase().trim().split('<')[0].split('(')[0].trim();
+        const mapped = this.dimensionTypeMappings[base];
+        if (mapped)
+            return mapped;
+        if (base.includes('timestamp') || base.includes('date'))
+            return 'time';
+        if (base.includes('int') || base.includes('float') || base.includes('double') || base.includes('decimal') || base.includes('real'))
+            return 'number';
+        if (base.includes('bool'))
+            return 'boolean';
+        return 'string';
+    }
+    generateSampleQuery(tableName, limit, columns) {
+        const columnList = columns && columns.length > 0 ? columns.map((c) => this.quoteIdentifier(c)).join(', ') : '*';
+        return `SELECT ${columnList} FROM ${tableName} LIMIT ${limit}`;
+    }
+    generateColumnSampleQuery(tableName, columnName, limit) {
+        const quoted = this.quoteIdentifier(columnName);
+        return `SELECT ${quoted} FROM ${tableName} WHERE ${quoted} IS NOT NULL LIMIT ${limit}`;
+    }
+    generateCardinalitySampleQuery(tableName, columnName, sampleSize) {
+        return `
+      SELECT approx_distinct(${columnName}) AS cardinality
+      FROM (
+        SELECT ${columnName}
+        FROM ${tableName}
+        WHERE ${columnName} IS NOT NULL
+        LIMIT ${sampleSize}
+      )
+    `;
+    }
+    generateRandomizedCardinalitySampleQuery(tableName, columnName, sampleSize) {
+        return `
+      SELECT approx_distinct(${columnName}) AS cardinality
+      FROM (
+        SELECT ${columnName}
+        FROM ${tableName}
+        WHERE ${columnName} IS NOT NULL
+        ORDER BY rand()
+        LIMIT ${sampleSize}
+      )
+    `;
+    }
+    generateDistinctValuesQuery(tableName, columnName, limit) {
+        return `
+      SELECT DISTINCT CAST(${columnName} AS VARCHAR) AS val
+      FROM ${tableName}
+      WHERE ${columnName} IS NOT NULL
+      ORDER BY val
+      LIMIT ${limit}
+    `;
+    }
+    generateColumnStatisticsQuery(_schemaName, _tableName) {
+        return null;
+    }
+    getNullCountExpression(column) {
+        return `COUNT_IF(${column} IS NULL)`;
+    }
+    getDistinctCountExpression(column) {
+        return `approx_distinct(${column})`;
+    }
+    textLengthExpression(columnSql) {
+        return `LENGTH(CAST(${columnSql} AS VARCHAR))`;
+    }
+    castToText(columnSql) {
+        return `CAST(${columnSql} AS VARCHAR)`;
+    }
+    getSampleValueAggregation(innerSql) {
+        return `(SELECT array_join(array_agg(CAST(value AS VARCHAR)), '\u001f') FROM (${innerSql}) AS relationship_profile_values)`;
+    }
+    getLimitOffsetClause(limit, offset) {
+        const safeLimit = Math.max(1, Math.floor(limit));
+        const safeOffset = offset !== undefined ? Math.floor(offset) : 0;
+        return safeOffset > 0 ? `OFFSET ${safeOffset} LIMIT ${safeLimit}` : `LIMIT ${safeLimit}`;
+    }
+    getTopClause(_limit) {
+        return '';
+    }
+    getTableSampleClause(_samplePct) {
+        return '';
+    }
+    getRandomSampleFilter(samplePct) {
+        if (samplePct <= 0 || samplePct >= 1)
+            return '';
+        return `rand() < ${samplePct}`;
+    }
+}

package/dist/connectors/athena/live-database-introspection.d.ts

@@ -0,0 +1,10 @@
+import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
+import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
+import { type KtxAthenaClientFactory } from './connector.js';
+interface CreateAthenaLiveDatabaseIntrospectionOptions {
+    connections: Record<string, KtxProjectConnectionConfig>;
+    clientFactory?: KtxAthenaClientFactory;
+    now?: () => Date;
+}
+export declare function createAthenaLiveDatabaseIntrospection(options: CreateAthenaLiveDatabaseIntrospectionOptions): LiveDatabaseIntrospectionPort;
+export {};

package/dist/connectors/athena/live-database-introspection.js

@@ -0,0 +1,24 @@
+import { KtxAthenaScanConnector, } from './connector.js';
+export function createAthenaLiveDatabaseIntrospection(options) {
+    return {
+        async extractSchema(connectionId, introspectionOptions) {
+            const connection = options.connections[connectionId];
+            const connector = new KtxAthenaScanConnector({
+                connectionId,
+                connection,
+                clientFactory: options.clientFactory,
+                now: options.now,
+            });
+            try {
+                return await connector.introspect({
+                    connectionId,
+                    driver: 'athena',
+                    ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}),
+                }, { runId: `athena-${connectionId}` });
+            }
+            finally {
+                await connector.cleanup();
+            }
+        },
+    };
+}

package/dist/connectors/duckdb/connector.d.ts

@@ -0,0 +1,60 @@
+import { type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxConnectorTestResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaSnapshot, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
+export interface KtxDuckDbConnectionConfig {
+    driver?: string;
+    path?: string;
+    url?: string;
+    [key: string]: unknown;
+}
+/** @internal */
+export interface DuckDbDatabasePathInput {
+    connectionId: string;
+    projectDir?: string;
+    connection: KtxDuckDbConnectionConfig | undefined;
+}
+export interface KtxDuckDbScanConnectorOptions extends DuckDbDatabasePathInput {
+    now?: () => Date;
+}
+export interface KtxDuckDbColumnDistinctValuesOptions {
+    maxCardinality: number;
+    limit: number;
+    sampleSize?: number;
+}
+export interface KtxDuckDbColumnDistinctValuesResult {
+    values: string[] | null;
+    cardinality: number;
+}
+export declare function isKtxDuckDbConnectionConfig(connection: KtxDuckDbConnectionConfig | undefined): connection is KtxDuckDbConnectionConfig;
+/** @internal */
+export declare function duckDbDatabasePathFromConfig(input: DuckDbDatabasePathInput): string;
+export declare class KtxDuckDbScanConnector implements KtxScanConnector {
+    readonly id: string;
+    readonly driver: "duckdb";
+    readonly capabilities: import("../../context/scan/types.js").KtxConnectorCapabilities;
+    private readonly connectionId;
+    private readonly dbPath;
+    private readonly now;
+    private readonly dialect;
+    private instance;
+    private connection;
+    constructor(options: KtxDuckDbScanConnectorOptions);
+    testConnection(): Promise<KtxConnectorTestResult>;
+    introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot>;
+    listSchemas(): Promise<string[]>;
+    listTables(_schemas?: string[]): Promise<KtxTableListEntry[]>;
+    sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise<KtxTableSampleResult>;
+    sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise<KtxColumnSampleResult>;
+    columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise<KtxColumnStatsResult | null>;
+    executeReadOnly(input: KtxReadOnlyQueryInput, _ctx: KtxScanContext): Promise<KtxQueryResult>;
+    getColumnDistinctValues(table: KtxTableRef, columnName: string, options: KtxDuckDbColumnDistinctValuesOptions): Promise<KtxDuckDbColumnDistinctValuesResult | null>;
+    getTableRowCount(tableName: string): Promise<number>;
+    qTableName(table: Pick<KtxTableRef, 'name'>): string;
+    quoteIdentifier(identifier: string): string;
+    cleanup(): Promise<void>;
+    private db;
+    private query;
+    private readTableRows;
+    private readTable;
+    private readPrimaryKeyColumns;
+    private readForeignKeys;
+    private assertConnection;
+}

package/dist/connectors/duckdb/connector.js

@@ -0,0 +1,282 @@
+import { DuckDBInstance } from '@duckdb/node-api';
+import { existsSync, statSync } from 'node:fs';
+import { isAbsolute, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { resolveStringReference } from '../shared/string-reference.js';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
+import { normalizeQueryRows } from '../../context/connections/query-executor.js';
+import { toJsonSafeRows } from '../shared/duckdb-json-safe.js';
+import { connectorTestFailure, createKtxConnectorCapabilities, } from '../../context/scan/types.js';
+import { scopedTableNames } from '../../context/scan/table-ref.js';
+const MAIN_SCHEMA = 'main';
+// `path` may be an env:/file: reference; `url` resolves env: only, since file:
+// on a url is a native URI form (handled by duckDbPathFromUrl), not a file read.
+function stringConfigValue(connection, key) {
+    const value = connection?.[key];
+    if (typeof value !== 'string' || value.trim().length === 0) {
+        return undefined;
+    }
+    const trimmed = value.trim();
+    if (key === 'url') {
+        return trimmed.startsWith('env:') ? (process.env[trimmed.slice('env:'.length)] ?? '') : trimmed;
+    }
+    return resolveStringReference(trimmed, process.env);
+}
+function duckDbPathFromUrl(url) {
+    if (url.startsWith('file:')) {
+        return fileURLToPath(url);
+    }
+    if (url.startsWith('duckdb:')) {
+        const parsed = new URL(url);
+        return decodeURIComponent(parsed.pathname);
+    }
+    return url;
+}
+export function isKtxDuckDbConnectionConfig(connection) {
+    return String(connection?.driver ?? '').toLowerCase() === 'duckdb';
+}
+/** @internal */
+export function duckDbDatabasePathFromConfig(input) {
+    const inputDriver = input.connection?.driver ?? 'unknown';
+    if (!isKtxDuckDbConnectionConfig(input.connection)) {
+        throw new Error(`Native DuckDB connector cannot run driver "${inputDriver}"`);
+    }
+    const configuredPath = stringConfigValue(input.connection, 'path') ?? duckDbPathFromUrl(stringConfigValue(input.connection, 'url') ?? '');
+    if (!configuredPath) {
+        throw new Error(`Native DuckDB connector requires connections.${input.connectionId}.path or url`);
+    }
+    return isAbsolute(configuredPath) ? configuredPath : resolve(input.projectDir ?? process.cwd(), configuredPath);
+}
+export class KtxDuckDbScanConnector {
+    id;
+    driver = 'duckdb';
+    capabilities = createKtxConnectorCapabilities({
+        tableSampling: true,
+        columnSampling: true,
+        columnStats: false,
+        readOnlySql: true,
+        nestedAnalysis: false,
+        formalForeignKeys: true,
+        estimatedRowCounts: true,
+    });
+    connectionId;
+    dbPath;
+    now;
+    dialect = getSqlDialectForDriver('duckdb');
+    instance = null;
+    connection = null;
+    constructor(options) {
+        this.connectionId = options.connectionId;
+        this.dbPath = duckDbDatabasePathFromConfig(options);
+        this.now = options.now ?? (() => new Date());
+        this.id = `duckdb:${options.connectionId}`;
+    }
+    async testConnection() {
+        try {
+            if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) {
+                return { success: false, error: `File not found: ${this.dbPath}` };
+            }
+            await this.query('SELECT 1');
+            return { success: true };
+        }
+        catch (error) {
+            return connectorTestFailure(error);
+        }
+    }
+    async introspect(input, _ctx) {
+        this.assertConnection(input.connectionId);
+        const scopedNames = input.tableScope ? scopedTableNames(input.tableScope, { catalog: null, db: null }) : null;
+        const tableRows = await this.readTableRows(scopedNames);
+        const tables = [];
+        for (const row of tableRows) {
+            tables.push(await this.readTable(row));
+        }
+        return {
+            connectionId: this.connectionId,
+            driver: 'duckdb',
+            extractedAt: this.now().toISOString(),
+            scope: {},
+            metadata: {
+                file_path: this.dbPath,
+                table_count: tables.length,
+                total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
+            },
+            tables,
+        };
+    }
+    async listSchemas() {
+        return [MAIN_SCHEMA];
+    }
+    async listTables(_schemas) {
+        const rows = await this.readTableRows(null);
+        return rows.map((row) => ({
+            catalog: null,
+            schema: MAIN_SCHEMA,
+            name: row.table_name,
+            kind: row.table_type === 'VIEW' ? 'view' : 'table',
+        }));
+    }
+    async sampleTable(input, _ctx) {
+        this.assertConnection(input.connectionId);
+        const result = await this.query(this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns));
+        return { headers: result.headers, rows: result.rows, totalRows: result.totalRows };
+    }
+    async sampleColumn(input, _ctx) {
+        this.assertConnection(input.connectionId);
+        const result = await this.query(this.dialect.generateColumnSampleQuery(this.qTableName(input.table), input.column, input.limit));
+        const values = result.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => row[0]);
+        return { values, nullCount: null, distinctCount: null };
+    }
+    async columnStats(_input, _ctx) {
+        return null;
+    }
+    async executeReadOnly(input, _ctx) {
+        this.assertConnection(input.connectionId);
+        const result = await this.query(limitSqlForExecution(input.sql, input.maxRows));
+        return { ...result, rowCount: result.rows.length };
+    }
+    async getColumnDistinctValues(table, columnName, options) {
+        const sampleSize = options.sampleSize ?? 10000;
+        const tableName = this.qTableName(table);
+        const quotedColumn = this.dialect.quoteIdentifier(columnName);
+        const cardinalityResult = await this.query(this.dialect.generateCardinalitySampleQuery(tableName, quotedColumn, sampleSize));
+        if (cardinalityResult.rows.length === 0) {
+            return null;
+        }
+        const cardinality = Number(cardinalityResult.rows[0][0]);
+        if (Number.isNaN(cardinality)) {
+            return null;
+        }
+        if (cardinality === 0) {
+            return { values: [], cardinality: 0 };
+        }
+        if (cardinality > options.maxCardinality) {
+            return { values: null, cardinality };
+        }
+        const valuesResult = await this.query(this.dialect.generateDistinctValuesQuery(tableName, quotedColumn, options.limit));
+        return {
+            values: valuesResult.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => String(row[0])),
+            cardinality,
+        };
+    }
+    async getTableRowCount(tableName) {
+        const result = await this.query(`SELECT COUNT(*) AS count FROM ${this.dialect.quoteIdentifier(tableName)}`);
+        return Number(result.rows[0]?.[0] ?? 0);
+    }
+    qTableName(table) {
+        return this.dialect.formatTableName(table);
+    }
+    quoteIdentifier(identifier) {
+        return this.dialect.quoteIdentifier(identifier);
+    }
+    async cleanup() {
+        this.connection?.closeSync();
+        this.instance?.closeSync();
+        this.connection = null;
+        this.instance = null;
+    }
+    async db() {
+        if (!this.connection) {
+            // DuckDBInstance.create() creates the file if missing, so this pre-check
+            // enforces the never-create rule. Do not remove it.
+            if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) {
+                throw new Error(`File not found: ${this.dbPath}`);
+            }
+            this.instance = await DuckDBInstance.create(this.dbPath, { access_mode: 'read_only' });
+            this.connection = await this.instance.connect();
+        }
+        return this.connection;
+    }
+    async query(sql) {
+        const connection = await this.db();
+        const reader = await connection.runAndReadAll(assertReadOnlySql(sql));
+        const rows = toJsonSafeRows(normalizeQueryRows(reader.getRows()));
+        return {
+            headers: reader.columnNames(),
+            rows,
+            totalRows: rows.length,
+        };
+    }
+    async readTableRows(scopedNames) {
+        if (scopedNames && scopedNames.length === 0) {
+            return [];
+        }
+        const scopeClause = scopedNames
+            ? `AND table_name IN (${scopedNames.map((name) => `'${name.replaceAll("'", "''")}'`).join(', ')})`
+            : '';
+        const result = await this.query(`SELECT table_name, table_type
+       FROM information_schema.tables
+       WHERE table_schema = '${MAIN_SCHEMA}' ${scopeClause}
+       ORDER BY table_name`);
+        return result.rows.map((row) => ({ table_name: String(row[0]), table_type: String(row[1]) }));
+    }
+    async readTable(table) {
+        const columnsResult = await this.query(`SELECT column_name, data_type, is_nullable
+       FROM information_schema.columns
+       WHERE table_schema = '${MAIN_SCHEMA}' AND table_name = '${table.table_name.replaceAll("'", "''")}'
+       ORDER BY ordinal_position`);
+        const columns = columnsResult.rows.map((row) => ({
+            column_name: String(row[0]),
+            data_type: String(row[1]),
+            is_nullable: String(row[2]),
+        }));
+        const primaryKeys = await this.readPrimaryKeyColumns(table.table_name);
+        const isView = table.table_type === 'VIEW';
+        const estimatedRows = isView ? null : await this.getTableRowCount(table.table_name);
+        return {
+            catalog: null,
+            db: null,
+            name: table.table_name,
+            kind: isView ? 'view' : 'table',
+            comment: null,
+            estimatedRows,
+            columns: columns.map((column) => ({
+                name: column.column_name,
+                nativeType: column.data_type,
+                normalizedType: this.dialect.mapDataType(column.data_type),
+                dimensionType: this.dialect.mapToDimensionType(column.data_type),
+                nullable: column.is_nullable === 'YES' && !primaryKeys.has(column.column_name),
+                primaryKey: primaryKeys.has(column.column_name),
+                comment: null,
+            })),
+            foreignKeys: await this.readForeignKeys(table.table_name),
+        };
+    }
+    async readPrimaryKeyColumns(tableName) {
+        const result = await this.query(`SELECT kcu.column_name
+       FROM information_schema.table_constraints tc
+       JOIN information_schema.key_column_usage kcu
+         ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema
+       WHERE tc.table_schema = '${MAIN_SCHEMA}'
+         AND tc.table_name = '${tableName.replaceAll("'", "''")}'
+         AND tc.constraint_type = 'PRIMARY KEY'`);
+        return new Set(result.rows.map((row) => String(row[0])));
+    }
+    async readForeignKeys(tableName) {
+        // information_schema.constraint_column_usage in DuckDB returns the constrained
+        // columns (source), not the referenced columns. Use duckdb_constraints() which
+        // exposes constraint_column_names and referenced_column_names directly.
+        const result = await this.query(`SELECT unnest(constraint_column_names) AS from_column,
+              referenced_table,
+              unnest(referenced_column_names) AS to_column,
+              constraint_name
+       FROM duckdb_constraints()
+       WHERE schema_name = '${MAIN_SCHEMA}'
+         AND table_name = '${tableName.replaceAll("'", "''")}'
+         AND constraint_type = 'FOREIGN KEY'`);
+        return result.rows.map((row) => ({
+            fromColumn: String(row[0]),
+            toCatalog: null,
+            toDb: null,
+            toTable: String(row[1]),
+            toColumn: String(row[2]),
+            constraintName: row[3] === null ? null : String(row[3]),
+        }));
+    }
+    assertConnection(connectionId) {
+        if (connectionId !== this.connectionId) {
+            throw new Error(`ktx DuckDB connector ${this.id} cannot serve connection ${connectionId}`);
+        }
+    }
+}

package/dist/connectors/duckdb/dialect.d.ts

@@ -0,0 +1,31 @@
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
+import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
+type DuckDbTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
+/** @internal */
+export declare class KtxDuckDbDialect implements KtxSqlDialect {
+    readonly type: "duckdb";
+    private readonly typeMappings;
+    quoteIdentifier(identifier: string): string;
+    formatTableName(table: DuckDbTableNameRef): string;
+    formatDisplayRef(table: DuckDbTableNameRef): string;
+    parseDisplayRef(display: string): KtxTableRef | null;
+    columnDisplayTablePartCount(): 1 | 2 | 3;
+    mapDataType(nativeType: string): string;
+    mapToDimensionType(nativeType: string): KtxSchemaDimensionType;
+    generateSampleQuery(tableName: string, limit: number, columns?: string[]): string;
+    generateColumnSampleQuery(tableName: string, columnName: string, limit: number): string;
+    getRandomSampleFilter(samplePct: number): string;
+    getTableSampleClause(samplePct: number): string;
+    getLimitOffsetClause(limit: number, offset?: number): string;
+    getTopClause(_limit: number): string;
+    getNullCountExpression(column: string): string;
+    getDistinctCountExpression(column: string): string;
+    textLengthExpression(columnSql: string): string;
+    castToText(columnSql: string): string;
+    getSampleValueAggregation(innerSql: string): string;
+    generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string;
+    generateDistinctValuesQuery(tableName: string, columnName: string, limit: number): string;
+    generateColumnStatisticsQuery(_schemaName: string, _tableName: string): string | null;
+    generateRandomizedCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string;
+}
+export {};