@kaelio/ktx 0.13.1 → 0.15.0

package/dist/connectors/mongodb/connector.js

@@ -0,0 +1,307 @@
+import { MongoClient } from 'mongodb';
+import { resolveKtxConfigReference } from '../../context/core/config-reference.js';
+import { connectorTestFailure, createKtxConnectorCapabilities, } from '../../context/scan/types.js';
+import { scopedTableNames } from '../../context/scan/table-ref.js';
+import { getDialectForDriver } from '../../context/connections/dialects.js';
+import { inferKtxMongoCollectionColumns, MONGO_ID_FIELD } from './schema-inference.js';
+const DEFAULT_SAMPLE_SIZE = 1000;
+const SAMPLE_MAX_TIME_MS = 30_000;
+class DefaultMongoClient {
+    client;
+    connected = false;
+    constructor(url) {
+        this.client = new MongoClient(url);
+    }
+    async connectedClient() {
+        if (!this.connected) {
+            await this.client.connect();
+            this.connected = true;
+        }
+        return this.client;
+    }
+    async listCollections(databaseName) {
+        const client = await this.connectedClient();
+        const collections = await client.db(databaseName).listCollections({}, { nameOnly: false }).toArray();
+        return collections.map((collection) => ({ name: collection.name, type: collection.type }));
+    }
+    async estimatedDocumentCount(databaseName, collectionName) {
+        const client = await this.connectedClient();
+        return client.db(databaseName).collection(collectionName).estimatedDocumentCount();
+    }
+    async find(databaseName, collectionName, options) {
+        const client = await this.connectedClient();
+        return client
+            .db(databaseName)
+            .collection(collectionName)
+            .find({}, { sort: options.sort, limit: options.limit, maxTimeMS: SAMPLE_MAX_TIME_MS, ...(options.projection ? { projection: options.projection } : {}) })
+            .toArray();
+    }
+    async ping(databaseName) {
+        const client = await this.connectedClient();
+        await client.db(databaseName).command({ ping: 1 });
+    }
+    async close() {
+        if (this.connected) {
+            await this.client.close();
+            this.connected = false;
+        }
+    }
+}
+class DefaultMongoClientFactory {
+    create(url) {
+        return new DefaultMongoClient(url);
+    }
+}
+export function isKtxMongoDbConnectionConfig(connection) {
+    return String(connection?.driver ?? '').toLowerCase() === 'mongodb';
+}
+function databaseFromUrl(url) {
+    try {
+        const path = new URL(url).pathname.replace(/^\/+/, '');
+        const database = path.split('/')[0];
+        return database && database.length > 0 ? decodeURIComponent(database) : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function configuredDatabases(connection, fallback) {
+    if (Array.isArray(connection.databases)) {
+        const selected = connection.databases
+            .filter((database) => typeof database === 'string' && database.trim().length > 0)
+            .map((database) => database.trim());
+        if (selected.length > 0) {
+            return [...new Set(selected)];
+        }
+    }
+    const single = typeof connection.database === 'string' && connection.database.trim().length > 0
+        ? connection.database.trim()
+        : fallback;
+    return single ? [single] : [];
+}
+function positiveInteger(value, fallback) {
+    return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : fallback;
+}
+function normalizeSampleValue(value) {
+    if (value === null || value === undefined) {
+        return null;
+    }
+    if (value instanceof Date) {
+        return value.toISOString();
+    }
+    if (typeof value === 'object') {
+        const bsontype = value._bsontype;
+        return typeof bsontype === 'string' ? String(value) : JSON.stringify(value);
+    }
+    return value;
+}
+function unionDocumentKeys(documents) {
+    const keys = [];
+    const seen = new Set();
+    for (const document of documents) {
+        for (const key of Object.keys(document)) {
+            if (!seen.has(key)) {
+                seen.add(key);
+                keys.push(key);
+            }
+        }
+    }
+    return keys;
+}
+export class KtxMongoDbScanConnector {
+    id;
+    driver = 'mongodb';
+    capabilities = createKtxConnectorCapabilities({
+        tableSampling: true,
+        columnSampling: true,
+        columnStats: false,
+        readOnlySql: false,
+        nestedAnalysis: true,
+        formalForeignKeys: false,
+        estimatedRowCounts: true,
+    });
+    connectionId;
+    connection;
+    url;
+    databases;
+    sampleSize;
+    orderBy;
+    enabledTables;
+    clientFactory;
+    now;
+    dialect = getDialectForDriver('mongodb');
+    client = null;
+    constructor(options) {
+        const connection = options.connection ?? {};
+        const inputDriver = connection.driver ?? 'unknown';
+        if (!isKtxMongoDbConnectionConfig(connection)) {
+            throw new Error(`Native MongoDB connector cannot run driver "${inputDriver}"`);
+        }
+        const env = options.env ?? process.env;
+        const url = resolveKtxConfigReference(typeof connection.url === 'string' ? connection.url.trim() : undefined, env);
+        if (!url) {
+            throw new Error(`Native MongoDB connector requires connections.${options.connectionId}.url`);
+        }
+        const databases = configuredDatabases(connection, databaseFromUrl(url));
+        if (databases.length === 0) {
+            throw new Error(`Native MongoDB connector requires connections.${options.connectionId}.databases (or a database in the URL)`);
+        }
+        const enabledTables = Array.isArray(connection.enabled_tables)
+            ? new Set(connection.enabled_tables
+                .filter((table) => typeof table === 'string' && table.trim().length > 0)
+                .map((table) => table.trim()))
+            : null;
+        this.connectionId = options.connectionId;
+        this.connection = connection;
+        this.url = url;
+        this.databases = databases;
+        this.sampleSize = positiveInteger(connection.sample_size, DEFAULT_SAMPLE_SIZE);
+        this.orderBy = typeof connection.order_by === 'string' && connection.order_by.trim().length > 0
+            ? connection.order_by.trim()
+            : MONGO_ID_FIELD;
+        this.enabledTables = enabledTables && enabledTables.size > 0 ? enabledTables : null;
+        this.clientFactory = options.clientFactory ?? new DefaultMongoClientFactory();
+        this.now = options.now ?? (() => new Date());
+        this.id = `mongodb:${options.connectionId}`;
+    }
+    async testConnection() {
+        try {
+            await this.clientForQuery().ping(this.databases[0]);
+            return { success: true };
+        }
+        catch (error) {
+            return connectorTestFailure(error);
+        }
+    }
+    async introspect(input, _ctx) {
+        this.assertConnection(input.connectionId);
+        const client = this.clientForQuery();
+        const tables = [];
+        for (const database of this.databases) {
+            const scopedNames = input.tableScope
+                ? new Set(scopedTableNames(input.tableScope, { catalog: null, db: database }))
+                : null;
+            const collections = await client.listCollections(database);
+            for (const collection of collections) {
+                if (collection.name.startsWith('system.')) {
+                    continue;
+                }
+                if (scopedNames && !scopedNames.has(collection.name)) {
+                    continue;
+                }
+                if (this.enabledTables && !this.enabledTables.has(`${database}.${collection.name}`)) {
+                    continue;
+                }
+                tables.push(await this.introspectCollection(client, database, collection));
+            }
+        }
+        return {
+            connectionId: this.connectionId,
+            driver: 'mongodb',
+            extractedAt: this.now().toISOString(),
+            scope: { schemas: this.databases },
+            metadata: {
+                databases: this.databases,
+                sample_size: this.sampleSize,
+                order_by: this.orderBy,
+                table_count: tables.length,
+                total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0),
+            },
+            tables,
+        };
+    }
+    async introspectCollection(client, database, collection) {
+        const isView = collection.type === 'view';
+        // estimatedDocumentCount issues a count command, which MongoDB rejects on a
+        // view (CommandNotSupportedOnView); only count real collections.
+        const estimatedRows = isView ? null : await client.estimatedDocumentCount(database, collection.name);
+        const documents = await client.find(database, collection.name, {
+            sort: { [this.orderBy]: -1 },
+            limit: this.sampleSize,
+        });
+        return {
+            catalog: null,
+            db: database,
+            name: collection.name,
+            kind: isView ? 'view' : 'table',
+            comment: null,
+            estimatedRows,
+            columns: inferKtxMongoCollectionColumns(documents, this.dialect),
+            foreignKeys: [],
+        };
+    }
+    async sampleTable(input, _ctx) {
+        this.assertConnection(input.connectionId);
+        const { database, collection } = this.resolveTableRef(input.table);
+        const documents = await this.clientForQuery().find(database, collection, {
+            sort: { [this.orderBy]: -1 },
+            limit: input.limit,
+        });
+        const headers = input.columns && input.columns.length > 0 ? input.columns : unionDocumentKeys(documents);
+        const rows = documents.map((document) => headers.map((header) => normalizeSampleValue(document[header])));
+        return { headers, rows, totalRows: documents.length };
+    }
+    async sampleColumn(input, _ctx) {
+        this.assertConnection(input.connectionId);
+        const { database, collection } = this.resolveTableRef(input.table);
+        const documents = await this.clientForQuery().find(database, collection, {
+            sort: { [this.orderBy]: -1 },
+            limit: input.limit,
+            projection: { [input.column]: 1 },
+        });
+        const values = [];
+        let nullCount = 0;
+        for (const document of documents) {
+            const value = document[input.column];
+            if (value === null || value === undefined) {
+                nullCount += 1;
+                continue;
+            }
+            values.push(normalizeSampleValue(value));
+        }
+        return { values, nullCount, distinctCount: null };
+    }
+    async listSchemas() {
+        return [...this.databases];
+    }
+    async listTables(schemas) {
+        const client = this.clientForQuery();
+        const databases = schemas && schemas.length > 0 ? schemas : this.databases;
+        const entries = [];
+        for (const database of databases) {
+            const collections = await client.listCollections(database);
+            for (const collection of collections) {
+                if (collection.name.startsWith('system.')) {
+                    continue;
+                }
+                entries.push({
+                    catalog: null,
+                    schema: database,
+                    name: collection.name,
+                    kind: collection.type === 'view' ? 'view' : 'table',
+                });
+            }
+        }
+        return entries;
+    }
+    async cleanup() {
+        if (this.client) {
+            await this.client.close();
+            this.client = null;
+        }
+    }
+    resolveTableRef(table) {
+        return { database: table.db ?? this.databases[0], collection: table.name };
+    }
+    clientForQuery() {
+        if (!this.client) {
+            this.client = this.clientFactory.create(this.url);
+        }
+        return this.client;
+    }
+    assertConnection(connectionId) {
+        if (connectionId !== this.connectionId) {
+            throw new Error(`ktx MongoDB connector ${this.id} cannot serve connection ${connectionId}`);
+        }
+    }
+}

package/dist/connectors/mongodb/dialect.d.ts

@@ -0,0 +1,17 @@
+import type { KtxDialect } from '../../context/connections/dialects.js';
+import type { KtxDialectTableRef } from '../../context/connections/dialect-helpers.js';
+import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
+/**
+ * Display/type-mapping half of the dialect contract for MongoDB. Collections map
+ * to `db.collection` display refs (ansi two-part shape). MongoDB is a non-SQL
+ * source, so it implements {@link KtxDialect} only — never {@link KtxSqlDialect}.
+ */
+export declare class KtxMongoDbDialect implements KtxDialect {
+    readonly type: "mongodb";
+    private readonly typeMappings;
+    formatDisplayRef(table: KtxDialectTableRef): string;
+    parseDisplayRef(display: string): KtxTableRef | null;
+    columnDisplayTablePartCount(): 1 | 2 | 3;
+    mapDataType(nativeType: string): string;
+    mapToDimensionType(nativeType: string): KtxSchemaDimensionType;
+}

package/dist/connectors/mongodb/dialect.js

@@ -0,0 +1,50 @@
+import { columnDisplayPartCount, formatDialectDisplayRef, parseDialectDisplayRef, } from '../../context/connections/dialect-helpers.js';
+/**
+ * Display/type-mapping half of the dialect contract for MongoDB. Collections map
+ * to `db.collection` display refs (ansi two-part shape). MongoDB is a non-SQL
+ * source, so it implements {@link KtxDialect} only — never {@link KtxSqlDialect}.
+ */
+export class KtxMongoDbDialect {
+    type = 'mongodb';
+    typeMappings = {
+        objectid: 'string',
+        string: 'string',
+        uuid: 'string',
+        binary: 'string',
+        regex: 'string',
+        int: 'number',
+        long: 'number',
+        double: 'number',
+        decimal: 'number',
+        bool: 'boolean',
+        date: 'time',
+        timestamp: 'time',
+        object: 'string',
+        array: 'string',
+        json: 'string',
+        null: 'string',
+        mixed: 'string',
+    };
+    formatDisplayRef(table) {
+        return formatDialectDisplayRef(table, 'ansi');
+    }
+    parseDisplayRef(display) {
+        return parseDialectDisplayRef(display, 'ansi');
+    }
+    columnDisplayTablePartCount() {
+        return columnDisplayPartCount('ansi');
+    }
+    mapDataType(nativeType) {
+        const normalized = nativeType.toLowerCase().trim();
+        if (normalized === 'object' || normalized === 'array') {
+            return 'json';
+        }
+        return normalized || 'mixed';
+    }
+    mapToDimensionType(nativeType) {
+        if (!nativeType) {
+            return 'string';
+        }
+        return this.typeMappings[nativeType.toLowerCase().trim()] ?? 'string';
+    }
+}

package/dist/connectors/mongodb/live-database-introspection.d.ts

@@ -0,0 +1,10 @@
+import type { LiveDatabaseIntrospectionPort } from '../../context/ingest/adapters/live-database/types.js';
+import type { KtxProjectConnectionConfig } from '../../context/project/config.js';
+import { type KtxMongoClientFactory } from './connector.js';
+interface CreateMongoDbLiveDatabaseIntrospectionOptions {
+    connections: Record<string, KtxProjectConnectionConfig>;
+    clientFactory?: KtxMongoClientFactory;
+    now?: () => Date;
+}
+export declare function createMongoDbLiveDatabaseIntrospection(options: CreateMongoDbLiveDatabaseIntrospectionOptions): LiveDatabaseIntrospectionPort;
+export {};

package/dist/connectors/mongodb/live-database-introspection.js

@@ -0,0 +1,24 @@
+import { KtxMongoDbScanConnector, } from './connector.js';
+export function createMongoDbLiveDatabaseIntrospection(options) {
+    return {
+        async extractSchema(connectionId, introspectionOptions) {
+            const connection = options.connections[connectionId];
+            const connector = new KtxMongoDbScanConnector({
+                connectionId,
+                connection,
+                clientFactory: options.clientFactory,
+                now: options.now,
+            });
+            try {
+                return await connector.introspect({
+                    connectionId,
+                    driver: 'mongodb',
+                    ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}),
+                }, { runId: `mongodb-${connectionId}` });
+            }
+            finally {
+                await connector.cleanup();
+            }
+        },
+    };
+}

package/dist/connectors/mongodb/schema-inference.d.ts

@@ -0,0 +1,20 @@
+import type { KtxSchemaColumn } from '../../context/scan/types.js';
+import type { KtxDialect } from '../../context/connections/dialects.js';
+export type KtxMongoDocument = Record<string, unknown>;
+/** Top-level field name MongoDB guarantees on every document; used as the primary key. */
+export declare const MONGO_ID_FIELD = "_id";
+/**
+ * Canonical BSON type name for a sampled value as the `mongodb` driver hydrates
+ * it: BSON wrapper objects expose `_bsontype`; everything else maps from the JS
+ * runtime type. Sub-documents and arrays collapse to opaque `object`/`array`.
+ * @internal
+ */
+export declare function bsonTypeOf(value: unknown): string;
+/**
+ * Infer a flat column schema from sampled documents. Each top-level field becomes
+ * one column: BSON types are unioned (a field seen with >1 type is `mixed` and
+ * treated as a string), nullability is derived from field presence and observed
+ * nulls, and sub-documents/arrays remain a single opaque `json` column. `_id` is
+ * the non-nullable primary key.
+ */
+export declare function inferKtxMongoCollectionColumns(documents: readonly KtxMongoDocument[], dialect: KtxDialect): KtxSchemaColumn[];

package/dist/connectors/mongodb/schema-inference.js

@@ -0,0 +1,110 @@
+/** Top-level field name MongoDB guarantees on every document; used as the primary key. */
+export const MONGO_ID_FIELD = '_id';
+const BSON_TYPE_NAMES = {
+    objectid: 'objectId',
+    int32: 'int',
+    long: 'long',
+    double: 'double',
+    decimal128: 'decimal',
+    binary: 'binary',
+    uuid: 'uuid',
+    timestamp: 'timestamp',
+    bsonregexp: 'regex',
+    bsonsymbol: 'string',
+};
+/**
+ * Canonical BSON type name for a sampled value as the `mongodb` driver hydrates
+ * it: BSON wrapper objects expose `_bsontype`; everything else maps from the JS
+ * runtime type. Sub-documents and arrays collapse to opaque `object`/`array`.
+ * @internal
+ */
+export function bsonTypeOf(value) {
+    if (value === null || value === undefined) {
+        return 'null';
+    }
+    if (typeof value === 'string') {
+        return 'string';
+    }
+    if (typeof value === 'boolean') {
+        return 'bool';
+    }
+    if (typeof value === 'number') {
+        return Number.isInteger(value) ? 'int' : 'double';
+    }
+    if (typeof value === 'bigint') {
+        return 'long';
+    }
+    if (value instanceof Date) {
+        return 'date';
+    }
+    if (Array.isArray(value)) {
+        return 'array';
+    }
+    if (typeof value === 'object') {
+        const bsontype = value._bsontype;
+        if (typeof bsontype === 'string') {
+            return BSON_TYPE_NAMES[bsontype.toLowerCase()] ?? bsontype;
+        }
+        return 'object';
+    }
+    return 'mixed';
+}
+function resolveNativeType(types) {
+    if (types.size === 0) {
+        return 'null';
+    }
+    if (types.size > 1) {
+        return 'mixed';
+    }
+    return [...types][0];
+}
+/**
+ * Infer a flat column schema from sampled documents. Each top-level field becomes
+ * one column: BSON types are unioned (a field seen with >1 type is `mixed` and
+ * treated as a string), nullability is derived from field presence and observed
+ * nulls, and sub-documents/arrays remain a single opaque `json` column. `_id` is
+ * the non-nullable primary key.
+ */
+export function inferKtxMongoCollectionColumns(documents, dialect) {
+    const total = documents.length;
+    const order = [];
+    const fields = new Map();
+    for (const document of documents) {
+        if (!document || typeof document !== 'object') {
+            continue;
+        }
+        for (const [name, value] of Object.entries(document)) {
+            let accumulator = fields.get(name);
+            if (!accumulator) {
+                accumulator = { types: new Set(), present: 0, nullSeen: false };
+                fields.set(name, accumulator);
+                order.push(name);
+            }
+            accumulator.present += 1;
+            const bsonType = bsonTypeOf(value);
+            if (bsonType === 'null') {
+                accumulator.nullSeen = true;
+            }
+            else {
+                accumulator.types.add(bsonType);
+            }
+        }
+    }
+    return order.map((name) => {
+        const accumulator = fields.get(name);
+        const nativeType = resolveNativeType(accumulator.types);
+        const isId = name === MONGO_ID_FIELD;
+        const nullable = isId
+            ? false
+            : accumulator.present < total || accumulator.nullSeen || accumulator.types.size === 0;
+        return {
+            name,
+            nativeType,
+            normalizedType: dialect.mapDataType(nativeType),
+            dimensionType: dialect.mapToDimensionType(nativeType),
+            nullable,
+            primaryKey: isId,
+            comment: null,
+        };
+    });
+}

package/dist/connectors/mysql/connector.d.ts

@@ -96,6 +96,7 @@ export declare class KtxMysqlScanConnector implements KtxScanConnector {
     private readonly poolFactory;
     private readonly endpointResolver?;
     private readonly now;
+    private readonly deadlineMs;
     private readonly dialect;
     private pool;
     private resolvedEndpoint;

package/dist/connectors/mysql/connector.js

@@ -1,5 +1,6 @@
 import mysql from 'mysql2/promise';
-import { getDialectForDriver } from '../../context/connections/dialects.js';
+import { getSqlDialectForDriver } from '../../context/connections/dialects.js';
+import { resolveQueryDeadlineMs, queryDeadlineExceededError } from '../../context/connections/query-deadline.js';
 import { resolveStringReference } from '../shared/string-reference.js';
 import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
 import { constraintDiscoveryWarning, tryConstraintQuery, } from '../../context/scan/constraint-discovery.js';
@@ -96,6 +97,10 @@ function isDeniedError(error) {
         code === 'ER_SPECIFIC_ACCESS_DENIED_ERROR' ||
         code === 'ER_DBACCESS_DENIED_ERROR');
 }
+// errno 3024 = ER_QUERY_TIMEOUT, raised when max_execution_time is exceeded.
+function isMysqlTimeoutError(error) {
+    return Boolean(error) && typeof error === 'object' && error.errno === 3024;
+}
 function pushConstraintWarnings(warnings, schemas, kind) {
     for (const schema of schemas) {
         warnings.push(constraintDiscoveryWarning({ schema, kind }));
@@ -183,7 +188,8 @@ export class KtxMysqlScanConnector {
     poolFactory;
     endpointResolver;
     now;
-    dialect = getDialectForDriver('mysql');
+    deadlineMs;
+    dialect = getSqlDialectForDriver('mysql');
     pool = null;
     resolvedEndpoint = null;
     constructor(options) {
@@ -197,6 +203,7 @@ export class KtxMysqlScanConnector {
         this.poolFactory = options.poolFactory ?? new DefaultMysqlPoolFactory();
         this.endpointResolver = options.endpointResolver;
         this.now = options.now ?? (() => new Date());
+        this.deadlineMs = resolveQueryDeadlineMs(this.connection);
         this.id = `mysql:${options.connectionId}`;
     }
     async testConnection() {
@@ -478,6 +485,9 @@ export class KtxMysqlScanConnector {
         const pool = await this.poolForQuery();
         const connection = await pool.getConnection();
         try {
+            // max_execution_time (ms) bounds read-only SELECTs server-side; our path
+            // only runs SELECT/WITH, so the session setting always applies.
+            await connection.query('SET SESSION max_execution_time = ?', [this.deadlineMs]);
             const [rows, fields] = await connection.query(assertReadOnlySql(sql), queryParams(params));
             const headers = fields.map((field) => field.name);
             const headerTypes = fields.map((field) => String(field.type ?? 'unknown'));
@@ -488,6 +498,12 @@ export class KtxMysqlScanConnector {
                 totalRows: rows.length,
             };
         }
+        catch (error) {
+            if (isMysqlTimeoutError(error)) {
+                throw queryDeadlineExceededError(this.deadlineMs, { cause: error });
+            }
+            throw error;
+        }
         finally {
             connection.release();
         }

package/dist/connectors/mysql/dialect.d.ts

@@ -1,8 +1,8 @@
-import type { KtxDialect } from '../../context/connections/dialects.js';
+import type { KtxSqlDialect } from '../../context/connections/dialects.js';
 import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js';
 type MysqlTableNameRef = Pick<KtxTableRef, 'name'> & Partial<Pick<KtxTableRef, 'catalog' | 'db'>>;
 /** @internal */
-export declare class KtxMysqlDialect implements KtxDialect {
+export declare class KtxMysqlDialect implements KtxSqlDialect {
     readonly type: "mysql";
     private readonly typeMappings;
     quoteIdentifier(identifier: string): string;

package/dist/connectors/postgres/connector.d.ts

@@ -112,6 +112,7 @@ export declare class KtxPostgresScanConnector implements KtxScanConnector {
     private readonly poolFactory;
     private readonly endpointResolver?;
     private readonly now;
+    private readonly deadlineMs;
     private readonly dialect;
     private pool;
     private lastIdlePoolError;