@kaelio/ktx 0.4.0 → 0.5.0

package/node_modules/@ktx/context/dist/ingest/adapters/metabase/local-metabase.adapter.d.ts

@@ -1,13 +0,0 @@
-import type { KtxLocalProject, KtxProjectConnectionConfig } from '../../../project/index.js';
-import { type MetabaseClientLogger } from './client.js';
-import { type MetabaseClientConfig, type MetabaseClientRuntimeConfig } from './client-port.js';
-import type { MetabaseFetchLogger } from './fetch.js';
-import { MetabaseSourceAdapter } from './metabase.adapter.js';
-export declare function metabaseRuntimeConfigFromLocalConnection(connectionId: string, connection: KtxProjectConnectionConfig | undefined, env?: NodeJS.ProcessEnv): MetabaseClientRuntimeConfig;
-interface CreateLocalMetabaseSourceAdapterOptions {
-    env?: NodeJS.ProcessEnv;
-    defaultClientConfig?: MetabaseClientConfig;
-    logger?: MetabaseClientLogger & MetabaseFetchLogger;
-}
-export declare function createLocalMetabaseSourceAdapter(project: KtxLocalProject, options?: CreateLocalMetabaseSourceAdapterOptions): MetabaseSourceAdapter;
-export {};

package/node_modules/@ktx/context/dist/ingest/adapters/metabase/local-metabase.adapter.js

@@ -1,41 +0,0 @@
-import { ktxLocalStateDbPath } from '../../../project/index.js';
-import { resolveKtxConfigReference } from '../../../core/config-reference.js';
-import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory, } from './client.js';
-import { IngestMetabaseClientFactory, } from './client-port.js';
-import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js';
-import { MetabaseSourceAdapter } from './metabase.adapter.js';
-function stringField(value) {
-    return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
-}
-function hasNetworkProxy(connection) {
-    return connection.networkProxy != null || connection.network_proxy != null;
-}
-export function metabaseRuntimeConfigFromLocalConnection(connectionId, connection, env = process.env) {
-    if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
-        throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
-    }
-    if (hasNetworkProxy(connection)) {
-        throw new Error(`Standalone KTX does not support proxy-bearing Metabase connections yet. Use hosted Metabase ingest for "${connectionId}" until the KTX Metabase proxy support spec lands.`);
-    }
-    const apiUrl = stringField(connection.api_url);
-    const literalApiKey = stringField(connection.api_key);
-    const apiKeyRef = stringField(connection.api_key_ref);
-    const apiKey = literalApiKey ?? (apiKeyRef ? resolveKtxConfigReference(apiKeyRef, env) : null);
-    if (!apiUrl) {
-        throw new Error(`Connection "${connectionId}" is missing metabase api_url`);
-    }
-    if (!apiKey) {
-        throw new Error(`Connection "${connectionId}" is missing metabase api_key or api_key_ref`);
-    }
-    return { apiUrl, apiKey };
-}
-export function createLocalMetabaseSourceAdapter(project, options = {}) {
-    const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
-    const sourceStateReader = new KtxYamlMetabaseSourceStateReader(project, { discoveryCache });
-    const connectionFactory = new DefaultMetabaseConnectionClientFactory((metabaseConnectionId) => metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId], options.env), options.defaultClientConfig ?? DEFAULT_METABASE_CLIENT_CONFIG, options.logger);
-    return new MetabaseSourceAdapter({
-        clientFactory: new IngestMetabaseClientFactory(connectionFactory),
-        sourceStateReader,
-        ...(options.logger ? { logger: options.logger } : {}),
-    });
-}

package/node_modules/@ktx/context/dist/ingest/adapters/metabase/local-source-state-store.d.ts

@@ -1,43 +0,0 @@
-import { type KtxLocalProject } from '../../../project/index.js';
-import type { DiscoveredMetabaseDatabase } from './mapping.js';
-import type { MetabaseSourceState, MetabaseSourceStateReader } from './source-state-port.js';
-export type LocalMetabaseMappingSource = 'ktx.yaml' | 'refresh';
-interface LocalMetabaseDiscoveryCacheOptions {
-    dbPath: string;
-    now?: () => Date;
-}
-export interface RefreshLocalMetabaseDiscoveredDatabasesInput {
-    connectionId: string;
-    discovered: DiscoveredMetabaseDatabase[];
-}
-export interface LocalMetabaseDiscoveredDatabaseRow extends DiscoveredMetabaseDatabase {
-    updatedAt: string;
-}
-export interface LocalMetabaseMappingListRow {
-    metabaseDatabaseId: number;
-    metabaseDatabaseName: string | null;
-    metabaseEngine: string | null;
-    metabaseHost: string | null;
-    metabaseDbName: string | null;
-    targetConnectionId: string | null;
-    syncEnabled: boolean;
-    source: LocalMetabaseMappingSource;
-}
-export declare class LocalMetabaseDiscoveryCache {
-    private readonly db;
-    private readonly now;
-    constructor(options: LocalMetabaseDiscoveryCacheOptions);
-    refreshDiscoveredDatabases(input: RefreshLocalMetabaseDiscoveredDatabasesInput): Promise<void>;
-    listDiscoveredDatabases(connectionId: string): Promise<LocalMetabaseDiscoveredDatabaseRow[]>;
-    getDiscoveredDatabase(connectionId: string, metabaseDatabaseId: number): Promise<LocalMetabaseDiscoveredDatabaseRow | null>;
-}
-export declare class KtxYamlMetabaseSourceStateReader implements MetabaseSourceStateReader {
-    private readonly project;
-    private readonly options;
-    constructor(project: Pick<KtxLocalProject, 'config'>, options?: {
-        discoveryCache?: LocalMetabaseDiscoveryCache;
-    });
-    getSourceState(connectionId: string): Promise<MetabaseSourceState>;
-    listDatabaseMappings(connectionId: string): Promise<LocalMetabaseMappingListRow[]>;
-}
-export {};

package/node_modules/@ktx/context/dist/ingest/adapters/metabase/local-source-state-store.js

@@ -1,176 +0,0 @@
-import { mkdirSync } from 'node:fs';
-import { dirname } from 'node:path';
-import Database from 'better-sqlite3';
-import { parseMetabaseMappingBootstrap, } from '../../../project/index.js';
-function selectionState(bootstrap) {
-    return [
-        ...bootstrap.selections.collections.map((id) => ({ selectionType: 'collection', metabaseObjectId: id })),
-        ...bootstrap.selections.items.map((id) => ({ selectionType: 'item', metabaseObjectId: id })),
-    ];
-}
-function configuredMappingIds(bootstrap) {
-    return [...new Set([...Object.keys(bootstrap.databaseMappings), ...Object.keys(bootstrap.syncEnabled)].map(Number))].sort((left, right) => left - right);
-}
-function discoveredRowToDatabase(row) {
-    return {
-        id: row.metabase_database_id,
-        name: row.metabase_database_name,
-        engine: row.metabase_engine,
-        host: row.metabase_host,
-        dbName: row.metabase_db_name,
-        updatedAt: row.updated_at,
-    };
-}
-function emptyMetabaseSourceState() {
-    return {
-        syncMode: 'ALL',
-        selections: [],
-        defaultTagNames: [],
-        mappings: [],
-    };
-}
-export class LocalMetabaseDiscoveryCache {
-    db;
-    now;
-    constructor(options) {
-        mkdirSync(dirname(options.dbPath), { recursive: true });
-        this.db = new Database(options.dbPath);
-        this.db.pragma('journal_mode = WAL');
-        this.db.pragma('foreign_keys = ON');
-        this.now = options.now ?? (() => new Date());
-        this.db.exec(`
-      CREATE TABLE IF NOT EXISTS local_metabase_discovered_databases (
-        metabase_connection_id TEXT NOT NULL,
-        metabase_database_id INTEGER NOT NULL,
-        metabase_database_name TEXT NOT NULL,
-        metabase_engine TEXT NOT NULL,
-        metabase_host TEXT,
-        metabase_db_name TEXT,
-        updated_at TEXT NOT NULL,
-        PRIMARY KEY (metabase_connection_id, metabase_database_id)
-      );
-    `);
-    }
-    async refreshDiscoveredDatabases(input) {
-        const timestamp = this.now().toISOString();
-        const refresh = this.db.transaction(() => {
-            const upsert = this.db.prepare(`
-        INSERT INTO local_metabase_discovered_databases (
-          metabase_connection_id,
-          metabase_database_id,
-          metabase_database_name,
-          metabase_engine,
-          metabase_host,
-          metabase_db_name,
-          updated_at
-        )
-        VALUES (?, ?, ?, ?, ?, ?, ?)
-        ON CONFLICT(metabase_connection_id, metabase_database_id) DO UPDATE SET
-          metabase_database_name = excluded.metabase_database_name,
-          metabase_engine = excluded.metabase_engine,
-          metabase_host = excluded.metabase_host,
-          metabase_db_name = excluded.metabase_db_name,
-          updated_at = excluded.updated_at
-      `);
-            for (const database of input.discovered) {
-                upsert.run(input.connectionId, database.id, database.name, database.engine, database.host, database.dbName, timestamp);
-            }
-        });
-        refresh();
-    }
-    async listDiscoveredDatabases(connectionId) {
-        const rows = this.db
-            .prepare(`
-        SELECT
-          metabase_database_id,
-          metabase_database_name,
-          metabase_engine,
-          metabase_host,
-          metabase_db_name,
-          updated_at
-        FROM local_metabase_discovered_databases
-        WHERE metabase_connection_id = ?
-        ORDER BY metabase_database_id
-      `)
-            .all(connectionId);
-        return rows.map(discoveredRowToDatabase);
-    }
-    async getDiscoveredDatabase(connectionId, metabaseDatabaseId) {
-        const row = this.db
-            .prepare(`
-        SELECT
-          metabase_database_id,
-          metabase_database_name,
-          metabase_engine,
-          metabase_host,
-          metabase_db_name,
-          updated_at
-        FROM local_metabase_discovered_databases
-        WHERE metabase_connection_id = ? AND metabase_database_id = ?
-      `)
-            .get(connectionId, metabaseDatabaseId);
-        return row ? discoveredRowToDatabase(row) : null;
-    }
-}
-export class KtxYamlMetabaseSourceStateReader {
-    project;
-    options;
-    constructor(project, options = {}) {
-        this.project = project;
-        this.options = options;
-    }
-    async getSourceState(connectionId) {
-        const connection = this.project.config.connections[connectionId];
-        if (!connection || String(connection.driver ?? '').toLowerCase() !== 'metabase') {
-            return emptyMetabaseSourceState();
-        }
-        const bootstrap = parseMetabaseMappingBootstrap(connectionId, connection);
-        const discovered = new Map((await this.options.discoveryCache?.listDiscoveredDatabases(connectionId))?.map((database) => [database.id, database]) ??
-            []);
-        return {
-            syncMode: bootstrap.syncMode,
-            selections: selectionState(bootstrap),
-            defaultTagNames: bootstrap.defaultTagNames,
-            mappings: configuredMappingIds(bootstrap).map((id) => {
-                const metadata = discovered.get(id);
-                return {
-                    metabaseDatabaseId: id,
-                    metabaseDatabaseName: metadata?.name ?? null,
-                    metabaseEngine: metadata?.engine ?? null,
-                    metabaseHost: metadata?.host ?? null,
-                    metabaseDbName: metadata?.dbName ?? null,
-                    targetConnectionId: bootstrap.databaseMappings[String(id)] ?? null,
-                    syncEnabled: bootstrap.syncEnabled[String(id)] ?? false,
-                };
-            }),
-        };
-    }
-    async listDatabaseMappings(connectionId) {
-        const state = await this.getSourceState(connectionId);
-        const configuredRows = state.mappings.map((mapping) => ({
-            metabaseDatabaseId: mapping.metabaseDatabaseId,
-            metabaseDatabaseName: mapping.metabaseDatabaseName,
-            metabaseEngine: mapping.metabaseEngine,
-            metabaseHost: mapping.metabaseHost ?? null,
-            metabaseDbName: mapping.metabaseDbName ?? null,
-            targetConnectionId: mapping.targetConnectionId,
-            syncEnabled: mapping.syncEnabled,
-            source: 'ktx.yaml',
-        }));
-        const configuredIds = new Set(configuredRows.map((row) => row.metabaseDatabaseId));
-        const discoveredRows = (await this.options.discoveryCache?.listDiscoveredDatabases(connectionId))?.filter((database) => !configuredIds.has(database.id)) ?? [];
-        return [
-            ...configuredRows,
-            ...discoveredRows.map((database) => ({
-                metabaseDatabaseId: database.id,
-                metabaseDatabaseName: database.name,
-                metabaseEngine: database.engine,
-                metabaseHost: database.host,
-                metabaseDbName: database.dbName,
-                targetConnectionId: null,
-                syncEnabled: false,
-                source: 'refresh',
-            })),
-        ].sort((left, right) => left.metabaseDatabaseId - right.metabaseDatabaseId);
-    }
-}

package/node_modules/@ktx/context/dist/ingest/adapters/metabase/local-source-state-store.test.js

@@ -1,99 +0,0 @@
-import { mkdtemp, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-import { buildDefaultKtxProjectConfig } from '../../../project/index.js';
-import { connectionConfigSchema } from '../../../project/driver-schemas.js';
-import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './local-source-state-store.js';
-describe('Metabase YAML source state and discovery cache', () => {
-    let tempDir;
-    let discoveryCache;
-    beforeEach(async () => {
-        tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-cache-'));
-        discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
-    });
-    afterEach(async () => {
-        await rm(tempDir, { recursive: true, force: true });
-    });
-    function projectWithMetabaseMappings(mappings) {
-        return {
-            config: {
-                ...buildDefaultKtxProjectConfig(),
-                connections: {
-                    'prod-metabase': connectionConfigSchema.parse({
-                        driver: 'metabase',
-                        api_url: 'https://metabase.example.com',
-                        mappings,
-                    }),
-                },
-            },
-        };
-    }
-    it('reads Metabase mapping intent from ktx.yaml config', async () => {
-        const reader = new KtxYamlMetabaseSourceStateReader(projectWithMetabaseMappings({
-            databaseMappings: { '2': 'warehouse' },
-            syncEnabled: { '2': true },
-            syncMode: 'ONLY',
-            selections: { collections: [12], items: [99] },
-            defaultTagNames: ['analytics'],
-        }), { discoveryCache });
-        await expect(reader.getSourceState('prod-metabase')).resolves.toEqual({
-            syncMode: 'ONLY',
-            defaultTagNames: ['analytics'],
-            selections: [
-                { selectionType: 'collection', metabaseObjectId: 12 },
-                { selectionType: 'item', metabaseObjectId: 99 },
-            ],
-            mappings: [
-                {
-                    metabaseDatabaseId: 2,
-                    metabaseDatabaseName: null,
-                    metabaseEngine: null,
-                    metabaseHost: null,
-                    metabaseDbName: null,
-                    targetConnectionId: 'warehouse',
-                    syncEnabled: true,
-                },
-            ],
-        });
-    });
-    it('enriches YAML mapping rows with recreatable discovery metadata', async () => {
-        await discoveryCache.refreshDiscoveredDatabases({
-            connectionId: 'prod-metabase',
-            discovered: [{ id: 2, name: 'Analytics', engine: 'postgres', host: 'pg.internal', dbName: 'analytics' }],
-        });
-        const reader = new KtxYamlMetabaseSourceStateReader(projectWithMetabaseMappings({
-            databaseMappings: { '2': 'warehouse' },
-            syncEnabled: { '2': true },
-        }), { discoveryCache });
-        await expect(reader.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
-            {
-                metabaseDatabaseId: 2,
-                metabaseDatabaseName: 'Analytics',
-                metabaseEngine: 'postgres',
-                metabaseHost: 'pg.internal',
-                metabaseDbName: 'analytics',
-                targetConnectionId: 'warehouse',
-                syncEnabled: true,
-                source: 'ktx.yaml',
-            },
-        ]);
-    });
-    it('lists discovered-only rows as refresh cache data without turning them into config state', async () => {
-        await discoveryCache.refreshDiscoveredDatabases({
-            connectionId: 'prod-metabase',
-            discovered: [{ id: 7, name: 'Unmapped', engine: 'mysql', host: 'mysql.internal', dbName: 'sales' }],
-        });
-        const reader = new KtxYamlMetabaseSourceStateReader(projectWithMetabaseMappings({}), { discoveryCache });
-        await expect(reader.getSourceState('prod-metabase')).resolves.toMatchObject({ mappings: [] });
-        await expect(reader.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
-            {
-                metabaseDatabaseId: 7,
-                metabaseDatabaseName: 'Unmapped',
-                targetConnectionId: null,
-                syncEnabled: false,
-                source: 'refresh',
-            },
-        ]);
-    });
-});

package/node_modules/@ktx/context/dist/ingest/adapters/metabase/mapping.d.ts

@@ -1,92 +0,0 @@
-import type { MetabaseRuntimeClient } from './client-port.js';
-export declare const METABASE_ENGINE_TO_CONNECTION_TYPE: {
-    readonly postgres: "POSTGRESQL";
-    readonly bigquery: "BIGQUERY";
-    readonly 'bigquery-cloud-sdk': "BIGQUERY";
-    readonly snowflake: "SNOWFLAKE";
-    readonly sqlserver: "SQLSERVER";
-    readonly mysql: "MYSQL";
-};
-export type MetabaseMappedConnectionType = (typeof METABASE_ENGINE_TO_CONNECTION_TYPE)[keyof typeof METABASE_ENGINE_TO_CONNECTION_TYPE];
-export interface DiscoveredMetabaseDatabase {
-    id: number;
-    name: string;
-    engine: string;
-    host: string | null;
-    dbName: string | null;
-}
-export interface MetabaseMappingDrift {
-    unmappedDiscovered: DiscoveredMetabaseDatabase[];
-    staleMappings: Array<{
-        id: string;
-        reason: 'database_not_found';
-    }>;
-    inSync: Array<{
-        id: number;
-        ktxConnectionId: string;
-    }>;
-}
-export interface MappingPhysicalInfo {
-    metabaseEngine: string | null;
-    metabaseDbName: string | null;
-    metabaseHost: string | null;
-}
-export interface KtxConnectionPhysicalInfo {
-    connection_type: string;
-    database?: unknown;
-    host?: unknown;
-    account?: unknown;
-    dataset_id?: unknown;
-    project_id?: unknown;
-    [key: string]: unknown;
-}
-export interface PhysicalMismatchInput {
-    mappingId: string;
-    metabase: MappingPhysicalInfo;
-    target: KtxConnectionPhysicalInfo;
-}
-export interface PhysicalMismatch {
-    mappingId: string;
-    reason: string;
-}
-export interface MappingRefreshReport {
-    drift: MetabaseMappingDrift;
-    physicalMismatches: PhysicalMismatch[];
-}
-export type MetabaseMappingValidationResult = {
-    ok: true;
-} | {
-    ok: false;
-    errors: Array<{
-        key: string;
-        reason: string;
-    }>;
-};
-export interface AutoMatchCandidate {
-    id: string;
-    name: string;
-    connection_type: string;
-    connection_params: unknown;
-}
-export interface AutoMatchResult {
-    connectionId: string;
-    connectionName: string;
-    reason: 'host_and_database' | 'database_only' | 'host_only';
-}
-export declare function discoverMetabaseDatabases(client: Pick<MetabaseRuntimeClient, 'getDatabases'>): Promise<DiscoveredMetabaseDatabase[]>;
-export declare function computeMetabaseMappingDrift(args: {
-    currentMappings: Record<string, string | null | undefined>;
-    discovered: DiscoveredMetabaseDatabase[];
-}): MetabaseMappingDrift;
-export declare function validateMetabaseMappings(args: {
-    mappings: Record<string, string | null | undefined>;
-    knownKtxConnectionIds: Set<string>;
-}): MetabaseMappingValidationResult;
-export declare function validateMappingPhysicalMatch(mapping: MappingPhysicalInfo, target: KtxConnectionPhysicalInfo): string | null;
-export declare function computeMetabaseMappingPhysicalMismatches(inputs: PhysicalMismatchInput[]): PhysicalMismatch[];
-export declare function refreshMetabaseMapping(args: {
-    client: Pick<MetabaseRuntimeClient, 'getDatabases'>;
-    currentMappings: Record<string, string | null | undefined>;
-    resolveKtxConnectionPhysicalInfo: (ktxConnectionId: string) => Promise<KtxConnectionPhysicalInfo | null>;
-}): Promise<MappingRefreshReport>;
-export declare function findBestMatch(mapping: MappingPhysicalInfo, candidates: AutoMatchCandidate[]): AutoMatchResult | null;