@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/dist/commands/connection-metabase-setup.js

@@ -0,0 +1,540 @@
+import { cancel, confirm, intro, isCancel, log, multiselect, note, outro, password, select, text, } from '@clack/prompts';
+import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
+import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory, LocalMetabaseSourceStateReader, MetabaseClient, metabaseRuntimeConfigFromLocalConnection, validateMappingPhysicalMatch, } from '@ktx/context/ingest';
+import { ktxLocalStateDbPath, loadKtxProject, serializeKtxProjectConfig, } from '@ktx/context/project';
+import { createClackSpinner } from '../clack.js';
+import { withMenuOptionsSpacing, withMultiselectNavigation } from '../prompt-navigation.js';
+import { runKtxPublicIngest } from '../public-ingest.js';
+function isMetabaseConnection(connection) {
+    return (String(connection?.driver ?? '')
+        .trim()
+        .toLowerCase() === 'metabase');
+}
+function stringField(value) {
+    return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
+}
+function uniqueSorted(values) {
+    return [...new Set(values)].sort((a, b) => a - b);
+}
+function resolveMetabaseUrl(connection) {
+    return stringField(connection?.api_url) ?? stringField(connection?.apiUrl) ?? stringField(connection?.url);
+}
+function resolveLiteralMetabaseApiKey(connection) {
+    return stringField(connection?.api_key) ?? stringField(connection?.apiKey);
+}
+function listMetabaseConnectionIds(project) {
+    return Object.entries(project.config.connections)
+        .filter(([_connectionId, connection]) => isMetabaseConnection(connection))
+        .map(([connectionId]) => connectionId)
+        .sort();
+}
+function listWarehouseConnectionIds(project) {
+    return Object.entries(project.config.connections)
+        .filter(([connectionId, connection]) => localConnectionToWarehouseDescriptor(connectionId, connection) != null)
+        .map(([connectionId]) => connectionId)
+        .sort();
+}
+function redactSecrets(message, secrets) {
+    let result = message;
+    for (const secret of secrets) {
+        if (!secret) {
+            continue;
+        }
+        result = result.split(secret).join('[redacted]');
+    }
+    return result;
+}
+async function createDefaultMetabaseClient(project, connectionId) {
+    const factory = new DefaultMetabaseConnectionClientFactory((metabaseConnectionId) => metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]), DEFAULT_METABASE_CLIENT_CONFIG);
+    return factory.createClient(connectionId);
+}
+async function defaultMintMetabaseApiKey(args) {
+    const loginClient = new MetabaseClient({ apiUrl: args.url, apiKey: '' }, DEFAULT_METABASE_CLIENT_CONFIG);
+    const sessionId = await loginClient.createSession(args.username, args.password);
+    const sessionClient = new MetabaseClient({ apiUrl: args.url, apiKey: sessionId, authHeaderName: 'X-Metabase-Session' }, DEFAULT_METABASE_CLIENT_CONFIG);
+    const groups = await sessionClient.getPermissionGroups();
+    const adminGroup = groups.find((group) => group.name === 'Administrators');
+    if (!adminGroup) {
+        throw new Error('Metabase Administrators group was not found; create an API key manually and pass --api-key');
+    }
+    const mintedKey = await sessionClient.createApiKey({
+        groupId: adminGroup.id,
+        name: `KTX CLI ${new Date().toISOString()}`,
+    });
+    const trimmedKey = stringField(mintedKey);
+    if (!trimmedKey) {
+        throw new Error('Metabase API key minting returned an empty key');
+    }
+    return trimmedKey;
+}
+function ensureNotCancelled(value, prompts) {
+    if (isCancel(value)) {
+        prompts.cancel('Setup cancelled.');
+        throw new Error('Setup cancelled.');
+    }
+    return value;
+}
+export function createClackMetabaseSetupPromptAdapter() {
+    return {
+        intro(title) {
+            intro(title);
+        },
+        outro(message) {
+            outro(message);
+        },
+        note(message, title) {
+            note(message, title);
+        },
+        log: {
+            info(message) {
+                log.info(message);
+            },
+            step(message) {
+                log.step(message);
+            },
+            success(message) {
+                log.success(message);
+            },
+            warn(message) {
+                log.warn(message);
+            },
+            error(message) {
+                log.error(message);
+            },
+        },
+        spinner() {
+            return createClackSpinner();
+        },
+        async select(options) {
+            return ensureNotCancelled(await select(withMenuOptionsSpacing(options)), this);
+        },
+        async multiselect(options) {
+            return ensureNotCancelled(await multiselect(withMenuOptionsSpacing(options)), this);
+        },
+        async text(options) {
+            return ensureNotCancelled(await text(options), this);
+        },
+        async password(options) {
+            return ensureNotCancelled(await password(options), this);
+        },
+        async confirm(options) {
+            return ensureNotCancelled(await confirm(options), this);
+        },
+        cancel(message) {
+            cancel(message);
+        },
+    };
+}
+function isInteractiveMetabaseSetupIo(args, io) {
+    return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
+}
+function normalizeDiscoveredDatabases(databases) {
+    return databases
+        .filter((database) => database.is_sample !== true)
+        .map((database) => ({
+        id: database.id,
+        name: database.name,
+        engine: stringField(database.engine) ?? 'unknown',
+        host: stringField(database.details?.host) ?? null,
+        dbName: stringField(database.details?.dbname) ?? null,
+    }));
+}
+function targetPhysicalInfo(project, connectionId) {
+    const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
+    if (!descriptor) {
+        return { connection_type: 'UNKNOWN' };
+    }
+    return {
+        connection_type: descriptor.connection_type,
+        host: descriptor.host ?? null,
+        database: descriptor.database ?? null,
+        account: descriptor.account ?? null,
+        project_id: descriptor.project_id ?? null,
+        dataset_id: descriptor.dataset_id ?? null,
+        ...descriptor.connection_params,
+    };
+}
+function noteMetabaseSetupSummary(options) {
+    const mappingLines = options.mappings
+        .map((mapping) => `  ${mapping.metabaseDatabaseId} -> ${mapping.targetConnectionId}`)
+        .join('\n');
+    const syncLines = options.syncEnabledDatabaseIds.map((id) => `  ${id}`).join('\n');
+    options.prompts.note([
+        `Connection: ${options.connectionId}`,
+        `URL: ${options.url}`,
+        '',
+        'Mappings:',
+        mappingLines || '  (none)',
+        '',
+        'Sync enabled:',
+        syncLines || '  (none)',
+    ].join('\n'), 'Summary');
+}
+export async function runKtxConnectionMetabaseSetup(args, io, deps = {}) {
+    let apiKeyForRedaction = args.apiKey;
+    let passwordForRedaction = args.metabasePassword;
+    const interactiveIo = io;
+    const isInteractive = isInteractiveMetabaseSetupIo(args, interactiveIo);
+    const prompts = deps.prompts ?? (isInteractive ? createClackMetabaseSetupPromptAdapter() : undefined);
+    try {
+        if (isInteractive && prompts) {
+            prompts.intro('KTX Metabase setup');
+        }
+        const project = await loadKtxProject({ projectDir: args.projectDir });
+        const existingMetabaseConnectionIds = listMetabaseConnectionIds(project);
+        let connectionId;
+        if (args.connectionId) {
+            connectionId = args.connectionId;
+        }
+        else if (existingMetabaseConnectionIds.length === 1) {
+            const onlyMetabaseConnectionId = existingMetabaseConnectionIds[0];
+            if (!onlyMetabaseConnectionId) {
+                throw new Error('No Metabase connection id was resolved');
+            }
+            connectionId = onlyMetabaseConnectionId;
+        }
+        else if (existingMetabaseConnectionIds.length > 1) {
+            if (!isInteractive || !prompts) {
+                throw new Error(`Multiple Metabase connections found (${existingMetabaseConnectionIds.join(', ')}); select one with --id`);
+            }
+            connectionId = await prompts.select({
+                message: 'Select the Metabase connection to configure',
+                options: existingMetabaseConnectionIds.map((id) => ({ value: id, label: id })),
+            });
+        }
+        else {
+            connectionId = 'metabase';
+        }
+        const existingConnection = project.config.connections[connectionId];
+        const warehouseConnectionIds = listWarehouseConnectionIds(project);
+        if (warehouseConnectionIds.length === 0) {
+            throw new Error('Add a warehouse connection first');
+        }
+        let url = args.url ?? resolveMetabaseUrl(existingConnection);
+        let apiKey = args.apiKey ?? resolveLiteralMetabaseApiKey(existingConnection);
+        apiKeyForRedaction = apiKey;
+        if (!url && isInteractive && prompts) {
+            url = stringField(await prompts.text({
+                message: 'Metabase API URL',
+                placeholder: 'http://localhost:3000',
+            }));
+        }
+        if (args.inputMode === 'disabled' && !url) {
+            throw new Error('missing Metabase URL');
+        }
+        if (!args.apiKey && !args.mintApiKey && apiKey && isInteractive && prompts && !args.yes) {
+            const reuse = await prompts.confirm({
+                message: `Reuse the existing Metabase API key from connections.${connectionId}?`,
+                initialValue: true,
+            });
+            if (!reuse) {
+                apiKey = undefined;
+                apiKeyForRedaction = undefined;
+            }
+        }
+        if (args.mintApiKey) {
+            let username = stringField(args.metabaseUsername);
+            let metabasePassword = stringField(args.metabasePassword);
+            if (isInteractive && prompts) {
+                if (!username) {
+                    username = stringField(await prompts.text({ message: 'Metabase admin username' }));
+                }
+                if (!metabasePassword) {
+                    metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
+                }
+            }
+            if (!username) {
+                throw new Error('--mint-api-key requires --username');
+            }
+            if (!metabasePassword) {
+                throw new Error('--mint-api-key requires --password');
+            }
+            if (!url) {
+                throw new Error('Metabase URL is required (use --url)');
+            }
+            passwordForRedaction = metabasePassword;
+            apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)({ url, username, password: metabasePassword }, io);
+            apiKeyForRedaction = apiKey;
+        }
+        if (!apiKey && isInteractive && prompts) {
+            const credentialMode = await prompts.select({
+                message: 'Metabase credentials',
+                options: [
+                    { value: 'paste', label: 'Paste API key' },
+                    { value: 'mint', label: 'Mint API key' },
+                ],
+            });
+            if (credentialMode === 'paste') {
+                apiKey = stringField(await prompts.password({ message: 'Metabase API key' }));
+                apiKeyForRedaction = apiKey;
+            }
+            else {
+                const username = stringField(await prompts.text({ message: 'Metabase admin username' }));
+                const metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
+                if (!username) {
+                    throw new Error('Metabase username is required');
+                }
+                if (!metabasePassword) {
+                    throw new Error('Metabase password is required');
+                }
+                if (!url) {
+                    throw new Error('Metabase URL is required (use --url)');
+                }
+                passwordForRedaction = metabasePassword;
+                apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)({ url, username, password: metabasePassword }, io);
+                apiKeyForRedaction = apiKey;
+            }
+        }
+        if (args.inputMode === 'disabled' && !apiKey) {
+            throw new Error('missing Metabase API key');
+        }
+        if (!url) {
+            throw new Error('Metabase URL is required (use --url)');
+        }
+        if (!apiKey) {
+            throw new Error('Metabase API key is required (use --api-key)');
+        }
+        const transientConnectionConfig = {
+            ...(existingConnection ?? {}),
+            driver: 'metabase',
+            api_url: url,
+            api_key: apiKey,
+        };
+        const configWithTransient = {
+            ...project.config,
+            connections: {
+                ...project.config.connections,
+                [connectionId]: transientConnectionConfig,
+            },
+        };
+        const discoveryProject = { ...project, config: configWithTransient };
+        for (const mapping of args.mappings) {
+            if (!configWithTransient.connections[mapping.targetConnectionId]) {
+                throw new Error(`Target connection "${mapping.targetConnectionId}" does not exist`);
+            }
+        }
+        const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(discoveryProject, connectionId);
+        try {
+            const authSpinner = isInteractive && prompts ? prompts.spinner() : undefined;
+            authSpinner?.start('Testing Metabase connection');
+            const testResult = await client.testConnection();
+            if (!testResult.success) {
+                authSpinner?.error('Metabase authentication failed');
+                throw new Error(`Metabase authentication failed. Replace connections.${connectionId}.api_key or use --mint-api-key.`);
+            }
+            authSpinner?.stop('Metabase reachable');
+            const discoverySpinner = isInteractive && prompts ? prompts.spinner() : undefined;
+            discoverySpinner?.start('Discovering Metabase databases');
+            const discovered = normalizeDiscoveredDatabases(await client.getDatabases());
+            discoverySpinner?.stop(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`);
+            if (isInteractive && prompts) {
+                prompts.log.success(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`);
+            }
+            if (discovered.length === 0) {
+                throw new Error('Metabase auth worked but no usable databases were returned');
+            }
+            let resolvedMappings = args.mappings;
+            let resolvedSyncEnabledDatabaseIds = args.syncEnabledDatabaseIds;
+            if (resolvedSyncEnabledDatabaseIds.length === 0 && args.yes && resolvedMappings.length > 0) {
+                resolvedSyncEnabledDatabaseIds = uniqueSorted(resolvedMappings.map((mapping) => mapping.metabaseDatabaseId));
+            }
+            if (resolvedMappings.length === 0 && resolvedSyncEnabledDatabaseIds.length === 0) {
+                const onlyDiscoveredDatabase = discovered.length === 1 ? discovered[0] : undefined;
+                const compatibleWarehouses = onlyDiscoveredDatabase
+                    ? warehouseConnectionIds.filter((warehouseConnectionId) => {
+                        const mismatchReason = validateMappingPhysicalMatch({
+                            metabaseEngine: onlyDiscoveredDatabase.engine,
+                            metabaseDbName: onlyDiscoveredDatabase.dbName,
+                            metabaseHost: onlyDiscoveredDatabase.host,
+                        }, targetPhysicalInfo(project, warehouseConnectionId));
+                        return !mismatchReason;
+                    })
+                    : [];
+                const onlyWarehouseConnectionId = compatibleWarehouses[0];
+                if (onlyDiscoveredDatabase && compatibleWarehouses.length === 1 && onlyWarehouseConnectionId) {
+                    if (args.yes) {
+                        resolvedMappings = [
+                            { metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
+                        ];
+                        resolvedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
+                    }
+                    else if (isInteractive && prompts) {
+                        const proposedMappings = [
+                            { metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
+                        ];
+                        const proposedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
+                        noteMetabaseSetupSummary({
+                            prompts,
+                            connectionId,
+                            url,
+                            mappings: proposedMappings,
+                            syncEnabledDatabaseIds: proposedSyncEnabledDatabaseIds,
+                        });
+                        const confirmed = await prompts.confirm({
+                            message: `Map Metabase database "${onlyDiscoveredDatabase.name}" (${onlyDiscoveredDatabase.id}) to "${onlyWarehouseConnectionId}" and enable sync?`,
+                            initialValue: true,
+                        });
+                        if (!confirmed) {
+                            prompts.cancel('Setup cancelled.');
+                            throw new Error('Setup cancelled.');
+                        }
+                        resolvedMappings = proposedMappings;
+                        resolvedSyncEnabledDatabaseIds = proposedSyncEnabledDatabaseIds;
+                    }
+                    else {
+                        throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
+                    }
+                }
+                else if (isInteractive && prompts) {
+                    const selectedDatabaseIds = await prompts.multiselect({
+                        message: withMultiselectNavigation('Select Metabase databases to configure'),
+                        options: discovered.map((database) => ({
+                            value: database.id,
+                            label: `${database.id}: ${database.name}`,
+                            hint: [database.engine, database.host, database.dbName].filter(Boolean).join(' • '),
+                        })),
+                        required: true,
+                    });
+                    resolvedMappings = [];
+                    for (const databaseId of selectedDatabaseIds) {
+                        const database = discovered.find((candidate) => candidate.id === databaseId);
+                        if (!database) {
+                            throw new Error(`Selected database id ${databaseId} was not discovered`);
+                        }
+                        const existingMapping = args.mappings.find((mapping) => mapping.metabaseDatabaseId === databaseId);
+                        if (existingMapping) {
+                            resolvedMappings.push(existingMapping);
+                            continue;
+                        }
+                        const targetConnectionId = await prompts.select({
+                            message: `Map Metabase database ${database.id} ("${database.name}") to which KTX connection?`,
+                            options: warehouseConnectionIds.map((warehouseId) => ({ value: warehouseId, label: warehouseId })),
+                        });
+                        resolvedMappings.push({ metabaseDatabaseId: databaseId, targetConnectionId });
+                    }
+                    const syncIds = await prompts.multiselect({
+                        message: withMultiselectNavigation('Enable sync for which databases?'),
+                        options: selectedDatabaseIds.map((id) => ({ value: id, label: String(id) })),
+                        initialValues: selectedDatabaseIds,
+                        required: true,
+                    });
+                    resolvedSyncEnabledDatabaseIds = uniqueSorted(syncIds);
+                    if (!args.yes) {
+                        noteMetabaseSetupSummary({
+                            prompts,
+                            connectionId,
+                            url,
+                            mappings: resolvedMappings,
+                            syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds,
+                        });
+                        const confirmed = await prompts.confirm({
+                            message: 'Write changes to ktx.yaml and enable sync?',
+                            initialValue: true,
+                        });
+                        if (!confirmed) {
+                            prompts.cancel('Setup cancelled.');
+                            throw new Error('Setup cancelled.');
+                        }
+                    }
+                }
+                else if (args.inputMode === 'disabled') {
+                    throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
+                }
+            }
+            if (args.inputMode === 'disabled' &&
+                resolvedMappings.length > 0 &&
+                resolvedSyncEnabledDatabaseIds.length === 0) {
+                throw new Error('Metabase sync selection is required in --no-input mode; pass --sync <metabaseDatabaseId>');
+            }
+            const discoveredIds = new Set(discovered.map((database) => database.id));
+            for (const mapping of resolvedMappings) {
+                if (!discoveredIds.has(mapping.metabaseDatabaseId)) {
+                    throw new Error(`Mapped database id ${mapping.metabaseDatabaseId} was not discovered`);
+                }
+            }
+            for (const syncId of resolvedSyncEnabledDatabaseIds) {
+                if (!discoveredIds.has(syncId)) {
+                    throw new Error(`Sync database id ${syncId} was not discovered`);
+                }
+            }
+            await project.fileStore.writeFile('ktx.yaml', serializeKtxProjectConfig(configWithTransient), 'ktx', 'ktx@example.com', `Setup Metabase connection ${connectionId}`);
+            const updatedProject = await loadKtxProject({ projectDir: args.projectDir });
+            const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
+            await store.refreshDiscoveredDatabases({ connectionId, discovered });
+            for (const mapping of resolvedMappings) {
+                await store.upsertDatabaseMapping({
+                    connectionId,
+                    metabaseDatabaseId: mapping.metabaseDatabaseId,
+                    targetConnectionId: mapping.targetConnectionId,
+                    syncEnabled: false,
+                    source: 'cli',
+                });
+            }
+            for (const metabaseDatabaseId of resolvedSyncEnabledDatabaseIds) {
+                await store.setMappingSyncEnabled({
+                    connectionId,
+                    metabaseDatabaseId,
+                    syncEnabled: true,
+                });
+            }
+            const existingSyncState = await store.getSourceState(connectionId);
+            await store.setSyncState({
+                connectionId,
+                syncMode: args.syncMode,
+                defaultTagNames: existingSyncState.defaultTagNames,
+                selections: existingSyncState.selections,
+            });
+            const unhydrated = await store.getUnhydratedSyncEnabledMappingIds(connectionId);
+            if (unhydrated.length > 0) {
+                io.stderr.write(`Sync-enabled mappings are missing discovery metadata; run ktx connection mapping refresh ${connectionId} --auto-accept\n`);
+                return 1;
+            }
+            const rows = await store.listDatabaseMappings(connectionId);
+            const physicalFailures = rows.flatMap((row) => {
+                if (!row.targetConnectionId) {
+                    return [];
+                }
+                const reason = validateMappingPhysicalMatch({ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost }, updatedProject.config.connections[row.targetConnectionId]
+                    ? targetPhysicalInfo(updatedProject, row.targetConnectionId)
+                    : { connection_type: 'UNKNOWN' });
+                return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
+            });
+            if (physicalFailures.length > 0) {
+                for (const failure of physicalFailures) {
+                    io.stderr.write(`${failure}\n`);
+                }
+                return 1;
+            }
+            io.stdout.write(`Connection: ${connectionId}\n`);
+            io.stdout.write(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
+            io.stdout.write(`Next: ktx ingest ${connectionId} --project-dir ${args.projectDir}\n`);
+            if (args.runIngest) {
+                const ingestRunner = deps.runPublicIngest ?? runKtxPublicIngest;
+                const exitCode = await ingestRunner({
+                    command: 'run',
+                    projectDir: args.projectDir,
+                    targetConnectionId: connectionId,
+                    all: false,
+                    json: false,
+                    inputMode: 'disabled',
+                }, io);
+                if (exitCode !== 0) {
+                    io.stderr.write(`Ingest failed; re-run: ktx ingest ${connectionId} --project-dir ${args.projectDir}\n`);
+                    return 1;
+                }
+            }
+            if (isInteractive && prompts) {
+                prompts.outro('Metabase setup complete');
+            }
+            return 0;
+        }
+        finally {
+            await client.cleanup();
+        }
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        io.stderr.write(`${redactSecrets(message, [apiKeyForRedaction ?? '', passwordForRedaction ?? '', args.apiKey ?? ''])}\n`);
+        return 1;
+    }
+}

package/dist/commands/connection-metabase-setup.test.d.ts

@@ -0,0 +1 @@
+export {};