@kaelio/ktx 0.4.1 → 0.6.0

package/dist/setup-databases.js

@@ -3,24 +3,33 @@ import { readFile, writeFile } from 'node:fs/promises';
 import { delimiter, dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { promisify } from 'node:util';
-import { loadKtxProject, markKtxSetupStateStepComplete, serializeKtxProjectConfig, setKtxSetupDatabaseConnectionIds, } from '@ktx/context/project';
+import { getDriverRegistration } from './context/connections/drivers.js';
+import { queryHistoryDialectForConnection } from './context/ingest/adapters/historic-sql/connection-dialect.js';
+import { runHistoricSqlReadinessProbe, } from './context/ingest/historic-sql-probes.js';
+import { serializeKtxProjectConfig } from './context/project/config.js';
+import { loadKtxProject } from './context/project/project.js';
+import { markKtxSetupStateStepComplete, setKtxSetupDatabaseConnectionIds } from './context/project/setup-config.js';
+import { errorMessage, flushPrefixedBufferedCommandOutput, writePrefixedLines, } from './clack.js';
 import { runKtxConnection } from './connection.js';
 import { pickDatabaseScope as defaultPickDatabaseScope, } from './database-tree-picker.js';
 import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
 import { runKtxScan } from './scan.js';
 import { applySetupDatabaseContextDepth } from './setup-database-context-depth.js';
 import { writeProjectLocalSecretReference } from './setup-secrets.js';
-import { createKtxSetupPromptAdapter, } from './setup-prompts.js';
+import { isDemoConnection } from './telemetry/demo-detect.js';
+import { emitTelemetryEvent } from './telemetry/index.js';
+import { createKtxSetupPromptAdapter, createKtxSetupUiAdapter, } from './setup-prompts.js';
 const HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY = 6;
+const KTX_QUICKSTART_URL = 'https://docs.kaelio.com/ktx/docs/getting-started/quickstart';
 const execFileAsync = promisify(execFileCallback);
 const DRIVER_OPTIONS = [
-    { value: 'sqlite', label: 'SQLite' },
     { value: 'postgres', label: 'PostgreSQL' },
+    { value: 'bigquery', label: 'BigQuery' },
+    { value: 'snowflake', label: 'Snowflake' },
     { value: 'mysql', label: 'MySQL' },
     { value: 'clickhouse', label: 'ClickHouse' },
     { value: 'sqlserver', label: 'SQL Server' },
-    { value: 'bigquery', label: 'BigQuery' },
-    { value: 'snowflake', label: 'Snowflake' },
+    { value: 'sqlite', label: 'SQLite' },
 ];
 const DRIVER_LABELS = Object.fromEntries(DRIVER_OPTIONS.map((option) => [option.value, option.label]));
 const HISTORIC_SQL_DIALECT_BY_DRIVER = {
@@ -37,6 +46,13 @@ const DEFAULT_CONNECTION_IDS = {
     bigquery: 'bigquery-warehouse',
     snowflake: 'snowflake-warehouse',
 };
+const SUGGESTED_SCOPE_PATTERN = /^(mart|prod|analytics|core|dim|fact|gold)(_|$)/i;
+const EXCLUDED_SCOPE_PATTERN = /^(information_schema|pg_catalog|pg_toast|_airbyte_|mysql$|performance_schema$|sys$)/i;
+function defaultSuggest(values) {
+    const excluded = new Set(values.filter((value) => EXCLUDED_SCOPE_PATTERN.test(value)));
+    const suggested = new Set(values.filter((value) => !excluded.has(value) && SUGGESTED_SCOPE_PATTERN.test(value)));
+    return { excluded, suggested };
+}
 const SCOPE_DISCOVERY_SPECS = {
     postgres: {
         noun: 'schema',
@@ -44,10 +60,22 @@ const SCOPE_DISCOVERY_SPECS = {
         promptLabel: 'PostgreSQL schemas',
         configArrayField: 'schemas',
         configSingleField: 'schema',
-        defaultSelection(schemas) {
-            const nonPublic = schemas.filter((s) => s !== 'public');
-            return nonPublic.length > 0 ? nonPublic : schemas;
-        },
+        suggest: defaultSuggest,
+    },
+    mysql: {
+        noun: 'database',
+        nounPlural: 'databases',
+        promptLabel: 'MySQL databases',
+        configArrayField: 'schemas',
+        configSingleField: 'schema',
+        suggest: defaultSuggest,
+    },
+    clickhouse: {
+        noun: 'database',
+        nounPlural: 'databases',
+        promptLabel: 'ClickHouse databases',
+        configArrayField: 'databases',
+        suggest: defaultSuggest,
     },
     sqlserver: {
         noun: 'schema',
@@ -55,7 +83,7 @@ const SCOPE_DISCOVERY_SPECS = {
         promptLabel: 'SQL Server schemas',
         configArrayField: 'schemas',
         configSingleField: 'schema',
-        defaultSelection: (schemas) => schemas,
+        suggest: defaultSuggest,
     },
     bigquery: {
         noun: 'dataset',
@@ -63,7 +91,7 @@ const SCOPE_DISCOVERY_SPECS = {
         promptLabel: 'BigQuery datasets',
         configArrayField: 'dataset_ids',
         configSingleField: 'dataset_id',
-        defaultSelection: (datasets) => datasets,
+        suggest: defaultSuggest,
     },
     snowflake: {
         noun: 'schema',
@@ -71,10 +99,7 @@ const SCOPE_DISCOVERY_SPECS = {
         promptLabel: 'Snowflake schemas',
         configArrayField: 'schema_names',
         configSingleField: 'schema_name',
-        defaultSelection(schemas) {
-            const nonPublic = schemas.filter((s) => s !== 'PUBLIC');
-            return nonPublic.length > 0 ? nonPublic : schemas;
-        },
+        suggest: defaultSuggest,
     },
 };
 const DRIVER_CONNECTION_DEFAULTS = {
@@ -169,113 +194,40 @@ function migrateLegacyHistoricSqlConnection(connection) {
     const { dialect: _dialect, ...queryHistory } = legacy;
     return withQueryHistoryConfig(connection, queryHistory);
 }
-function historicSqlProbeFailureLines(error) {
-    if (error instanceof Error && error.name === 'HistoricSqlExtensionMissingError') {
-        return [
-            '  FAIL pg_stat_statements extension is not installed in the connection database',
-            '  Fix: Run (against this database): CREATE EXTENSION pg_stat_statements;',
-            "  Fix: Ensure shared_preload_libraries includes 'pg_stat_statements'.",
-        ];
-    }
-    if (error instanceof Error && error.name === 'HistoricSqlGrantsMissingError') {
-        return [
-            '  FAIL Postgres connection role lacks pg_read_all_stats',
-            '  Fix: Run: GRANT pg_read_all_stats TO <connection role>;',
-        ];
-    }
-    if (error instanceof Error && error.name === 'HistoricSqlVersionUnsupportedError') {
-        return [`  FAIL ${error.message}`];
-    }
-    return [`  FAIL Query history probe failed: ${error instanceof Error ? error.message : String(error)}`];
-}
-async function defaultHistoricSqlProbe(input) {
-    if (input.dialect !== 'postgres') {
+function setupHistoricSqlProbeResult(outcome) {
+    if (!outcome) {
         return { ok: true, lines: [] };
     }
-    const project = await loadKtxProject({ projectDir: input.projectDir });
-    const connection = project.config.connections[input.connectionId];
-    const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]);
-    const postgresConnection = connection;
-    if (!isKtxPostgresConnectionConfig(postgresConnection)) {
-        return {
-            ok: false,
-            lines: [`  FAIL Connection ${input.connectionId} is not a native Postgres connection.`],
-        };
-    }
-    const client = new KtxPostgresHistoricSqlQueryClient({
-        connectionId: input.connectionId,
-        connection: postgresConnection,
-    });
-    try {
-        const result = await new PostgresPgssReader().probe(client);
+    if (outcome.ok) {
+        const { detail, warnings } = outcome.runner.formatSuccessDetail(outcome.result);
         return {
             ok: true,
-            lines: [
-                `  OK pg_stat_statements ready (${result.pgServerVersion})`,
-                ...result.warnings.map((warning) => `  ! ${warning}`),
-            ],
+            lines: [`  OK ${detail}`, ...warnings.map((warning) => `  ! ${warning}`)],
         };
     }
-    catch (error) {
-        return { ok: false, lines: historicSqlProbeFailureLines(error) };
-    }
-    finally {
-        await client.cleanup();
-    }
+    const advice = outcome.runner.fixAdvice(outcome.error);
+    return {
+        ok: false,
+        lines: [`  FAIL ${advice.failHeadline}`, `  Fix: ${advice.remediation}`],
+    };
 }
 async function defaultListSchemas(projectDir, connectionId) {
     const project = await loadKtxProject({ projectDir });
     const connection = project.config.connections[connectionId];
     const driver = normalizeDriver(connection?.driver);
-    if (driver === 'postgres') {
-        const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres');
-        if (!isKtxPostgresConnectionConfig(connection))
-            return [];
-        const connector = new KtxPostgresScanConnector({ connectionId, connection });
-        try {
-            return await connector.listSchemas();
-        }
-        finally {
-            await connector.cleanup();
-        }
-    }
-    if (driver === 'sqlserver') {
-        const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('@ktx/connector-sqlserver');
-        if (!isKtxSqlServerConnectionConfig(connection))
-            return [];
-        const connector = new KtxSqlServerScanConnector({ connectionId, connection });
-        try {
-            return await connector.listSchemas();
-        }
-        finally {
-            await connector.cleanup();
-        }
-    }
-    if (driver === 'bigquery') {
-        const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('@ktx/connector-bigquery');
-        if (!isKtxBigQueryConnectionConfig(connection))
-            return [];
-        const connector = new KtxBigQueryScanConnector({ connectionId, connection });
-        try {
-            return await connector.listDatasets();
-        }
-        finally {
-            await connector.cleanup();
-        }
+    const registration = driver ? getDriverRegistration(driver) : undefined;
+    if (!registration)
+        return [];
+    const connectorModule = await registration.load();
+    if (!connectorModule.isConnectionConfig(connection))
+        return [];
+    const connector = connectorModule.createScanConnector({ connectionId, connection, projectDir });
+    try {
+        return await connector.listSchemas();
     }
-    if (driver === 'snowflake') {
-        const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('@ktx/connector-snowflake');
-        if (!isKtxSnowflakeConnectionConfig(connection))
-            return [];
-        const connector = new KtxSnowflakeScanConnector({ connectionId, connection });
-        try {
-            return await connector.listSchemas();
-        }
-        finally {
-            await connector.cleanup();
-        }
+    finally {
+        await connector.cleanup?.();
     }
-    return [];
 }
 function configuredSchemas(connection, driver) {
     if (!connection)
@@ -291,79 +243,19 @@ async function defaultListTables(projectDir, connectionId, schemasOverride) {
     const connection = project.config.connections[connectionId];
     const driver = normalizeDriver(connection?.driver);
     const schemas = schemasOverride ?? (driver ? configuredSchemas(connection, driver) : undefined);
-    if (driver === 'postgres') {
-        const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres');
-        if (!isKtxPostgresConnectionConfig(connection))
-            return [];
-        const connector = new KtxPostgresScanConnector({ connectionId, connection });
-        try {
-            return await connector.listTables(schemas);
-        }
-        finally {
-            await connector.cleanup();
-        }
-    }
-    if (driver === 'mysql') {
-        const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('@ktx/connector-mysql');
-        if (!isKtxMysqlConnectionConfig(connection))
-            return [];
-        const connector = new KtxMysqlScanConnector({ connectionId, connection });
-        try {
-            return await connector.listTables(schemas);
-        }
-        finally {
-            await connector.cleanup();
-        }
-    }
-    if (driver === 'sqlserver') {
-        const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('@ktx/connector-sqlserver');
-        if (!isKtxSqlServerConnectionConfig(connection))
-            return [];
-        const connector = new KtxSqlServerScanConnector({ connectionId, connection });
-        try {
-            return await connector.listTables(schemas);
-        }
-        finally {
-            await connector.cleanup();
-        }
-    }
-    if (driver === 'bigquery') {
-        const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('@ktx/connector-bigquery');
-        if (!isKtxBigQueryConnectionConfig(connection))
-            return [];
-        const connector = new KtxBigQueryScanConnector({ connectionId, connection });
-        try {
-            return await connector.listTables(schemas);
-        }
-        finally {
-            await connector.cleanup();
-        }
-    }
-    if (driver === 'snowflake') {
-        const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('@ktx/connector-snowflake');
-        if (!isKtxSnowflakeConnectionConfig(connection))
-            return [];
-        const connector = new KtxSnowflakeScanConnector({ connectionId, connection });
-        try {
-            return await connector.listTables(schemas);
-        }
-        finally {
-            await connector.cleanup();
-        }
+    const registration = driver ? getDriverRegistration(driver) : undefined;
+    if (!registration)
+        return [];
+    const connectorModule = await registration.load();
+    if (!connectorModule.isConnectionConfig(connection))
+        return [];
+    const connector = connectorModule.createScanConnector({ connectionId, connection, projectDir });
+    try {
+        return await connector.listTables(schemas);
     }
-    if (driver === 'clickhouse') {
-        const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('@ktx/connector-clickhouse');
-        if (!isKtxClickHouseConnectionConfig(connection))
-            return [];
-        const connector = new KtxClickHouseScanConnector({ connectionId, connection });
-        try {
-            return await connector.listTables(schemas);
-        }
-        finally {
-            await connector.cleanup();
-        }
+    finally {
+        await connector.cleanup?.();
     }
-    return [];
 }
 function existingConnectionIdsByDriver(connections, driver) {
     return Object.entries(connections)
@@ -394,6 +286,7 @@ function configuredPrimarySourcesPrompt(connectionIds) {
         message: `Databases configured: ${connectionIds.join(', ')}\nWhat would you like to do?`,
         options: [
             { value: 'continue', label: 'Continue to context sources' },
+            { value: 'skip-sources', label: 'Skip context sources' },
             { value: 'edit', label: 'Edit an existing database' },
             { value: 'add', label: 'Add another database' },
         ],
@@ -441,6 +334,32 @@ function normalizeFileReference(value) {
     }
     return `file:${normalized}`;
 }
+function displayFileReference(value) {
+    if (value === undefined)
+        return undefined;
+    if (value.startsWith('file:'))
+        return value.slice('file:'.length);
+    return value;
+}
+function scriptedScopeConfigForDriver(driver, databaseSchemas) {
+    if (databaseSchemas.length === 0)
+        return {};
+    const registration = getDriverRegistration(driver);
+    if (!registration?.scopeConfigKey)
+        return {};
+    return { [registration.scopeConfigKey]: databaseSchemas };
+}
+function databaseNameFromLiteralUrl(url) {
+    if (url.startsWith('env:') || url.startsWith('file:')) {
+        return undefined;
+    }
+    try {
+        return new URL(url).pathname.replace(/^\/+/, '') || undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
 async function promptCredential(input) {
     const value = await input.prompts.password({
         message: withTextInputNavigation(input.message),
@@ -504,7 +423,7 @@ async function buildFieldsConnectionConfig(input) {
         database,
         username,
         ...(passwordRef ? { password: passwordRef } : {}),
-        ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+        ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
     };
 }
 async function buildPastedUrlConnectionConfig(input) {
@@ -519,9 +438,10 @@ async function buildPastedUrlConnectionConfig(input) {
         return {
             driver: input.driver,
             url,
-            ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+            ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
         };
     }
+    const database = input.driver === 'clickhouse' ? databaseNameFromLiteralUrl(url) : undefined;
     if (urlHasCredentials(url)) {
         const ref = await writeProjectLocalSecretReference({
             projectDir: input.args.projectDir,
@@ -531,13 +451,15 @@ async function buildPastedUrlConnectionConfig(input) {
         return {
             driver: input.driver,
             url: ref,
-            ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+            ...(database ? { database } : {}),
+            ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
         };
     }
     return {
         driver: input.driver,
         url,
-        ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+        ...(database ? { database } : {}),
+        ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
     };
 }
 async function buildUrlConnectionConfig(input) {
@@ -546,6 +468,7 @@ async function buildUrlConnectionConfig(input) {
     if (input.args.databaseUrl) {
         const url = normalizeInputReference(input.args.databaseUrl);
         if (urlHasCredentials(url)) {
+            const database = input.driver === 'clickhouse' ? databaseNameFromLiteralUrl(url) : undefined;
             const ref = await writeProjectLocalSecretReference({
                 projectDir: input.args.projectDir,
                 fileName: `${input.connectionId}-url`,
@@ -554,13 +477,16 @@ async function buildUrlConnectionConfig(input) {
             return {
                 driver: input.driver,
                 url: ref,
-                ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+                ...(database ? { database } : {}),
+                ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
             };
         }
+        const database = input.driver === 'clickhouse' ? databaseNameFromLiteralUrl(url) : undefined;
         return {
             driver: input.driver,
             url,
-            ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+            ...(database ? { database } : {}),
+            ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
         };
     }
     const label = driverLabel(input.driver);
@@ -602,22 +528,19 @@ async function buildConnectionConfig(input) {
         });
     }
     if (driver === 'bigquery') {
-        const datasetId = await promptText(prompts, 'BigQuery dataset\nFor example analytics.', stringConfigField(input.existingConnection, 'dataset_id'));
-        if (datasetId === undefined)
-            return 'back';
-        const credentialsPath = await promptText(prompts, 'Path to service account JSON file', stringConfigField(input.existingConnection, 'credentials_json'));
+        const credentialsPath = await promptText(prompts, 'Path to service account JSON file', displayFileReference(stringConfigField(input.existingConnection, 'credentials_json')));
         if (credentialsPath === undefined)
             return 'back';
         const location = await promptText(prompts, 'BigQuery location\nPress Enter for US, or enter a location like EU.', stringConfigField(input.existingConnection, 'location') ?? 'US');
         if (location === undefined)
             return 'back';
-        if (!datasetId || !credentialsPath)
+        if (!credentialsPath)
             return null;
         return {
             driver: 'bigquery',
-            dataset_id: datasetId,
             credentials_json: normalizeFileReference(credentialsPath),
             ...(location ? { location } : {}),
+            ...scriptedScopeConfigForDriver('bigquery', args.databaseSchemas),
         };
     }
     if (driver === 'snowflake') {
@@ -630,36 +553,83 @@ async function buildConnectionConfig(input) {
         const database = await promptText(prompts, 'Snowflake database name', stringConfigField(input.existingConnection, 'database'));
         if (database === undefined)
             return 'back';
-        const schemaName = await promptText(prompts, 'Snowflake schema\nPress Enter for PUBLIC, or enter a schema name.', stringConfigField(input.existingConnection, 'schema_name') ?? 'PUBLIC');
-        if (schemaName === undefined)
-            return 'back';
         const username = await promptText(prompts, 'Snowflake username', stringConfigField(input.existingConnection, 'username'));
         if (username === undefined)
             return 'back';
-        const passwordRef = await promptCredential({
-            prompts,
-            message: 'Snowflake password',
-            projectDir: args.projectDir,
-            connectionId: input.connectionId,
-            secretName: 'password', // pragma: allowlist secret
+        const authChoice = await prompts.select({
+            message: 'Snowflake authentication method',
+            options: [
+                { value: 'password', label: 'Password' },
+                { value: 'rsa', label: 'Key-pair (RSA / JWT)' },
+                { value: 'back', label: 'Back' },
+            ],
         });
-        if (passwordRef === 'back')
-            return 'back'; // pragma: allowlist secret
+        if (authChoice === 'back')
+            return 'back';
+        const authMethod = authChoice === 'rsa' ? 'rsa' : 'password';
+        let passwordRef = null;
+        let privateKeyInput;
+        let passphraseRef = null;
+        if (authMethod === 'password') {
+            const ref = await promptCredential({
+                prompts,
+                message: 'Snowflake password',
+                projectDir: args.projectDir,
+                connectionId: input.connectionId,
+                secretName: 'password', // pragma: allowlist secret
+            });
+            if (ref === 'back')
+                return 'back'; // pragma: allowlist secret
+            passwordRef = ref;
+        }
+        else {
+            privateKeyInput = await promptText(prompts, 'Path to Snowflake private key (PEM)\nFor example ~/.ssh/snowflake_rsa_key.p8, or $ENV_VAR / env:NAME / file:/abs/path.', displayFileReference(stringConfigField(input.existingConnection, 'privateKey')));
+            if (privateKeyInput === undefined)
+                return 'back';
+            const phr = await promptCredential({
+                prompts,
+                message: 'Private key passphrase (optional)\nPress Enter to skip.',
+                projectDir: args.projectDir,
+                connectionId: input.connectionId,
+                secretName: 'snowflake-passphrase', // pragma: allowlist secret
+            });
+            if (phr === 'back')
+                return 'back';
+            passphraseRef = phr;
+        }
         const role = await promptText(prompts, 'Snowflake role (optional)\nPress Enter to skip.', stringConfigField(input.existingConnection, 'role'));
         if (role === undefined)
             return 'back';
-        const resolvedPasswordRef = passwordRef ?? stringConfigField(input.existingConnection, 'password');
-        if (!account || !warehouse || !database || !schemaName || !username || !resolvedPasswordRef)
+        if (authMethod === 'password') {
+            const resolvedPasswordRef = passwordRef ?? stringConfigField(input.existingConnection, 'password');
+            if (!account || !warehouse || !database || !username || !resolvedPasswordRef)
+                return null;
+            return {
+                driver: 'snowflake',
+                authMethod: 'password',
+                account,
+                warehouse,
+                database,
+                username,
+                password: resolvedPasswordRef,
+                ...(role ? { role } : {}),
+            };
+        }
+        const resolvedPrivateKey = privateKeyInput
+            ? normalizeFileReference(privateKeyInput)
+            : stringConfigField(input.existingConnection, 'privateKey');
+        if (!account || !warehouse || !database || !username || !resolvedPrivateKey)
             return null;
+        const resolvedPassphrase = passphraseRef ?? stringConfigField(input.existingConnection, 'passphrase');
         return {
             driver: 'snowflake',
-            authMethod: 'password',
+            authMethod: 'rsa',
             account,
             warehouse,
             database,
-            schema_name: schemaName,
             username,
-            password: resolvedPasswordRef,
+            privateKey: resolvedPrivateKey,
+            ...(resolvedPassphrase ? { passphrase: resolvedPassphrase } : {}),
             ...(role ? { role } : {}),
         };
     }
@@ -698,8 +668,7 @@ async function maybeApplyHistoricSqlConfig(input) {
             return input.connection;
         }
     }
-    const existingRecord = queryHistoryConfigRecord(input.connection) ?? historicSqlConfigRecord(input.connection) ?? {};
-    const { dialect: _dialect, ...existing } = existingRecord;
+    const { dialect: _dialect, ...existing } = queryHistoryConfigRecord(input.connection) ?? {};
     if (!enabled) {
         return withQueryHistoryConfig(input.connection, { ...existing, enabled: false });
     }
@@ -770,23 +739,6 @@ function createBufferedCommandIo() {
         },
     };
 }
-function flushBufferedCommandOutput(io, bufferedIo) {
-    const stdout = bufferedIo.stdoutText();
-    const stderr = bufferedIo.stderrText();
-    if (stdout.length > 0) {
-        io.stdout.write(stdout);
-    }
-    if (stderr.length > 0) {
-        io.stderr.write(stderr);
-    }
-}
-function writePrefixedLines(write, output) {
-    for (const line of output.split(/\r?\n/)) {
-        if (line.length > 0) {
-            write(`│  ${line}\n`);
-        }
-    }
-}
 function envWithCurrentNodeFirst(env = process.env) {
     return {
         ...env,
@@ -852,10 +804,6 @@ async function defaultRebuildNativeSqlite(io) {
         return typeof error?.code === 'number' ? error.code : 1;
     }
 }
-function flushPrefixedBufferedCommandOutput(io, bufferedIo) {
-    writePrefixedLines((chunk) => io.stdout.write(chunk), bufferedIo.stdoutText());
-    writePrefixedLines((chunk) => io.stderr.write(chunk), bufferedIo.stderrText());
-}
 function nativeSqliteAbiMismatchDetail(output) {
     const mentionsBetterSqlite = /\bbetter-sqlite3\b|better_sqlite3/i.test(output);
     const mentionsAbiMismatch = /compiled against a different Node\.js version|NODE_MODULE_VERSION/i.test(output);
@@ -905,24 +853,43 @@ function writeSetupSection(io, title, lines) {
 }
 async function writeConnectionConfig(input) {
     const project = await loadKtxProject({ projectDir: input.projectDir });
-    const migratedConnections = Object.fromEntries(Object.entries(project.config.connections).map(([connectionId, connection]) => [
-        connectionId,
-        migrateLegacyHistoricSqlConnection(connection),
-    ]));
-    const nextConnection = migrateLegacyHistoricSqlConnection(input.connection);
     const config = {
         ...project.config,
         connections: {
-            ...migratedConnections,
-            [input.connectionId]: nextConnection,
+            ...project.config.connections,
+            [input.connectionId]: input.connection,
         },
     };
     await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
-    const queryHistory = queryHistoryConfigRecord(nextConnection);
+    if (input.io) {
+        await emitTelemetryEvent({
+            name: 'connection_added',
+            projectDir: input.projectDir,
+            io: input.io,
+            fields: {
+                driver: String(input.connection.driver ?? 'unknown').toLowerCase(),
+                isDemoConnection: isDemoConnection(input.connectionId, input.connection),
+            },
+        });
+    }
+    const queryHistory = queryHistoryConfigRecord(input.connection);
     if (queryHistory?.enabled === true) {
         await ensureHistoricSqlIngestDefaults(input.projectDir);
     }
 }
+async function disableConnectionQueryHistory(projectDir, connectionId) {
+    const project = await loadKtxProject({ projectDir });
+    const connection = project.config.connections[connectionId];
+    if (!connection) {
+        return;
+    }
+    const existing = queryHistoryConfigRecord(connection) ?? historicSqlConfigRecord(connection) ?? {};
+    await writeConnectionConfig({
+        projectDir,
+        connectionId,
+        connection: withQueryHistoryConfig(connection, { ...existing, enabled: false }),
+    });
+}
 async function createConnectionConfigRollback(projectDir, connectionId) {
     const project = await loadKtxProject({ projectDir });
     const previousConnection = project.config.connections[connectionId];
@@ -951,10 +918,16 @@ function withExistingPrimaryEditPromptDefaults(input) {
         if (!(Array.isArray(nextArray) && nextArray.length > 0) &&
             Array.isArray(previousArray) &&
             previousArray.length > 0) {
-            delete merged[spec.configSingleField];
+            if (spec.configSingleField) {
+                delete merged[spec.configSingleField];
+            }
             merged[spec.configArrayField] = previousArray;
         }
-        else if (!Object.hasOwn(input.next, spec.configArrayField) && !Object.hasOwn(input.next, spec.configSingleField)) {
+        else if (!Object.hasOwn(input.next, spec.configArrayField) &&
+            (!spec.configSingleField || !Object.hasOwn(input.next, spec.configSingleField))) {
+            if (!spec.configSingleField) {
+                return merged;
+            }
             const previousSingle = input.previous[spec.configSingleField];
             if (typeof previousSingle === 'string' && previousSingle.trim().length > 0) {
                 merged[spec.configSingleField] = previousSingle;
@@ -964,6 +937,9 @@ function withExistingPrimaryEditPromptDefaults(input) {
     if (!Object.hasOwn(input.next, 'enabled_tables') && Array.isArray(input.previous.enabled_tables)) {
         merged.enabled_tables = input.previous.enabled_tables;
     }
+    if (!Object.hasOwn(input.next, 'context') && input.previous.context !== undefined) {
+        merged.context = input.previous.context;
+    }
     return merged;
 }
 function configuredScopeValues(connection, spec) {
@@ -975,6 +951,9 @@ function configuredScopeValues(connection, spec) {
             .filter((v) => typeof v === 'string' && v.trim().length > 0)
             .map((v) => v.trim());
     }
+    if (!spec.configSingleField) {
+        return [];
+    }
     const singleVal = connection[spec.configSingleField];
     return typeof singleVal === 'string' && singleVal.trim().length > 0 ? [singleVal.trim()] : [];
 }
@@ -993,6 +972,13 @@ async function writeScopeConfig(input) {
         },
     });
 }
+async function promptCommaSeparatedScope(input) {
+    const example = input.spec.nounPlural === 'datasets' ? 'sales, marketing' : 'SALES, MARKETING';
+    const value = await promptText(input.prompts, `Enter ${input.spec.nounPlural} for ${input.connectionId} as a comma-separated list (e.g. ${example}).`);
+    if (value === undefined)
+        return undefined;
+    return unique(value.split(',').map((part) => part.trim()));
+}
 async function maybeConfigureDatabaseScope(input) {
     const project = await loadKtxProject({ projectDir: input.projectDir });
     const connection = project.config.connections[input.connectionId];
@@ -1044,74 +1030,77 @@ async function maybeConfigureDatabaseScope(input) {
             spec,
         });
     }
-    writeSetupSection(input.io, 'Discovering tables', [
-        `Connecting to ${input.connectionId}…`,
-    ]);
-    const schemasFilter = await (async () => {
-        if (cliSchemas.length > 0)
-            return cliSchemas;
-        if (!spec)
-            return [];
+    writeSetupSection(input.io, 'Discovering tables', [`Connecting to ${input.connectionId}…`]);
+    let effectiveCliSchemas = cliSchemas;
+    let listedSchemas;
+    if (cliSchemas.length > 0) {
+        listedSchemas = cliSchemas;
+    }
+    else if (!spec) {
+        listedSchemas = [];
+    }
+    else {
         try {
-            return unique(await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId));
+            listedSchemas = await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId);
         }
         catch (error) {
-            const detail = error instanceof Error ? error.message : String(error);
-            input.io.stderr.write(`Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; ${detail}\n`);
-            return [];
+            writePrefixedLines((chunk) => input.io.stderr.write(chunk), `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; ${errorMessage(error)}`);
+            const typed = await promptCommaSeparatedScope({
+                prompts: input.prompts,
+                connectionId: input.connectionId,
+                spec,
+            });
+            if (typed === undefined)
+                return 'back';
+            effectiveCliSchemas = typed;
+            listedSchemas = typed;
+            if (typed.length > 0) {
+                await writeScopeConfig({
+                    projectDir: input.projectDir,
+                    connectionId: input.connectionId,
+                    values: typed,
+                    spec,
+                });
+            }
         }
-    })();
-    let discovered;
-    try {
-        discovered = await (input.deps.listTables ?? defaultListTables)(input.projectDir, input.connectionId, schemasFilter.length > 0 ? schemasFilter : undefined);
     }
-    catch (error) {
-        const detail = error instanceof Error ? error.message : String(error);
-        input.io.stderr.write(input.forcePrompt === true
-            ? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}\n`
-            : `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}\n`);
-        return input.forcePrompt === true ? 'failed' : 'ready';
-    }
-    if (discovered.length === 0) {
-        if (input.forcePrompt === true) {
-            input.io.stderr.write(`No tables discovered for ${input.connectionId}; edit was not saved.\n`);
-        }
-        return input.forcePrompt === true ? 'failed' : 'ready';
+    const schemas = unique(listedSchemas);
+    if (spec && schemas.length === 0) {
+        return 'ready';
     }
-    const allQualified = discovered.map((t) => `${t.schema}.${t.name}`);
-    const schemasInDiscovery = unique(discovered.map((t) => t.schema));
-    const defaultSchemas = (() => {
-        if (cliSchemas.length > 0)
-            return cliSchemas;
-        if (!spec)
-            return schemasInDiscovery;
-        return spec.defaultSelection(schemasInDiscovery);
-    })();
+    const schemaSuggestion = effectiveCliSchemas.length > 0
+        ? { excluded: new Set(), suggested: new Set(effectiveCliSchemas) }
+        : spec?.suggest(schemas) ?? { excluded: new Set(), suggested: new Set() };
     const existingEnabled = hasExistingTables && input.forcePrompt === true
-        ? (existingTables ?? []).filter((table) => typeof table === 'string' && allQualified.includes(table))
+        ? (existingTables ?? []).filter((table) => typeof table === 'string')
         : [];
-    let activeSchemas;
-    let enabledTables;
-    if (discovered.length === 1) {
-        enabledTables = allQualified;
-        activeSchemas = spec ? schemasInDiscovery : [];
-    }
-    else {
-        const pickResult = await (input.deps.pickDatabaseScope ?? defaultPickDatabaseScope)({
+    let pickResult;
+    try {
+        pickResult = await (input.deps.pickDatabaseScope ?? defaultPickDatabaseScope)({
             connectionId: input.connectionId,
             schemaNoun: spec?.noun ?? 'schema',
             schemaNounPlural: spec?.nounPlural ?? 'schemas',
-            discovered,
+            schemas,
+            schemaSuggestion,
             existing: { enabledTables: existingEnabled },
-            defaultSchemas,
             supportsSchemaScope: spec !== undefined,
+            initialSchemas: effectiveCliSchemas.length > 0 ? effectiveCliSchemas : undefined,
+            prompts: input.prompts,
+            listTablesForSchemas: (selectedSchemas) => (input.deps.listTables ?? defaultListTables)(input.projectDir, input.connectionId, selectedSchemas),
         }, input.io);
-        if (pickResult.kind === 'back') {
-            return 'back';
-        }
-        enabledTables = pickResult.enabledTables;
-        activeSchemas = pickResult.activeSchemas;
     }
+    catch (error) {
+        const detail = errorMessage(error);
+        writePrefixedLines((chunk) => input.io.stderr.write(chunk), input.forcePrompt === true
+            ? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}`
+            : `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}`);
+        return input.forcePrompt === true ? 'failed' : 'ready';
+    }
+    if (pickResult.kind === 'back') {
+        return 'back';
+    }
+    const enabledTables = pickResult.enabledTables;
+    const activeSchemas = pickResult.activeSchemas;
     if (spec) {
         await writeScopeConfig({
             projectDir: input.projectDir,
@@ -1128,6 +1117,7 @@ async function maybeConfigureDatabaseScope(input) {
         projectDir: input.projectDir,
         connectionId: input.connectionId,
         connection: { ...currentConnection, enabled_tables: enabledTables },
+        io: input.io,
     });
     if (spec && activeSchemas.length > 0) {
         const capitalNounPlural = spec.nounPlural[0].toUpperCase() + spec.nounPlural.slice(1);
@@ -1136,7 +1126,7 @@ async function maybeConfigureDatabaseScope(input) {
         ]);
     }
     writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [
-        `✓ ${enabledTables.length}/${discovered.length} tables enabled`,
+        `✓ ${enabledTables.length} tables enabled`,
     ]);
     return 'ready';
 }
@@ -1173,23 +1163,31 @@ async function maybeRunHistoricSqlSetupProbe(input) {
     const project = await loadKtxProject({ projectDir: input.projectDir });
     const connection = project.config.connections[input.connectionId];
     const queryHistory = queryHistoryConfigRecord(connection) ?? historicSqlConfigRecord(connection);
-    const driver = normalizeDriver(connection?.driver);
-    if (queryHistory?.enabled !== true || driver !== 'postgres') {
-        return;
+    if (queryHistory?.enabled !== true) {
+        return true;
+    }
+    if (!connection) {
+        return true;
+    }
+    const dialect = queryHistoryDialectForConnection(connection);
+    if (!dialect) {
+        return true;
     }
     input.io.stdout.write('│  Query history probe...\n');
-    const probe = input.deps.historicSqlProbe ?? defaultHistoricSqlProbe;
-    const result = await probe({
+    const probe = input.deps.historicSqlReadinessProbe ?? runHistoricSqlReadinessProbe;
+    const result = setupHistoricSqlProbeResult(await probe({
         projectDir: input.projectDir,
         connectionId: input.connectionId,
-        dialect: 'postgres',
-    });
+        connection,
+        env: process.env,
+    }));
     for (const line of result.lines) {
         input.io.stdout.write(`│${line}\n`);
     }
     if (!result.ok) {
-        input.io.stdout.write('│  Setup written; first ingest run will fail until fixed.\n');
+        input.io.stdout.write('│  Setup written; query history will be skipped until fixed.\n');
     }
+    return result.ok;
 }
 async function applyHistoricSqlConfigToExistingConnection(input) {
     if (input.args.inputMode === 'disabled' &&
@@ -1255,8 +1253,8 @@ async function validateAndScanConnection(input) {
     const testIo = createBufferedCommandIo();
     const testCode = await testConnection(input.projectDir, input.connectionId, testIo);
     if (testCode !== 0) {
-        flushBufferedCommandOutput(input.io, testIo);
-        input.io.stderr.write(`Connection test failed for ${input.connectionId}.\n`);
+        flushPrefixedBufferedCommandOutput(input.io, testIo);
+        writePrefixedLines((chunk) => input.io.stderr.write(chunk), `Connection test failed for ${input.connectionId}.`);
         return 'failed';
     }
     const testOutput = testIo.stdoutText();
@@ -1268,7 +1266,7 @@ async function validateAndScanConnection(input) {
     if (scopeStatus !== 'ready') {
         return scopeStatus;
     }
-    await maybeRunHistoricSqlSetupProbe({
+    const queryHistoryAvailable = await maybeRunHistoricSqlSetupProbe({
         projectDir: input.projectDir,
         connectionId: input.connectionId,
         io: input.io,
@@ -1314,7 +1312,7 @@ async function validateAndScanConnection(input) {
             ].join('\n'));
         }
         if (scanCode !== 0) {
-            return 'failed';
+            return queryHistoryAvailable ? 'failed' : 'failed-query-history-unavailable';
         }
     }
     const scanOutput = scanIo.stdoutText();
@@ -1336,6 +1334,7 @@ async function chooseDrivers(args, io, prompts, options) {
         return 'missing-input';
     }
     const initialValues = unique(options?.initialDrivers ?? []);
+    createKtxSetupUiAdapter().note(`Get demo credentials from the Quickstart: ${KTX_QUICKSTART_URL}`, '🎁 Need a warehouse to play with?', io);
     const choices = await prompts.multiselect({
         message: withMultiselectNavigation('Which databases should KTX connect to?'),
         options: [...DRIVER_OPTIONS],
@@ -1430,7 +1429,7 @@ async function runPrimarySourceFullEdit(input) {
     const existing = project.config.connections[input.connectionId];
     const driver = normalizeDriver(existing?.driver);
     if (!existing || !driver) {
-        input.io.stderr.write(`Connection "${input.connectionId}" is not a configured database.\n`);
+        writePrefixedLines((chunk) => input.io.stderr.write(chunk), `Connection "${input.connectionId}" is not a configured database.`);
         return 'failed';
     }
     const rollback = await createConnectionConfigRollback(input.projectDir, input.connectionId);
@@ -1464,14 +1463,10 @@ async function runPrimarySourceFullEdit(input) {
         connectionId: input.connectionId,
         connection: withExistingPrimaryEditPromptDefaults({
             previous: existing,
-            next: {
-                ...withHistoricSql,
-                ...(!Object.hasOwn(withHistoricSql, 'historicSql') && existing.historicSql !== undefined
-                    ? { historicSql: existing.historicSql }
-                    : {}),
-            },
+            next: withHistoricSql,
             driver,
         }),
+        io: input.io,
     });
     const validated = await validateAndScanConnection({
         projectDir: input.projectDir,
@@ -1484,7 +1479,7 @@ async function runPrimarySourceFullEdit(input) {
     });
     if (validated !== 'ready') {
         await rollback();
-        return validated;
+        return validated === 'failed-query-history-unavailable' ? 'failed' : validated;
     }
     return 'ready';
 }
@@ -1537,6 +1532,15 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
                 await markDatabasesComplete(args.projectDir, selectedConnectionIds);
                 return { status: 'ready', projectDir: args.projectDir, connectionIds: selectedConnectionIds };
             }
+            if (action === 'skip-sources') {
+                await markDatabasesComplete(args.projectDir, selectedConnectionIds);
+                return {
+                    status: 'ready',
+                    projectDir: args.projectDir,
+                    connectionIds: selectedConnectionIds,
+                    skipSources: true,
+                };
+            }
             if (action === 'edit') {
                 const connectionId = await choosePrimarySourceToEdit({
                     projectDir: args.projectDir,
@@ -1600,7 +1604,7 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
                 });
             }
             catch (error) {
-                io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+                writePrefixedLines((chunk) => io.stderr.write(chunk), errorMessage(error));
                 return { status: 'failed', projectDir: args.projectDir };
             }
             if (connectionChoice === 'back') {
@@ -1703,6 +1707,7 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
                     projectDir: args.projectDir,
                     connectionId: connectionChoice.connectionId,
                     connection: withContextDepth,
+                    io,
                 });
             }
             else {
@@ -1731,6 +1736,7 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
                     projectDir: args.projectDir,
                     connectionId: connectionChoice.connectionId,
                     connection: withContextDepth,
+                    io,
                 });
             }
             let connectionSkipped = false;
@@ -1753,14 +1759,18 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
                 }
                 if (args.inputMode === 'disabled')
                     return { status: 'failed', projectDir: args.projectDir };
+                const failureOptions = [
+                    { value: 'retry', label: 'Retry connection test' },
+                    { value: 're-enter', label: 'Re-enter connection details' },
+                    ...(setupStatus === 'failed-query-history-unavailable'
+                        ? [{ value: 'disable-query-history', label: 'Disable query history and retry' }]
+                        : []),
+                    { value: 'skip', label: 'Skip this database' },
+                    { value: 'back', label: 'Back' },
+                ];
                 const action = await prompts.select({
                     message: `Database setup failed for ${connectionChoice.connectionId}`,
-                    options: [
-                        { value: 'retry', label: 'Retry connection test' },
-                        { value: 're-enter', label: 'Re-enter connection details' },
-                        { value: 'skip', label: 'Skip this database' },
-                        { value: 'back', label: 'Back' },
-                    ],
+                    options: failureOptions,
                 });
                 if (action === 'back') {
                     if (!canReturnToDriverSelection)
@@ -1782,6 +1792,17 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
                         prompts,
                     });
                 }
+                else if (action === 'disable-query-history') {
+                    await disableConnectionQueryHistory(args.projectDir, connectionChoice.connectionId);
+                    setupStatus = await validateAndScanConnection({
+                        projectDir: args.projectDir,
+                        connectionId: connectionChoice.connectionId,
+                        io,
+                        deps,
+                        args,
+                        prompts,
+                    });
+                }
                 else if (action === 're-enter') {
                     const connection = await buildConnectionConfig({
                         driver,
@@ -1821,6 +1842,7 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
                         projectDir: args.projectDir,
                         connectionId: connectionChoice.connectionId,
                         connection: withContextDepth,
+                        io,
                     });
                     setupStatus = await validateAndScanConnection({
                         projectDir: args.projectDir,