@kaelio/ktx 0.4.1 → 0.5.0

package/dist/setup-databases.js

@@ -3,13 +3,17 @@ import { readFile, writeFile } from 'node:fs/promises';
 import { delimiter, dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { promisify } from 'node:util';
-import { loadKtxProject, markKtxSetupStateStepComplete, serializeKtxProjectConfig, setKtxSetupDatabaseConnectionIds, } from '@ktx/context/project';
+import { serializeKtxProjectConfig } from './context/project/config.js';
+import { loadKtxProject } from './context/project/project.js';
+import { markKtxSetupStateStepComplete, setKtxSetupDatabaseConnectionIds } from './context/project/setup-config.js';
 import { runKtxConnection } from './connection.js';
 import { pickDatabaseScope as defaultPickDatabaseScope, } from './database-tree-picker.js';
 import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js';
 import { runKtxScan } from './scan.js';
 import { applySetupDatabaseContextDepth } from './setup-database-context-depth.js';
 import { writeProjectLocalSecretReference } from './setup-secrets.js';
+import { isDemoConnection } from './telemetry/demo-detect.js';
+import { emitTelemetryEvent } from './telemetry/index.js';
 import { createKtxSetupPromptAdapter, } from './setup-prompts.js';
 const HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY = 6;
 const execFileAsync = promisify(execFileCallback);
@@ -37,6 +41,13 @@ const DEFAULT_CONNECTION_IDS = {
     bigquery: 'bigquery-warehouse',
     snowflake: 'snowflake-warehouse',
 };
+const SUGGESTED_SCOPE_PATTERN = /^(mart|prod|analytics|core|dim|fact|gold)(_|$)/i;
+const EXCLUDED_SCOPE_PATTERN = /^(information_schema|pg_catalog|pg_toast|_airbyte_|mysql$|performance_schema$|sys$)/i;
+function defaultSuggest(values) {
+    const excluded = new Set(values.filter((value) => EXCLUDED_SCOPE_PATTERN.test(value)));
+    const suggested = new Set(values.filter((value) => !excluded.has(value) && SUGGESTED_SCOPE_PATTERN.test(value)));
+    return { excluded, suggested };
+}
 const SCOPE_DISCOVERY_SPECS = {
     postgres: {
         noun: 'schema',
@@ -44,10 +55,22 @@ const SCOPE_DISCOVERY_SPECS = {
         promptLabel: 'PostgreSQL schemas',
         configArrayField: 'schemas',
         configSingleField: 'schema',
-        defaultSelection(schemas) {
-            const nonPublic = schemas.filter((s) => s !== 'public');
-            return nonPublic.length > 0 ? nonPublic : schemas;
-        },
+        suggest: defaultSuggest,
+    },
+    mysql: {
+        noun: 'database',
+        nounPlural: 'databases',
+        promptLabel: 'MySQL databases',
+        configArrayField: 'schemas',
+        configSingleField: 'schema',
+        suggest: defaultSuggest,
+    },
+    clickhouse: {
+        noun: 'database',
+        nounPlural: 'databases',
+        promptLabel: 'ClickHouse databases',
+        configArrayField: 'databases',
+        suggest: defaultSuggest,
     },
     sqlserver: {
         noun: 'schema',
@@ -55,7 +78,7 @@ const SCOPE_DISCOVERY_SPECS = {
         promptLabel: 'SQL Server schemas',
         configArrayField: 'schemas',
         configSingleField: 'schema',
-        defaultSelection: (schemas) => schemas,
+        suggest: defaultSuggest,
     },
     bigquery: {
         noun: 'dataset',
@@ -63,7 +86,7 @@ const SCOPE_DISCOVERY_SPECS = {
         promptLabel: 'BigQuery datasets',
         configArrayField: 'dataset_ids',
         configSingleField: 'dataset_id',
-        defaultSelection: (datasets) => datasets,
+        suggest: defaultSuggest,
     },
     snowflake: {
         noun: 'schema',
@@ -71,10 +94,7 @@ const SCOPE_DISCOVERY_SPECS = {
         promptLabel: 'Snowflake schemas',
         configArrayField: 'schema_names',
         configSingleField: 'schema_name',
-        defaultSelection(schemas) {
-            const nonPublic = schemas.filter((s) => s !== 'PUBLIC');
-            return nonPublic.length > 0 ? nonPublic : schemas;
-        },
+        suggest: defaultSuggest,
     },
 };
 const DRIVER_CONNECTION_DEFAULTS = {
@@ -131,12 +151,6 @@ function numberConfigField(connection, field) {
     const value = connection?.[field];
     return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
 }
-function historicSqlConfigRecord(connection) {
-    const historicSql = connection?.historicSql;
-    return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql)
-        ? historicSql
-        : null;
-}
 function contextRecord(connection) {
     const context = connection?.context;
     return context && typeof context === 'object' && !Array.isArray(context) ? context : {};
@@ -147,28 +161,15 @@ function queryHistoryConfigRecord(connection) {
         ? queryHistory
         : null;
 }
-function stripLegacyHistoricSql(connection) {
-    const { historicSql: _historicSql, ...rest } = connection;
-    return rest;
-}
 function withQueryHistoryConfig(connection, queryHistory) {
     return {
-        ...stripLegacyHistoricSql(connection),
+        ...connection,
         context: {
             ...contextRecord(connection),
             queryHistory,
         },
     };
 }
-function migrateLegacyHistoricSqlConnection(connection) {
-    const existingQueryHistory = queryHistoryConfigRecord(connection);
-    const legacy = historicSqlConfigRecord(connection);
-    if (existingQueryHistory || !legacy) {
-        return existingQueryHistory ? stripLegacyHistoricSql(connection) : connection;
-    }
-    const { dialect: _dialect, ...queryHistory } = legacy;
-    return withQueryHistoryConfig(connection, queryHistory);
-}
 function historicSqlProbeFailureLines(error) {
     if (error instanceof Error && error.name === 'HistoricSqlExtensionMissingError') {
         return [
@@ -178,6 +179,13 @@ function historicSqlProbeFailureLines(error) {
         ];
     }
     if (error instanceof Error && error.name === 'HistoricSqlGrantsMissingError') {
+        const dialect = error.dialect;
+        if (dialect === 'snowflake') {
+            return [
+                '  FAIL Snowflake role cannot read SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY',
+                '  Fix: Run (as ACCOUNTADMIN): GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE <connection role>;',
+            ];
+        }
         return [
             '  FAIL Postgres connection role lacks pg_read_all_stats',
             '  Fix: Run: GRANT pg_read_all_stats TO <connection role>;',
@@ -189,12 +197,22 @@ function historicSqlProbeFailureLines(error) {
     return [`  FAIL Query history probe failed: ${error instanceof Error ? error.message : String(error)}`];
 }
 async function defaultHistoricSqlProbe(input) {
-    if (input.dialect !== 'postgres') {
-        return { ok: true, lines: [] };
+    if (input.dialect === 'postgres') {
+        return probePostgresHistoricSql(input);
     }
+    if (input.dialect === 'snowflake') {
+        return probeSnowflakeHistoricSql(input);
+    }
+    return { ok: true, lines: [] };
+}
+async function probePostgresHistoricSql(input) {
     const project = await loadKtxProject({ projectDir: input.projectDir });
     const connection = project.config.connections[input.connectionId];
-    const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]);
+    const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient }, { isKtxPostgresConnectionConfig }] = await Promise.all([
+        import('./context/ingest/adapters/historic-sql/postgres-pgss-reader.js'),
+        import('./connectors/postgres/historic-sql-query-client.js'),
+        import('./connectors/postgres/connector.js'),
+    ]);
     const postgresConnection = connection;
     if (!isKtxPostgresConnectionConfig(postgresConnection)) {
         return {
@@ -223,12 +241,49 @@ async function defaultHistoricSqlProbe(input) {
         await client.cleanup();
     }
 }
+async function probeSnowflakeHistoricSql(input) {
+    const project = await loadKtxProject({ projectDir: input.projectDir });
+    const connection = project.config.connections[input.connectionId];
+    const [{ SnowflakeHistoricSqlQueryHistoryReader }, { KtxSnowflakeHistoricSqlQueryClient }, { isKtxSnowflakeConnectionConfig }] = await Promise.all([
+        import('./context/ingest/adapters/historic-sql/snowflake-query-history-reader.js'),
+        import('./connectors/snowflake/historic-sql-query-client.js'),
+        import('./connectors/snowflake/connector.js'),
+    ]);
+    if (!isKtxSnowflakeConnectionConfig(connection)) {
+        return {
+            ok: false,
+            lines: [`  FAIL Connection ${input.connectionId} is not a native Snowflake connection.`],
+        };
+    }
+    const client = new KtxSnowflakeHistoricSqlQueryClient({
+        connectionId: input.connectionId,
+        connection,
+        projectDir: input.projectDir,
+    });
+    try {
+        const result = await new SnowflakeHistoricSqlQueryHistoryReader().probe(client);
+        return {
+            ok: true,
+            lines: [
+                '  OK SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY accessible',
+                ...result.warnings.map((warning) => `  ! ${warning}`),
+            ],
+        };
+    }
+    catch (error) {
+        return { ok: false, lines: historicSqlProbeFailureLines(error) };
+    }
+    finally {
+        await client.cleanup();
+    }
+}
 async function defaultListSchemas(projectDir, connectionId) {
     const project = await loadKtxProject({ projectDir });
     const connection = project.config.connections[connectionId];
     const driver = normalizeDriver(connection?.driver);
     if (driver === 'postgres') {
-        const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres');
+        const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');
+        ;
         if (!isKtxPostgresConnectionConfig(connection))
             return [];
         const connector = new KtxPostgresScanConnector({ connectionId, connection });
@@ -240,7 +295,8 @@ async function defaultListSchemas(projectDir, connectionId) {
         }
     }
     if (driver === 'sqlserver') {
-        const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('@ktx/connector-sqlserver');
+        const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');
+        ;
         if (!isKtxSqlServerConnectionConfig(connection))
             return [];
         const connector = new KtxSqlServerScanConnector({ connectionId, connection });
@@ -251,8 +307,35 @@ async function defaultListSchemas(projectDir, connectionId) {
             await connector.cleanup();
         }
     }
+    if (driver === 'mysql') {
+        const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/connector.js');
+        ;
+        if (!isKtxMysqlConnectionConfig(connection))
+            return [];
+        const connector = new KtxMysqlScanConnector({ connectionId, connection });
+        try {
+            return await connector.listSchemas();
+        }
+        finally {
+            await connector.cleanup();
+        }
+    }
+    if (driver === 'clickhouse') {
+        const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/connector.js');
+        ;
+        if (!isKtxClickHouseConnectionConfig(connection))
+            return [];
+        const connector = new KtxClickHouseScanConnector({ connectionId, connection });
+        try {
+            return await connector.listSchemas();
+        }
+        finally {
+            await connector.cleanup();
+        }
+    }
     if (driver === 'bigquery') {
-        const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('@ktx/connector-bigquery');
+        const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');
+        ;
         if (!isKtxBigQueryConnectionConfig(connection))
             return [];
         const connector = new KtxBigQueryScanConnector({ connectionId, connection });
@@ -264,10 +347,11 @@ async function defaultListSchemas(projectDir, connectionId) {
         }
     }
     if (driver === 'snowflake') {
-        const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('@ktx/connector-snowflake');
+        const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');
+        ;
         if (!isKtxSnowflakeConnectionConfig(connection))
             return [];
-        const connector = new KtxSnowflakeScanConnector({ connectionId, connection });
+        const connector = new KtxSnowflakeScanConnector({ connectionId, connection, projectDir });
         try {
             return await connector.listSchemas();
         }
@@ -292,7 +376,8 @@ async function defaultListTables(projectDir, connectionId, schemasOverride) {
     const driver = normalizeDriver(connection?.driver);
     const schemas = schemasOverride ?? (driver ? configuredSchemas(connection, driver) : undefined);
     if (driver === 'postgres') {
-        const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres');
+        const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('./connectors/postgres/connector.js');
+        ;
         if (!isKtxPostgresConnectionConfig(connection))
             return [];
         const connector = new KtxPostgresScanConnector({ connectionId, connection });
@@ -304,7 +389,8 @@ async function defaultListTables(projectDir, connectionId, schemasOverride) {
         }
     }
     if (driver === 'mysql') {
-        const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('@ktx/connector-mysql');
+        const { KtxMysqlScanConnector, isKtxMysqlConnectionConfig } = await import('./connectors/mysql/connector.js');
+        ;
         if (!isKtxMysqlConnectionConfig(connection))
             return [];
         const connector = new KtxMysqlScanConnector({ connectionId, connection });
@@ -316,7 +402,8 @@ async function defaultListTables(projectDir, connectionId, schemasOverride) {
         }
     }
     if (driver === 'sqlserver') {
-        const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('@ktx/connector-sqlserver');
+        const { KtxSqlServerScanConnector, isKtxSqlServerConnectionConfig } = await import('./connectors/sqlserver/connector.js');
+        ;
         if (!isKtxSqlServerConnectionConfig(connection))
             return [];
         const connector = new KtxSqlServerScanConnector({ connectionId, connection });
@@ -328,7 +415,8 @@ async function defaultListTables(projectDir, connectionId, schemasOverride) {
         }
     }
     if (driver === 'bigquery') {
-        const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('@ktx/connector-bigquery');
+        const { KtxBigQueryScanConnector, isKtxBigQueryConnectionConfig } = await import('./connectors/bigquery/connector.js');
+        ;
         if (!isKtxBigQueryConnectionConfig(connection))
             return [];
         const connector = new KtxBigQueryScanConnector({ connectionId, connection });
@@ -340,10 +428,11 @@ async function defaultListTables(projectDir, connectionId, schemasOverride) {
         }
     }
     if (driver === 'snowflake') {
-        const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('@ktx/connector-snowflake');
+        const { KtxSnowflakeScanConnector, isKtxSnowflakeConnectionConfig } = await import('./connectors/snowflake/connector.js');
+        ;
         if (!isKtxSnowflakeConnectionConfig(connection))
             return [];
-        const connector = new KtxSnowflakeScanConnector({ connectionId, connection });
+        const connector = new KtxSnowflakeScanConnector({ connectionId, connection, projectDir });
         try {
             return await connector.listTables(schemas);
         }
@@ -352,7 +441,8 @@ async function defaultListTables(projectDir, connectionId, schemasOverride) {
         }
     }
     if (driver === 'clickhouse') {
-        const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('@ktx/connector-clickhouse');
+        const { KtxClickHouseScanConnector, isKtxClickHouseConnectionConfig } = await import('./connectors/clickhouse/connector.js');
+        ;
         if (!isKtxClickHouseConnectionConfig(connection))
             return [];
         const connector = new KtxClickHouseScanConnector({ connectionId, connection });
@@ -441,6 +531,33 @@ function normalizeFileReference(value) {
     }
     return `file:${normalized}`;
 }
+function displayFileReference(value) {
+    if (value === undefined)
+        return undefined;
+    if (value.startsWith('file:'))
+        return value.slice('file:'.length);
+    return value;
+}
+function scriptedScopeConfigForDriver(driver, databaseSchemas) {
+    if (databaseSchemas.length === 0)
+        return {};
+    if (driver === 'bigquery')
+        return { dataset_ids: databaseSchemas };
+    if (driver === 'clickhouse')
+        return { databases: databaseSchemas };
+    return { schemas: databaseSchemas };
+}
+function databaseNameFromLiteralUrl(url) {
+    if (url.startsWith('env:') || url.startsWith('file:')) {
+        return undefined;
+    }
+    try {
+        return new URL(url).pathname.replace(/^\/+/, '') || undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
 async function promptCredential(input) {
     const value = await input.prompts.password({
         message: withTextInputNavigation(input.message),
@@ -504,7 +621,7 @@ async function buildFieldsConnectionConfig(input) {
         database,
         username,
         ...(passwordRef ? { password: passwordRef } : {}),
-        ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+        ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
     };
 }
 async function buildPastedUrlConnectionConfig(input) {
@@ -519,9 +636,10 @@ async function buildPastedUrlConnectionConfig(input) {
         return {
             driver: input.driver,
             url,
-            ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+            ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
         };
     }
+    const database = input.driver === 'clickhouse' ? databaseNameFromLiteralUrl(url) : undefined;
     if (urlHasCredentials(url)) {
         const ref = await writeProjectLocalSecretReference({
             projectDir: input.args.projectDir,
@@ -531,13 +649,15 @@ async function buildPastedUrlConnectionConfig(input) {
         return {
             driver: input.driver,
             url: ref,
-            ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+            ...(database ? { database } : {}),
+            ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
         };
     }
     return {
         driver: input.driver,
         url,
-        ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+        ...(database ? { database } : {}),
+        ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
     };
 }
 async function buildUrlConnectionConfig(input) {
@@ -546,6 +666,7 @@ async function buildUrlConnectionConfig(input) {
     if (input.args.databaseUrl) {
         const url = normalizeInputReference(input.args.databaseUrl);
         if (urlHasCredentials(url)) {
+            const database = input.driver === 'clickhouse' ? databaseNameFromLiteralUrl(url) : undefined;
             const ref = await writeProjectLocalSecretReference({
                 projectDir: input.args.projectDir,
                 fileName: `${input.connectionId}-url`,
@@ -554,13 +675,16 @@ async function buildUrlConnectionConfig(input) {
             return {
                 driver: input.driver,
                 url: ref,
-                ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+                ...(database ? { database } : {}),
+                ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
             };
         }
+        const database = input.driver === 'clickhouse' ? databaseNameFromLiteralUrl(url) : undefined;
         return {
             driver: input.driver,
             url,
-            ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
+            ...(database ? { database } : {}),
+            ...scriptedScopeConfigForDriver(input.driver, input.args.databaseSchemas),
         };
     }
     const label = driverLabel(input.driver);
@@ -602,22 +726,19 @@ async function buildConnectionConfig(input) {
         });
     }
     if (driver === 'bigquery') {
-        const datasetId = await promptText(prompts, 'BigQuery dataset\nFor example analytics.', stringConfigField(input.existingConnection, 'dataset_id'));
-        if (datasetId === undefined)
-            return 'back';
-        const credentialsPath = await promptText(prompts, 'Path to service account JSON file', stringConfigField(input.existingConnection, 'credentials_json'));
+        const credentialsPath = await promptText(prompts, 'Path to service account JSON file', displayFileReference(stringConfigField(input.existingConnection, 'credentials_json')));
         if (credentialsPath === undefined)
             return 'back';
         const location = await promptText(prompts, 'BigQuery location\nPress Enter for US, or enter a location like EU.', stringConfigField(input.existingConnection, 'location') ?? 'US');
         if (location === undefined)
             return 'back';
-        if (!datasetId || !credentialsPath)
+        if (!credentialsPath)
             return null;
         return {
             driver: 'bigquery',
-            dataset_id: datasetId,
             credentials_json: normalizeFileReference(credentialsPath),
             ...(location ? { location } : {}),
+            ...scriptedScopeConfigForDriver('bigquery', args.databaseSchemas),
         };
     }
     if (driver === 'snowflake') {
@@ -630,36 +751,83 @@ async function buildConnectionConfig(input) {
         const database = await promptText(prompts, 'Snowflake database name', stringConfigField(input.existingConnection, 'database'));
         if (database === undefined)
             return 'back';
-        const schemaName = await promptText(prompts, 'Snowflake schema\nPress Enter for PUBLIC, or enter a schema name.', stringConfigField(input.existingConnection, 'schema_name') ?? 'PUBLIC');
-        if (schemaName === undefined)
-            return 'back';
         const username = await promptText(prompts, 'Snowflake username', stringConfigField(input.existingConnection, 'username'));
         if (username === undefined)
             return 'back';
-        const passwordRef = await promptCredential({
-            prompts,
-            message: 'Snowflake password',
-            projectDir: args.projectDir,
-            connectionId: input.connectionId,
-            secretName: 'password', // pragma: allowlist secret
+        const authChoice = await prompts.select({
+            message: 'Snowflake authentication method',
+            options: [
+                { value: 'password', label: 'Password' },
+                { value: 'rsa', label: 'Key-pair (RSA / JWT)' },
+                { value: 'back', label: 'Back' },
+            ],
         });
-        if (passwordRef === 'back')
-            return 'back'; // pragma: allowlist secret
+        if (authChoice === 'back')
+            return 'back';
+        const authMethod = authChoice === 'rsa' ? 'rsa' : 'password';
+        let passwordRef = null;
+        let privateKeyInput;
+        let passphraseRef = null;
+        if (authMethod === 'password') {
+            const ref = await promptCredential({
+                prompts,
+                message: 'Snowflake password',
+                projectDir: args.projectDir,
+                connectionId: input.connectionId,
+                secretName: 'password', // pragma: allowlist secret
+            });
+            if (ref === 'back')
+                return 'back'; // pragma: allowlist secret
+            passwordRef = ref;
+        }
+        else {
+            privateKeyInput = await promptText(prompts, 'Path to Snowflake private key (PEM)\nFor example ~/.ssh/snowflake_rsa_key.p8, or $ENV_VAR / env:NAME / file:/abs/path.', displayFileReference(stringConfigField(input.existingConnection, 'privateKey')));
+            if (privateKeyInput === undefined)
+                return 'back';
+            const phr = await promptCredential({
+                prompts,
+                message: 'Private key passphrase (optional)\nPress Enter to skip.',
+                projectDir: args.projectDir,
+                connectionId: input.connectionId,
+                secretName: 'snowflake-passphrase', // pragma: allowlist secret
+            });
+            if (phr === 'back')
+                return 'back';
+            passphraseRef = phr;
+        }
         const role = await promptText(prompts, 'Snowflake role (optional)\nPress Enter to skip.', stringConfigField(input.existingConnection, 'role'));
         if (role === undefined)
             return 'back';
-        const resolvedPasswordRef = passwordRef ?? stringConfigField(input.existingConnection, 'password');
-        if (!account || !warehouse || !database || !schemaName || !username || !resolvedPasswordRef)
+        if (authMethod === 'password') {
+            const resolvedPasswordRef = passwordRef ?? stringConfigField(input.existingConnection, 'password');
+            if (!account || !warehouse || !database || !username || !resolvedPasswordRef)
+                return null;
+            return {
+                driver: 'snowflake',
+                authMethod: 'password',
+                account,
+                warehouse,
+                database,
+                username,
+                password: resolvedPasswordRef,
+                ...(role ? { role } : {}),
+            };
+        }
+        const resolvedPrivateKey = privateKeyInput
+            ? normalizeFileReference(privateKeyInput)
+            : stringConfigField(input.existingConnection, 'privateKey');
+        if (!account || !warehouse || !database || !username || !resolvedPrivateKey)
             return null;
+        const resolvedPassphrase = passphraseRef ?? stringConfigField(input.existingConnection, 'passphrase');
         return {
             driver: 'snowflake',
-            authMethod: 'password',
+            authMethod: 'rsa',
             account,
             warehouse,
             database,
-            schema_name: schemaName,
             username,
-            password: resolvedPasswordRef,
+            privateKey: resolvedPrivateKey,
+            ...(resolvedPassphrase ? { passphrase: resolvedPassphrase } : {}),
             ...(role ? { role } : {}),
         };
     }
@@ -698,8 +866,7 @@ async function maybeApplyHistoricSqlConfig(input) {
             return input.connection;
         }
     }
-    const existingRecord = queryHistoryConfigRecord(input.connection) ?? historicSqlConfigRecord(input.connection) ?? {};
-    const { dialect: _dialect, ...existing } = existingRecord;
+    const { dialect: _dialect, ...existing } = queryHistoryConfigRecord(input.connection) ?? {};
     if (!enabled) {
         return withQueryHistoryConfig(input.connection, { ...existing, enabled: false });
     }
@@ -905,20 +1072,26 @@ function writeSetupSection(io, title, lines) {
 }
 async function writeConnectionConfig(input) {
     const project = await loadKtxProject({ projectDir: input.projectDir });
-    const migratedConnections = Object.fromEntries(Object.entries(project.config.connections).map(([connectionId, connection]) => [
-        connectionId,
-        migrateLegacyHistoricSqlConnection(connection),
-    ]));
-    const nextConnection = migrateLegacyHistoricSqlConnection(input.connection);
     const config = {
         ...project.config,
         connections: {
-            ...migratedConnections,
-            [input.connectionId]: nextConnection,
+            ...project.config.connections,
+            [input.connectionId]: input.connection,
         },
     };
     await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
-    const queryHistory = queryHistoryConfigRecord(nextConnection);
+    if (input.io) {
+        await emitTelemetryEvent({
+            name: 'connection_added',
+            projectDir: input.projectDir,
+            io: input.io,
+            fields: {
+                driver: String(input.connection.driver ?? 'unknown').toLowerCase(),
+                isDemoConnection: isDemoConnection(input.connectionId, input.connection),
+            },
+        });
+    }
+    const queryHistory = queryHistoryConfigRecord(input.connection);
     if (queryHistory?.enabled === true) {
         await ensureHistoricSqlIngestDefaults(input.projectDir);
     }
@@ -951,10 +1124,16 @@ function withExistingPrimaryEditPromptDefaults(input) {
         if (!(Array.isArray(nextArray) && nextArray.length > 0) &&
             Array.isArray(previousArray) &&
             previousArray.length > 0) {
-            delete merged[spec.configSingleField];
+            if (spec.configSingleField) {
+                delete merged[spec.configSingleField];
+            }
             merged[spec.configArrayField] = previousArray;
         }
-        else if (!Object.hasOwn(input.next, spec.configArrayField) && !Object.hasOwn(input.next, spec.configSingleField)) {
+        else if (!Object.hasOwn(input.next, spec.configArrayField) &&
+            (!spec.configSingleField || !Object.hasOwn(input.next, spec.configSingleField))) {
+            if (!spec.configSingleField) {
+                return merged;
+            }
             const previousSingle = input.previous[spec.configSingleField];
             if (typeof previousSingle === 'string' && previousSingle.trim().length > 0) {
                 merged[spec.configSingleField] = previousSingle;
@@ -964,6 +1143,9 @@ function withExistingPrimaryEditPromptDefaults(input) {
     if (!Object.hasOwn(input.next, 'enabled_tables') && Array.isArray(input.previous.enabled_tables)) {
         merged.enabled_tables = input.previous.enabled_tables;
     }
+    if (!Object.hasOwn(input.next, 'context') && input.previous.context !== undefined) {
+        merged.context = input.previous.context;
+    }
     return merged;
 }
 function configuredScopeValues(connection, spec) {
@@ -975,6 +1157,9 @@ function configuredScopeValues(connection, spec) {
             .filter((v) => typeof v === 'string' && v.trim().length > 0)
             .map((v) => v.trim());
     }
+    if (!spec.configSingleField) {
+        return [];
+    }
     const singleVal = connection[spec.configSingleField];
     return typeof singleVal === 'string' && singleVal.trim().length > 0 ? [singleVal.trim()] : [];
 }
@@ -993,6 +1178,13 @@ async function writeScopeConfig(input) {
         },
     });
 }
+async function promptCommaSeparatedScope(input) {
+    const example = input.spec.nounPlural === 'datasets' ? 'sales, marketing' : 'SALES, MARKETING';
+    const value = await promptText(input.prompts, `Enter ${input.spec.nounPlural} for ${input.connectionId} as a comma-separated list (e.g. ${example}).`);
+    if (value === undefined)
+        return undefined;
+    return unique(value.split(',').map((part) => part.trim()));
+}
 async function maybeConfigureDatabaseScope(input) {
     const project = await loadKtxProject({ projectDir: input.projectDir });
     const connection = project.config.connections[input.connectionId];
@@ -1044,74 +1236,78 @@ async function maybeConfigureDatabaseScope(input) {
             spec,
         });
     }
-    writeSetupSection(input.io, 'Discovering tables', [
-        `Connecting to ${input.connectionId}…`,
-    ]);
-    const schemasFilter = await (async () => {
-        if (cliSchemas.length > 0)
-            return cliSchemas;
-        if (!spec)
-            return [];
+    writeSetupSection(input.io, 'Discovering tables', [`Connecting to ${input.connectionId}…`]);
+    let effectiveCliSchemas = cliSchemas;
+    let listedSchemas;
+    if (cliSchemas.length > 0) {
+        listedSchemas = cliSchemas;
+    }
+    else if (!spec) {
+        listedSchemas = [];
+    }
+    else {
         try {
-            return unique(await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId));
+            listedSchemas = await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId);
         }
         catch (error) {
             const detail = error instanceof Error ? error.message : String(error);
             input.io.stderr.write(`Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; ${detail}\n`);
-            return [];
+            const typed = await promptCommaSeparatedScope({
+                prompts: input.prompts,
+                connectionId: input.connectionId,
+                spec,
+            });
+            if (typed === undefined)
+                return 'back';
+            effectiveCliSchemas = typed;
+            listedSchemas = typed;
+            if (typed.length > 0) {
+                await writeScopeConfig({
+                    projectDir: input.projectDir,
+                    connectionId: input.connectionId,
+                    values: typed,
+                    spec,
+                });
+            }
         }
-    })();
-    let discovered;
-    try {
-        discovered = await (input.deps.listTables ?? defaultListTables)(input.projectDir, input.connectionId, schemasFilter.length > 0 ? schemasFilter : undefined);
-    }
-    catch (error) {
-        const detail = error instanceof Error ? error.message : String(error);
-        input.io.stderr.write(input.forcePrompt === true
-            ? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}\n`
-            : `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}\n`);
-        return input.forcePrompt === true ? 'failed' : 'ready';
     }
-    if (discovered.length === 0) {
-        if (input.forcePrompt === true) {
-            input.io.stderr.write(`No tables discovered for ${input.connectionId}; edit was not saved.\n`);
-        }
-        return input.forcePrompt === true ? 'failed' : 'ready';
+    const schemas = unique(listedSchemas);
+    if (spec && schemas.length === 0) {
+        return 'ready';
     }
-    const allQualified = discovered.map((t) => `${t.schema}.${t.name}`);
-    const schemasInDiscovery = unique(discovered.map((t) => t.schema));
-    const defaultSchemas = (() => {
-        if (cliSchemas.length > 0)
-            return cliSchemas;
-        if (!spec)
-            return schemasInDiscovery;
-        return spec.defaultSelection(schemasInDiscovery);
-    })();
+    const schemaSuggestion = effectiveCliSchemas.length > 0
+        ? { excluded: new Set(), suggested: new Set(effectiveCliSchemas) }
+        : spec?.suggest(schemas) ?? { excluded: new Set(), suggested: new Set() };
     const existingEnabled = hasExistingTables && input.forcePrompt === true
-        ? (existingTables ?? []).filter((table) => typeof table === 'string' && allQualified.includes(table))
+        ? (existingTables ?? []).filter((table) => typeof table === 'string')
         : [];
-    let activeSchemas;
-    let enabledTables;
-    if (discovered.length === 1) {
-        enabledTables = allQualified;
-        activeSchemas = spec ? schemasInDiscovery : [];
-    }
-    else {
-        const pickResult = await (input.deps.pickDatabaseScope ?? defaultPickDatabaseScope)({
+    let pickResult;
+    try {
+        pickResult = await (input.deps.pickDatabaseScope ?? defaultPickDatabaseScope)({
             connectionId: input.connectionId,
             schemaNoun: spec?.noun ?? 'schema',
             schemaNounPlural: spec?.nounPlural ?? 'schemas',
-            discovered,
+            schemas,
+            schemaSuggestion,
             existing: { enabledTables: existingEnabled },
-            defaultSchemas,
             supportsSchemaScope: spec !== undefined,
+            initialSchemas: effectiveCliSchemas.length > 0 ? effectiveCliSchemas : undefined,
+            prompts: input.prompts,
+            listTablesForSchemas: (selectedSchemas) => (input.deps.listTables ?? defaultListTables)(input.projectDir, input.connectionId, selectedSchemas),
         }, input.io);
-        if (pickResult.kind === 'back') {
-            return 'back';
-        }
-        enabledTables = pickResult.enabledTables;
-        activeSchemas = pickResult.activeSchemas;
     }
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        input.io.stderr.write(input.forcePrompt === true
+            ? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}\n`
+            : `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}\n`);
+        return input.forcePrompt === true ? 'failed' : 'ready';
+    }
+    if (pickResult.kind === 'back') {
+        return 'back';
+    }
+    const enabledTables = pickResult.enabledTables;
+    const activeSchemas = pickResult.activeSchemas;
     if (spec) {
         await writeScopeConfig({
             projectDir: input.projectDir,
@@ -1128,6 +1324,7 @@ async function maybeConfigureDatabaseScope(input) {
         projectDir: input.projectDir,
         connectionId: input.connectionId,
         connection: { ...currentConnection, enabled_tables: enabledTables },
+        io: input.io,
     });
     if (spec && activeSchemas.length > 0) {
         const capitalNounPlural = spec.nounPlural[0].toUpperCase() + spec.nounPlural.slice(1);
@@ -1136,7 +1333,7 @@ async function maybeConfigureDatabaseScope(input) {
         ]);
     }
     writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [
-        `✓ ${enabledTables.length}/${discovered.length} tables enabled`,
+        `✓ ${enabledTables.length} tables enabled`,
     ]);
     return 'ready';
 }
@@ -1159,22 +1356,20 @@ async function ensureHistoricSqlIngestDefaults(projectDir) {
 }
 async function markDatabasesComplete(projectDir, connectionIds) {
     const project = await loadKtxProject({ projectDir });
-    const config = setKtxSetupDatabaseConnectionIds({
-        ...project.config,
-        connections: Object.fromEntries(Object.entries(project.config.connections).map(([connectionId, connection]) => [
-            connectionId,
-            migrateLegacyHistoricSqlConnection(connection),
-        ])),
-    }, unique(connectionIds));
+    const config = setKtxSetupDatabaseConnectionIds(project.config, unique(connectionIds));
     await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
     await markKtxSetupStateStepComplete(projectDir, 'databases');
 }
 async function maybeRunHistoricSqlSetupProbe(input) {
     const project = await loadKtxProject({ projectDir: input.projectDir });
     const connection = project.config.connections[input.connectionId];
-    const queryHistory = queryHistoryConfigRecord(connection) ?? historicSqlConfigRecord(connection);
+    const queryHistory = queryHistoryConfigRecord(connection);
     const driver = normalizeDriver(connection?.driver);
-    if (queryHistory?.enabled !== true || driver !== 'postgres') {
+    if (queryHistory?.enabled !== true) {
+        return;
+    }
+    const dialect = driver === 'postgres' ? 'postgres' : driver === 'snowflake' ? 'snowflake' : null;
+    if (!dialect) {
         return;
     }
     input.io.stdout.write('│  Query history probe...\n');
@@ -1182,13 +1377,13 @@ async function maybeRunHistoricSqlSetupProbe(input) {
     const result = await probe({
         projectDir: input.projectDir,
         connectionId: input.connectionId,
-        dialect: 'postgres',
+        dialect,
     });
     for (const line of result.lines) {
         input.io.stdout.write(`│${line}\n`);
     }
     if (!result.ok) {
-        input.io.stdout.write('│  Setup written; first ingest run will fail until fixed.\n');
+        input.io.stdout.write('│  Setup written; query history will be skipped until fixed.\n');
     }
 }
 async function applyHistoricSqlConfigToExistingConnection(input) {
@@ -1464,14 +1659,10 @@ async function runPrimarySourceFullEdit(input) {
         connectionId: input.connectionId,
         connection: withExistingPrimaryEditPromptDefaults({
             previous: existing,
-            next: {
-                ...withHistoricSql,
-                ...(!Object.hasOwn(withHistoricSql, 'historicSql') && existing.historicSql !== undefined
-                    ? { historicSql: existing.historicSql }
-                    : {}),
-            },
+            next: withHistoricSql,
             driver,
         }),
+        io: input.io,
     });
     const validated = await validateAndScanConnection({
         projectDir: input.projectDir,
@@ -1703,6 +1894,7 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
                     projectDir: args.projectDir,
                     connectionId: connectionChoice.connectionId,
                     connection: withContextDepth,
+                    io,
                 });
             }
             else {
@@ -1731,6 +1923,7 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
                     projectDir: args.projectDir,
                     connectionId: connectionChoice.connectionId,
                     connection: withContextDepth,
+                    io,
                 });
             }
             let connectionSkipped = false;
@@ -1821,6 +2014,7 @@ export async function runKtxSetupDatabasesStep(args, io, deps = {}) {
                         projectDir: args.projectDir,
                         connectionId: connectionChoice.connectionId,
                         connection: withContextDepth,
+                        io,
                     });
                     setupStatus = await validateAndScanConnection({
                         projectDir: args.projectDir,