@kaelio/ktx 0.8.0 → 0.9.0

package/dist/context/ingest/adapters/historic-sql/scope-floor.js

@@ -0,0 +1,229 @@
+import { access, readdir, readFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import YAML from 'yaml';
+import { getDriverRegistration } from '../../../connections/drivers.js';
+import { parseDottedTableEntry } from '../../../scan/enabled-tables.js';
+import { tableRefKey, tableRefSet } from '../../../scan/table-ref.js';
+import { readLiveDatabaseTableFiles } from '../live-database/stage.js';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function stringArray(value) {
+    return Array.isArray(value)
+        ? value
+            .filter((item) => typeof item === 'string' && item.trim().length > 0)
+            .map((item) => item.trim())
+        : [];
+}
+function tableRefsFromValues(values) {
+    if (!Array.isArray(values))
+        return [];
+    return values.flatMap((value) => {
+        if (typeof value === 'string') {
+            const ref = parseDottedTableEntry(value);
+            return ref ? [ref] : [];
+        }
+        if (isRecord(value) && typeof value.name === 'string' && value.name.length > 0) {
+            return [
+                {
+                    catalog: typeof value.catalog === 'string' ? value.catalog : null,
+                    db: typeof value.db === 'string' ? value.db : null,
+                    name: value.name,
+                },
+            ];
+        }
+        return [];
+    });
+}
+function declaredSchemas(driver, connection) {
+    const key = getDriverRegistration(driver)?.scopeConfigKey;
+    if (!key)
+        return [];
+    return [...new Set(stringArray(connection[key]))].sort();
+}
+function uniqueSortedTableRefs(refs) {
+    const byKey = new Map();
+    for (const ref of refs) {
+        byKey.set(tableRefKey(ref), ref);
+    }
+    return [...byKey.entries()]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([, ref]) => ref);
+}
+async function latestLiveDatabaseScanDir(projectDir, connectionId) {
+    const root = join(projectDir, 'raw-sources', connectionId, 'live-database');
+    let entries;
+    try {
+        entries = await readdir(root, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT')
+            return null;
+        throw error;
+    }
+    const syncDirs = entries
+        .filter((entry) => entry.isDirectory())
+        .map((entry) => entry.name)
+        .sort()
+        .reverse();
+    for (const syncDir of syncDirs) {
+        const absolute = join(root, syncDir);
+        try {
+            await access(join(absolute, 'connection.json'));
+            return absolute;
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}
+async function scannedTableRefs(projectDir, connectionId) {
+    const scanDir = await latestLiveDatabaseScanDir(projectDir, connectionId);
+    if (!scanDir) {
+        return { refs: [], catalogAvailable: false, warnings: [] };
+    }
+    try {
+        const tableFiles = await readLiveDatabaseTableFiles(scanDir);
+        return {
+            refs: uniqueSortedTableRefs(tableFiles.map(({ table }) => ({ catalog: table.catalog, db: table.db, name: table.name }))),
+            catalogAvailable: true,
+            warnings: [],
+        };
+    }
+    catch (error) {
+        return {
+            refs: [],
+            catalogAvailable: false,
+            warnings: [
+                `query_history_scope_floor_catalog_read_failed:live_database_scan:${error instanceof Error ? error.message : String(error)}`,
+            ],
+        };
+    }
+}
+async function listYamlFiles(root) {
+    try {
+        const entries = await readdir(root, { withFileTypes: true, recursive: true });
+        return entries
+            .filter((entry) => entry.isFile() && /\.ya?ml$/i.test(entry.name))
+            .map((entry) => relative(root, join(entry.parentPath, entry.name)).replace(/\\/g, '/'))
+            .sort();
+    }
+    catch (error) {
+        if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT')
+            return [];
+        throw error;
+    }
+}
+function refsFromManifest(content) {
+    const parsed = YAML.parse(content);
+    if (!isRecord(parsed) || !isRecord(parsed.tables))
+        return [];
+    return Object.values(parsed.tables).flatMap((entry) => {
+        if (!isRecord(entry) || typeof entry.table !== 'string')
+            return [];
+        const ref = parseDottedTableEntry(entry.table);
+        return ref ? [ref] : [];
+    });
+}
+function refsFromStandaloneSource(content) {
+    const parsed = YAML.parse(content);
+    if (!isRecord(parsed) || typeof parsed.table !== 'string')
+        return [];
+    const ref = parseDottedTableEntry(parsed.table);
+    return ref ? [ref] : [];
+}
+async function semanticTableRefs(projectDir, connectionId) {
+    const root = join(projectDir, 'semantic-layer', connectionId);
+    const files = await listYamlFiles(root);
+    const refs = [];
+    const warnings = [];
+    for (const file of files) {
+        try {
+            const content = await readFile(join(root, file), 'utf-8');
+            refs.push(...(file.startsWith('_schema/') ? refsFromManifest(content) : refsFromStandaloneSource(content)));
+        }
+        catch (error) {
+            warnings.push(`query_history_scope_floor_catalog_read_failed:${file}:${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    return { refs: uniqueSortedTableRefs(refs), warnings };
+}
+export async function resolveQueryHistoryScopeFloor(input) {
+    const explicitEnabledTables = [
+        ...tableRefsFromValues(input.storedQueryHistory.enabledTables),
+        ...tableRefsFromValues(input.connection.enabled_tables),
+    ];
+    const semanticTables = await semanticTableRefs(input.projectDir, input.connectionId);
+    const scannedTables = await scannedTableRefs(input.projectDir, input.connectionId);
+    const modeledTables = uniqueSortedTableRefs([
+        ...semanticTables.refs,
+        ...scannedTables.refs,
+        ...explicitEnabledTables,
+    ]);
+    const warnings = [...semanticTables.warnings, ...scannedTables.warnings];
+    if (explicitEnabledTables.length > 0) {
+        return {
+            enabledTables: explicitEnabledTables,
+            enabledTableKeys: tableRefSet(explicitEnabledTables),
+            enabledSchemas: [],
+            modeledTableCatalog: modeledTables,
+            floorDisabled: false,
+            warnings,
+        };
+    }
+    const explicitSchemas = stringArray(input.storedQueryHistory.enabledSchemas);
+    if (explicitSchemas.includes('*')) {
+        return {
+            enabledTables: [],
+            enabledTableKeys: null,
+            enabledSchemas: ['*'],
+            modeledTableCatalog: modeledTables,
+            floorDisabled: true,
+            warnings,
+        };
+    }
+    if (explicitSchemas.length > 0) {
+        if (!scannedTables.catalogAvailable || modeledTables.length === 0) {
+            return {
+                enabledTables: [],
+                enabledTableKeys: null,
+                enabledSchemas: ['*'],
+                modeledTableCatalog: modeledTables,
+                floorDisabled: true,
+                warnings: [...warnings, 'query_history_scope_floor_disabled:catalog_unavailable'],
+            };
+        }
+        return {
+            enabledTables: [],
+            enabledTableKeys: null,
+            enabledSchemas: [...new Set(explicitSchemas)].sort(),
+            modeledTableCatalog: modeledTables,
+            floorDisabled: false,
+            warnings,
+        };
+    }
+    const schemas = new Set(declaredSchemas(input.driver, input.connection));
+    for (const ref of semanticTables.refs) {
+        if (ref.db)
+            schemas.add(ref.db);
+    }
+    if (schemas.size > 0 && (!scannedTables.catalogAvailable || modeledTables.length === 0)) {
+        return {
+            enabledTables: [],
+            enabledTableKeys: null,
+            enabledSchemas: ['*'],
+            modeledTableCatalog: modeledTables,
+            floorDisabled: true,
+            warnings: [...warnings, 'query_history_scope_floor_disabled:catalog_unavailable'],
+        };
+    }
+    return {
+        enabledTables: [],
+        enabledTableKeys: null,
+        enabledSchemas: [...schemas].sort(),
+        modeledTableCatalog: modeledTables,
+        floorDisabled: false,
+        warnings,
+    };
+}

package/dist/context/ingest/adapters/historic-sql/scope-membership.d.ts

@@ -0,0 +1,8 @@
+import type { KtxTableRef } from '../../../scan/types.js';
+export interface QueryHistoryScopeMembershipConfig {
+    enabledTables: readonly KtxTableRef[];
+    enabledSchemas: readonly string[];
+}
+export declare function isQueryHistoryScopeFloorDisabled(config: QueryHistoryScopeMembershipConfig): boolean;
+export declare function shouldFailOpenQueryHistoryScope(config: QueryHistoryScopeMembershipConfig): boolean;
+export declare function includedQueryHistoryTableRefs(tablesTouched: readonly KtxTableRef[], config: QueryHistoryScopeMembershipConfig): KtxTableRef[];

package/dist/context/ingest/adapters/historic-sql/scope-membership.js

@@ -0,0 +1,29 @@
+import { tableRefKey, tableRefSet } from '../../../scan/table-ref.js';
+function schemaNameForRef(ref) {
+    return ref.db && ref.db.length > 0 ? ref.db : null;
+}
+function schemaNamesFromConfig(enabledSchemas) {
+    return new Set(enabledSchemas.filter((schema) => schema !== '*'));
+}
+export function isQueryHistoryScopeFloorDisabled(config) {
+    return config.enabledSchemas.includes('*');
+}
+export function shouldFailOpenQueryHistoryScope(config) {
+    return (config.enabledTables.length === 0 &&
+        !isQueryHistoryScopeFloorDisabled(config) &&
+        config.enabledSchemas.length === 0);
+}
+export function includedQueryHistoryTableRefs(tablesTouched, config) {
+    if (config.enabledTables.length > 0) {
+        const enabled = tableRefSet(config.enabledTables);
+        return tablesTouched.filter((ref) => enabled.has(tableRefKey(ref)));
+    }
+    if (isQueryHistoryScopeFloorDisabled(config) || shouldFailOpenQueryHistoryScope(config)) {
+        return [...tablesTouched];
+    }
+    const schemas = schemaNamesFromConfig(config.enabledSchemas);
+    return tablesTouched.filter((ref) => {
+        const schema = schemaNameForRef(ref);
+        return schema !== null && schemas.has(schema);
+    });
+}

package/dist/context/ingest/adapters/historic-sql/snowflake-query-history-reader.js

@@ -148,26 +148,75 @@ export class SnowflakeHistoricSqlQueryHistoryReader {
     }
     async *fetchAggregated(client, window, config) {
         const sql = `
+WITH filtered_queries AS (
+  SELECT
+    query_hash,
+    query_text,
+    user_name,
+    start_time,
+    total_elapsed_time,
+    execution_status,
+    rows_produced
+  FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
+  WHERE query_text IS NOT NULL
+    AND query_type IN ('SELECT', 'MERGE')
+    AND start_time >= ${timestampLiteral(window.start)}
+    AND start_time < ${timestampLiteral(window.end)}
+),
+template_stats AS (
+  SELECT
+    query_hash AS template_id,
+    MIN(query_text) AS canonical_sql,
+    COUNT(*) AS executions,
+    COUNT(DISTINCT user_name) AS distinct_users,
+    MIN(start_time) AS first_seen,
+    MAX(start_time) AS last_seen,
+    APPROX_PERCENTILE(total_elapsed_time, 0.50) AS p50_ms,
+    APPROX_PERCENTILE(total_elapsed_time, 0.95) AS p95_ms,
+    DIV0(COUNT_IF(execution_status != 'SUCCESS'), COUNT(*)) AS error_rate,
+    SUM(rows_produced) AS rows_produced
+  FROM filtered_queries
+  GROUP BY query_hash
+  HAVING COUNT(*) >= ${config.minExecutions}
+),
+template_users AS (
+  SELECT
+    query_hash AS template_id,
+    user_name AS user,
+    COUNT(*) AS executions,
+    MAX(start_time) AS last_seen
+  FROM filtered_queries
+  GROUP BY query_hash, user_name
+)
 SELECT
-  query_hash AS template_id,
-  MIN(query_text) AS canonical_sql,
-  COUNT(*) AS executions,
-  COUNT(DISTINCT user_name) AS distinct_users,
-  MIN(start_time) AS first_seen,
-  MAX(start_time) AS last_seen,
-  APPROX_PERCENTILE(total_elapsed_time, 0.50) AS p50_ms,
-  APPROX_PERCENTILE(total_elapsed_time, 0.95) AS p95_ms,
-  DIV0(COUNT_IF(execution_status != 'SUCCESS'), COUNT(*)) AS error_rate,
-  SUM(rows_produced) AS rows_produced,
-  ARRAY_AGG(OBJECT_CONSTRUCT('user', user_name, 'executions', 1)) WITHIN GROUP (ORDER BY start_time DESC)::string AS top_users
-FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
-WHERE query_text IS NOT NULL
-  AND query_type IN ('SELECT', 'MERGE')
-  AND start_time >= ${timestampLiteral(window.start)}
-  AND start_time < ${timestampLiteral(window.end)}
-GROUP BY query_hash
-HAVING COUNT(*) >= ${config.minExecutions}
-ORDER BY executions DESC`.trim();
+  stats.template_id,
+  stats.canonical_sql,
+  stats.executions,
+  stats.distinct_users,
+  stats.first_seen,
+  stats.last_seen,
+  stats.p50_ms,
+  stats.p95_ms,
+  stats.error_rate,
+  stats.rows_produced,
+  ARRAY_AGG(
+    OBJECT_CONSTRUCT('user', users.user, 'executions', users.executions)
+  ) WITHIN GROUP (ORDER BY users.executions DESC, users.last_seen DESC)::string AS top_users
+FROM template_stats AS stats
+JOIN template_users AS users
+  ON users.template_id = stats.template_id
+GROUP BY
+  stats.template_id,
+  stats.canonical_sql,
+  stats.executions,
+  stats.distinct_users,
+  stats.first_seen,
+  stats.last_seen,
+  stats.p50_ms,
+  stats.p95_ms,
+  stats.error_rate,
+  stats.rows_produced
+ORDER BY stats.executions DESC`.trim();
         const result = await queryClient(client).executeQuery(sql);
         if (result.error) {
             throw grantsError(result.error);

package/dist/context/ingest/adapters/historic-sql/stage-unified.js

@@ -1,8 +1,10 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
+import { tableRefKey } from '../../../scan/table-ref.js';
 import { bucketDistinctUsers, bucketErrorRate, bucketExecutions, bucketFrequency, bucketP95Runtime, bucketRecency, } from './buckets.js';
 import { splitHistoricSqlPatternInputs } from './pattern-inputs.js';
 import { compileHistoricSqlRedactionPatterns, redactHistoricSqlText, } from './redaction.js';
+import { includedQueryHistoryTableRefs, isQueryHistoryScopeFloorDisabled, shouldFailOpenQueryHistoryScope, } from './scope-membership.js';
 import { HISTORIC_SQL_SOURCE_KEY, aggregatedTemplateSchema, historicSqlUnifiedPullConfigSchema, } from './types.js';
 const TRIVIAL_SQL_RE = /^\s*SELECT\s+(1|NOW\(\)|CURRENT_TIMESTAMP|VERSION\(\))\s*;?\s*$/i;
 const NOISE_PREFIX_RE = /^\s*(SHOW|DESCRIBE|DESC|EXPLAIN|USE|SET)\b/i;
@@ -17,9 +19,22 @@ function compilePatterns(patterns) {
 function matchesAny(value, patterns) {
     return !!value && patterns.some((pattern) => pattern.test(value));
 }
+// ktx's own warehouse scan emits relationship- and column-profiling probes that land in
+// pg_stat_statements (relationship-validation, relationship-composite-candidates, and each
+// dialect's relationship value aggregation). They are ktx introspection, not genuine query
+// usage, so they must not be mined back as query history. The markers are ktx-owned
+// identifiers, stable across dialects.
+function isKtxScanProbe(sql) {
+    if (/\brelationship_profile_values\b/i.test(sql)) {
+        return true;
+    }
+    return /\bchild_values\b/i.test(sql) && /\bparent_values\b/i.test(sql);
+}
 function shouldDropBySql(sql, config) {
     if (NOISE_PREFIX_RE.test(sql) || SYSTEM_TABLE_RE.test(sql))
         return true;
+    if (isKtxScanProbe(sql))
+        return true;
     if (config.filters.dropTrivialProbes !== false && TRIVIAL_SQL_RE.test(sql))
         return true;
     return false;
@@ -32,8 +47,7 @@ function shouldDropByUsers(template, config) {
     const matchingExecutions = template.topUsers
         .filter((entry) => matchesAny(entry.user, patterns))
         .reduce((sum, entry) => sum + entry.executions, 0);
-    const allExecutions = template.topUsers.reduce((sum, entry) => sum + entry.executions, 0);
-    const serviceOnly = allExecutions > 0 && matchingExecutions >= allExecutions;
+    const serviceOnly = template.stats.executions > 0 && matchingExecutions >= template.stats.executions;
     return service.mode === 'exclude' ? serviceOnly : !serviceOnly;
 }
 function shouldDropByFailure(template, config) {
@@ -49,38 +63,8 @@ function shouldDropTemplate(template, config) {
         return true;
     return false;
 }
-function normalizeTableIdentifier(value) {
-    return value.trim().toLowerCase();
-}
-function unqualifiedTableIdentifier(value) {
-    const parts = normalizeTableIdentifier(value).split('.').filter(Boolean);
-    return parts.at(-1) ?? '';
-}
-function buildEnabledTableFilter(enabledTables) {
-    if (enabledTables.length === 0) {
-        return null;
-    }
-    const exact = new Set(enabledTables.map(normalizeTableIdentifier).filter((value) => value.length > 0));
-    const unqualifiedCounts = new Map();
-    for (const table of exact) {
-        const unqualified = unqualifiedTableIdentifier(table);
-        if (unqualified.length > 0) {
-            unqualifiedCounts.set(unqualified, (unqualifiedCounts.get(unqualified) ?? 0) + 1);
-        }
-    }
-    return {
-        exact,
-        uniqueUnqualified: new Set([...unqualifiedCounts.entries()]
-            .filter(([, count]) => count === 1)
-            .map(([table]) => table)),
-    };
-}
-function isEnabledTable(table, filter) {
-    if (!filter) {
-        return true;
-    }
-    const normalized = normalizeTableIdentifier(table);
-    return filter.exact.has(normalized) || filter.uniqueUnqualified.has(unqualifiedTableIdentifier(normalized));
+function displayTableRef(ref) {
+    return [ref.catalog, ref.db, ref.name].filter((part) => !!part && part.length > 0).join('.');
 }
 function historicSqlWindowDays(config) {
     return 'windowDays' in config ? config.windowDays : 90;
@@ -107,9 +91,10 @@ function recordJoin(acc, otherTable, columns, executions) {
         acc.observedJoins.set(otherTable, byColumns);
     }
 }
-function accumulatorFor(table) {
+function accumulatorFor(tableRef) {
     return {
-        table,
+        tableRef,
+        table: displayTableRef(tableRef),
         executions: 0,
         distinctUsers: 0,
         errorRateNumerator: 0,
@@ -138,8 +123,8 @@ function addTemplate(acc, parsed) {
         }
     }
     const joinColumns = parsed.columnsByClause.join ?? [];
-    for (const otherTable of parsed.tablesTouched.filter((table) => table !== acc.table)) {
-        recordJoin(acc, otherTable, joinColumns, executions);
+    for (const otherTable of parsed.tablesTouched.filter((table) => tableRefKey(table) !== tableRefKey(acc.tableRef))) {
+        recordJoin(acc, displayTableRef(otherTable), joinColumns, executions);
     }
     acc.topTemplates.push(parsed.template);
 }
@@ -170,6 +155,7 @@ function toStagedTable(acc, now) {
     }));
     return {
         table: acc.table,
+        tableRef: acc.tableRef,
         stats: {
             executionsBucket: bucketExecutions(acc.executions),
             distinctUsersBucket: bucketDistinctUsers(acc.distinctUsers),
@@ -188,7 +174,7 @@ function toPatternsInput(parsedTemplates) {
             .map(({ template, tablesTouched }) => ({
             id: template.templateId,
             canonicalSql: template.canonicalSql,
-            tablesTouched: [...tablesTouched].sort(),
+            tablesTouched: [...tablesTouched].sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right))),
             executionsBucket: bucketExecutions(template.stats.executions),
             distinctUsersBucket: bucketDistinctUsers(template.stats.distinctUsers),
             dialect: template.dialect,
@@ -198,7 +184,6 @@ function toPatternsInput(parsedTemplates) {
 }
 export async function stageHistoricSqlAggregatedSnapshot(input) {
     const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig);
-    const enabledTableFilter = buildEnabledTableFilter(config.enabledTables);
     const redactors = compileHistoricSqlRedactionPatterns(config.redactionPatterns);
     const now = input.now ?? new Date();
     const windowStart = new Date(now.getTime() - historicSqlWindowDays(config) * 24 * 60 * 60 * 1000);
@@ -212,8 +197,25 @@ export async function stageHistoricSqlAggregatedSnapshot(input) {
             snapshot.push(parsed);
         }
     }
-    const analysis = await input.sqlAnalysis.analyzeBatch(snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql })), config.dialect);
-    const warnings = [];
+    const analysisItems = snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql }));
+    const analysisOptions = config.modeledTableCatalog.length > 0 ? { catalog: { tables: config.modeledTableCatalog } } : undefined;
+    const warnings = [
+        ...config.scopeFloorWarnings,
+        ...(shouldFailOpenQueryHistoryScope(config) ? ['query_history_scope_floor_disabled:empty_modeled_scope'] : []),
+    ];
+    let scopeDisabledByQualificationFailure = false;
+    let analysis;
+    try {
+        analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, config.dialect, analysisOptions);
+    }
+    catch (error) {
+        if (!analysisOptions || config.enabledTables.length > 0 || isQueryHistoryScopeFloorDisabled(config)) {
+            throw error;
+        }
+        warnings.push('query_history_scope_floor_disabled:catalog_qualification_failed');
+        scopeDisabledByQualificationFailure = true;
+        analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, config.dialect, undefined);
+    }
     const parsedTemplates = [];
     for (const template of snapshot) {
         const parsed = analysis.get(template.templateId);
@@ -221,8 +223,12 @@ export async function stageHistoricSqlAggregatedSnapshot(input) {
             warnings.push(`parse_failed:${template.templateId}`);
             continue;
         }
-        const tablesTouched = [...new Set(parsed.tablesTouched)].filter((table) => table.length > 0).sort();
-        const includedTables = tablesTouched.filter((table) => isEnabledTable(table, enabledTableFilter));
+        const tablesTouched = [...new Map(parsed.tablesTouched.map((ref) => [tableRefKey(ref), ref])).values()]
+            .filter((ref) => ref.name.length > 0)
+            .sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right)));
+        const includedTables = scopeDisabledByQualificationFailure
+            ? [...tablesTouched]
+            : includedQueryHistoryTableRefs(tablesTouched, config);
         if (includedTables.length === 0) {
             continue;
         }
@@ -235,19 +241,20 @@ export async function stageHistoricSqlAggregatedSnapshot(input) {
     }
     const byTable = new Map();
     for (const parsed of parsedTemplates) {
-        for (const table of parsed.includedTables) {
-            const acc = byTable.get(table) ?? accumulatorFor(table);
+        for (const tableRef of parsed.includedTables) {
+            const key = tableRefKey(tableRef);
+            const acc = byTable.get(key) ?? accumulatorFor(tableRef);
             addTemplate(acc, parsed);
-            byTable.set(table, acc);
+            byTable.set(key, acc);
         }
     }
     await mkdir(input.stagedDir, { recursive: true });
-    for (const [table, acc] of [...byTable.entries()].sort(([left], [right]) => left.localeCompare(right))) {
-        await writeJson(input.stagedDir, `tables/${table}.json`, toStagedTable(acc, now));
+    for (const [, acc] of [...byTable.entries()].sort((left, right) => left[0].localeCompare(right[0]))) {
+        await writeJson(input.stagedDir, `tables/${acc.table}.json`, toStagedTable(acc, now));
     }
     const patternsInput = toPatternsInput(parsedTemplates);
     const patternInputSplit = splitHistoricSqlPatternInputs(patternsInput);
-    const allWarnings = [...warnings, ...patternInputSplit.warnings];
+    const allWarnings = [...new Set([...warnings, ...patternInputSplit.warnings])];
     await writeJson(input.stagedDir, 'patterns-input.json', patternInputSplit.auditInput);
     for (const shard of patternInputSplit.shards) {
         await writeJson(input.stagedDir, shard.path, shard.input);

package/dist/context/ingest/adapters/historic-sql/types.d.ts

@@ -9,7 +9,19 @@ declare const historicSqlDialectSchema: z.ZodEnum<{
 export type HistoricSqlDialect = z.infer<typeof historicSqlDialectSchema>;
 export declare const historicSqlUnifiedPullConfigSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     minExecutions: z.ZodDefault<z.ZodNumber>;
-    enabledTables: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    enabledTables: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        catalog: z.ZodNullable<z.ZodString>;
+        db: z.ZodNullable<z.ZodString>;
+        name: z.ZodString;
+    }, z.core.$strict>>>;
+    enabledSchemas: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    modeledTableCatalog: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        catalog: z.ZodNullable<z.ZodString>;
+        db: z.ZodNullable<z.ZodString>;
+        name: z.ZodString;
+        columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strict>>>;
+    scopeFloorWarnings: z.ZodDefault<z.ZodArray<z.ZodString>>;
     filters: z.ZodDefault<z.ZodObject<{
         serviceAccounts: z.ZodOptional<z.ZodObject<{
             patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
@@ -41,7 +53,19 @@ export declare const historicSqlUnifiedPullConfigSchema: z.ZodDiscriminatedUnion
     windowDays: z.ZodDefault<z.ZodNumber>;
 }, z.core.$strip>, z.ZodObject<{
     minExecutions: z.ZodDefault<z.ZodNumber>;
-    enabledTables: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    enabledTables: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        catalog: z.ZodNullable<z.ZodString>;
+        db: z.ZodNullable<z.ZodString>;
+        name: z.ZodString;
+    }, z.core.$strict>>>;
+    enabledSchemas: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    modeledTableCatalog: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        catalog: z.ZodNullable<z.ZodString>;
+        db: z.ZodNullable<z.ZodString>;
+        name: z.ZodString;
+        columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strict>>>;
+    scopeFloorWarnings: z.ZodDefault<z.ZodArray<z.ZodString>>;
     filters: z.ZodDefault<z.ZodObject<{
         serviceAccounts: z.ZodOptional<z.ZodObject<{
             patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
@@ -95,6 +119,11 @@ export declare const aggregatedTemplateSchema: z.ZodObject<{
 export type AggregatedTemplate = z.infer<typeof aggregatedTemplateSchema>;
 export declare const stagedTableInputSchema: z.ZodObject<{
     table: z.ZodString;
+    tableRef: z.ZodObject<{
+        catalog: z.ZodNullable<z.ZodString>;
+        db: z.ZodNullable<z.ZodString>;
+        name: z.ZodString;
+    }, z.core.$strict>;
     stats: z.ZodObject<{
         executionsBucket: z.ZodString;
         distinctUsersBucket: z.ZodString;
@@ -121,7 +150,11 @@ export declare const stagedPatternsInputSchema: z.ZodObject<{
     templates: z.ZodArray<z.ZodObject<{
         id: z.ZodString;
         canonicalSql: z.ZodString;
-        tablesTouched: z.ZodArray<z.ZodString>;
+        tablesTouched: z.ZodArray<z.ZodObject<{
+            catalog: z.ZodNullable<z.ZodString>;
+            db: z.ZodNullable<z.ZodString>;
+            name: z.ZodString;
+        }, z.core.$strict>>;
         executionsBucket: z.ZodString;
         distinctUsersBucket: z.ZodString;
         dialect: z.ZodEnum<{

package/dist/context/ingest/adapters/historic-sql/types.js

@@ -2,9 +2,20 @@ import { z } from 'zod';
 export const HISTORIC_SQL_SOURCE_KEY = 'historic-sql';
 const historicSqlDialectSchema = z.enum(['snowflake', 'bigquery', 'postgres']);
 const filterModeSchema = z.enum(['exclude', 'include', 'mark-only']);
+const ktxTableRefSchema = z.object({
+    catalog: z.string().nullable(),
+    db: z.string().nullable(),
+    name: z.string().min(1),
+}).strict();
+const ktxTableRefWithColumnsSchema = ktxTableRefSchema.extend({
+    columns: z.array(z.string().min(1)).optional(),
+}).strict();
 const historicSqlCommonPullConfigSchema = z.object({
     minExecutions: z.number().int().nonnegative().default(5),
-    enabledTables: z.array(z.string().min(1)).default([]),
+    enabledTables: z.array(ktxTableRefSchema).default([]),
+    enabledSchemas: z.array(z.string().min(1)).default([]),
+    modeledTableCatalog: z.array(ktxTableRefWithColumnsSchema).default([]),
+    scopeFloorWarnings: z.array(z.string()).default([]),
     filters: z.object({
         serviceAccounts: z.object({
             patterns: z.array(z.string()).default([]),
@@ -54,6 +65,7 @@ export const aggregatedTemplateSchema = z.object({
 });
 export const stagedTableInputSchema = z.object({
     table: z.string().min(1),
+    tableRef: ktxTableRefSchema,
     stats: z.object({
         executionsBucket: z.string(),
         distinctUsersBucket: z.string(),
@@ -77,7 +89,7 @@ export const stagedPatternsInputSchema = z.object({
     templates: z.array(z.object({
         id: z.string(),
         canonicalSql: z.string(),
-        tablesTouched: z.array(z.string()),
+        tablesTouched: z.array(ktxTableRefSchema),
         executionsBucket: z.string(),
         distinctUsersBucket: z.string(),
         dialect: historicSqlDialectSchema,