@kaelio/ktx 0.8.0 → 0.10.0

package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.js

@@ -0,0 +1,194 @@
+import { z } from 'zod';
+import { tableRefKey } from '../../../scan/table-ref.js';
+import { bucketDistinctUsers, bucketExecutions, bucketRecency } from './buckets.js';
+import { compileHistoricSqlRedactionPatterns, redactHistoricSqlText, } from './redaction.js';
+import { includedQueryHistoryTableRefs } from './scope-membership.js';
+import { aggregatedTemplateSchema, historicSqlUnifiedPullConfigSchema, } from './types.js';
+const queryHistoryFilterAdjudicationSchema = z.object({
+    roles: z.array(z.object({
+        role: z.string().min(1),
+        exclude: z.boolean(),
+        reason: z.string().min(1),
+    }).strict()),
+}).strict();
+function emptyProposal(skipped, warnings = []) {
+    return { excludedRoles: [], consideredRoleCount: 0, skipped, warnings, parseFailedTemplateIds: [] };
+}
+function displayTableRef(ref) {
+    return [ref.catalog, ref.db, ref.name].filter((part) => !!part && part.length > 0).join('.');
+}
+function redactTemplateSqlForPicker(template, redactors) {
+    if (redactors.length === 0) {
+        return template;
+    }
+    return {
+        ...template,
+        canonicalSql: redactHistoricSqlText(template.canonicalSql, redactors),
+    };
+}
+/** @internal */
+export function regexEscapeForExactRolePattern(role) {
+    return `^${role.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&')}$`;
+}
+function recordRole(acc, template, tables, executions) {
+    acc.executions += executions;
+    acc.distinctUsers = Math.max(acc.distinctUsers, template.stats.distinctUsers);
+    acc.lastSeen = template.stats.lastSeen > acc.lastSeen ? template.stats.lastSeen : acc.lastSeen;
+    for (const table of tables) {
+        acc.tables.set(tableRefKey(table), table);
+    }
+    acc.templates.push(template);
+}
+function roleRecords(parsedTemplates, now) {
+    const byRole = new Map();
+    for (const parsed of parsedTemplates) {
+        for (const entry of parsed.template.topUsers) {
+            if (!entry.user || entry.user.trim().length === 0 || entry.executions <= 0) {
+                continue;
+            }
+            const role = entry.user.trim();
+            const acc = byRole.get(role) ??
+                {
+                    role,
+                    executions: 0,
+                    distinctUsers: 0,
+                    lastSeen: '1970-01-01T00:00:00.000Z',
+                    tables: new Map(),
+                    templates: [],
+                };
+            recordRole(acc, parsed.template, parsed.includedTables, entry.executions);
+            byRole.set(role, acc);
+        }
+    }
+    return [...byRole.values()]
+        .sort((left, right) => right.executions - left.executions || left.role.localeCompare(right.role))
+        .map((acc) => ({
+        role: acc.role,
+        inScopeTables: [...acc.tables.entries()]
+            .sort(([left], [right]) => left.localeCompare(right))
+            .slice(0, 25)
+            .map(([, ref]) => displayTableRef(ref)),
+        executionsBucket: bucketExecutions(acc.executions),
+        distinctUsersBucket: bucketDistinctUsers(acc.distinctUsers),
+        recencyBucket: bucketRecency(acc.lastSeen, now),
+        representativeTemplates: [...acc.templates]
+            .sort((left, right) => right.stats.executions - left.stats.executions || left.templateId.localeCompare(right.templateId))
+            .slice(0, 3)
+            .map((template) => ({
+            id: template.templateId,
+            canonicalSql: template.canonicalSql,
+            dialect: template.dialect,
+        })),
+    }));
+}
+function adjudicationSystemPrompt() {
+    return [
+        'You are helping ktx decide whether observed query-history roles are operational service accounts.',
+        'Default every role to keep. Mark exclude true only when the aggregate evidence clearly shows loader, ELT, reverse-ETL, export, refresh, or maintenance traffic rather than analyst or BI-dashboard usage.',
+        'Use only the observed role records. Do not rely on a hardcoded denylist. Return structured output only.',
+    ].join('\n');
+}
+export async function proposeQueryHistoryServiceAccountFilters(input) {
+    if (!input.llmRuntime) {
+        return emptyProposal({ reason: 'no-llm' });
+    }
+    const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig);
+    const redactors = compileHistoricSqlRedactionPatterns(config.redactionPatterns);
+    const now = input.now ?? new Date();
+    const windowDays = 'windowDays' in config ? config.windowDays : 90;
+    const windowStart = new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000);
+    const warnings = [];
+    const parseFailedTemplateIds = [];
+    const snapshot = [];
+    try {
+        for await (const row of input.reader.fetchAggregated(input.queryClient, { start: windowStart, end: now }, config)) {
+            snapshot.push(aggregatedTemplateSchema.parse(row));
+        }
+    }
+    catch (error) {
+        return emptyProposal(null, [
+            `query_history_filter_picker_read_failed:${error instanceof Error ? error.message : String(error)}`,
+        ]);
+    }
+    if (snapshot.length === 0) {
+        return emptyProposal({ reason: 'no-in-scope-history' });
+    }
+    const analysisItems = snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql }));
+    const analysisOptions = config.modeledTableCatalog.length > 0 ? { catalog: { tables: config.modeledTableCatalog } } : undefined;
+    let analysis;
+    try {
+        analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, input.dialect, analysisOptions);
+    }
+    catch (error) {
+        return emptyProposal({ reason: 'no-daemon' }, [
+            `query_history_filter_picker_analysis_failed:${error instanceof Error ? error.message : String(error)}`,
+        ]);
+    }
+    const parsedTemplates = [];
+    for (const template of snapshot) {
+        const parsed = analysis.get(template.templateId);
+        if (!parsed || parsed.error) {
+            parseFailedTemplateIds.push(template.templateId);
+            continue;
+        }
+        const tablesTouched = [...new Map(parsed.tablesTouched.map((ref) => [tableRefKey(ref), ref])).values()]
+            .filter((ref) => ref.name.length > 0)
+            .sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right)));
+        const includedTables = includedQueryHistoryTableRefs(tablesTouched, config);
+        if (includedTables.length === 0) {
+            continue;
+        }
+        parsedTemplates.push({
+            template: redactTemplateSqlForPicker(template, redactors),
+            tablesTouched,
+            includedTables,
+        });
+    }
+    const records = roleRecords(parsedTemplates, now);
+    if (records.length <= 1) {
+        return {
+            excludedRoles: [],
+            consideredRoleCount: records.length,
+            skipped: { reason: 'no-in-scope-history' },
+            warnings,
+            parseFailedTemplateIds,
+        };
+    }
+    let generated;
+    try {
+        generated = await input.llmRuntime.generateObject({
+            role: 'candidateExtraction',
+            system: adjudicationSystemPrompt(),
+            prompt: JSON.stringify({ connectionId: input.connectionId, dialect: input.dialect, roles: records }),
+            schema: queryHistoryFilterAdjudicationSchema,
+        });
+    }
+    catch (error) {
+        return {
+            excludedRoles: [],
+            consideredRoleCount: records.length,
+            skipped: { reason: 'no-llm' },
+            warnings: [
+                ...warnings,
+                `query_history_filter_picker_llm_failed:${error instanceof Error ? error.message : String(error)}`,
+            ],
+            parseFailedTemplateIds,
+        };
+    }
+    const knownRoles = new Set(records.map((record) => record.role));
+    const excludedRoles = generated.roles
+        .filter((role) => role.exclude && knownRoles.has(role.role))
+        .sort((left, right) => left.role.localeCompare(right.role))
+        .map((role) => ({
+        role: role.role,
+        reason: role.reason,
+        pattern: regexEscapeForExactRolePattern(role.role),
+    }));
+    return {
+        excludedRoles,
+        consideredRoleCount: records.length,
+        skipped: input.userServiceAccountsPresent ? { reason: 'user-block-present' } : null,
+        warnings,
+        parseFailedTemplateIds,
+    };
+}

package/dist/context/ingest/adapters/historic-sql/scope-floor.d.ts

@@ -0,0 +1,18 @@
+import { type KtxTableRefKey } from '../../../scan/table-ref.js';
+import type { KtxTableRef } from '../../../scan/types.js';
+export interface QueryHistoryScopeFloorInput {
+    projectDir: string;
+    connectionId: string;
+    driver: string;
+    connection: Record<string, unknown>;
+    storedQueryHistory: Record<string, unknown>;
+}
+export interface QueryHistoryScopeFloor {
+    enabledTables: KtxTableRef[];
+    enabledTableKeys: ReadonlySet<KtxTableRefKey> | null;
+    enabledSchemas: string[];
+    modeledTableCatalog: KtxTableRef[];
+    floorDisabled: boolean;
+    warnings: string[];
+}
+export declare function resolveQueryHistoryScopeFloor(input: QueryHistoryScopeFloorInput): Promise<QueryHistoryScopeFloor>;

package/dist/context/ingest/adapters/historic-sql/scope-floor.js

@@ -0,0 +1,229 @@
+import { access, readdir, readFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import YAML from 'yaml';
+import { getDriverRegistration } from '../../../connections/drivers.js';
+import { parseDottedTableEntry } from '../../../scan/enabled-tables.js';
+import { tableRefKey, tableRefSet } from '../../../scan/table-ref.js';
+import { readLiveDatabaseTableFiles } from '../live-database/stage.js';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function stringArray(value) {
+    return Array.isArray(value)
+        ? value
+            .filter((item) => typeof item === 'string' && item.trim().length > 0)
+            .map((item) => item.trim())
+        : [];
+}
+function tableRefsFromValues(values) {
+    if (!Array.isArray(values))
+        return [];
+    return values.flatMap((value) => {
+        if (typeof value === 'string') {
+            const ref = parseDottedTableEntry(value);
+            return ref ? [ref] : [];
+        }
+        if (isRecord(value) && typeof value.name === 'string' && value.name.length > 0) {
+            return [
+                {
+                    catalog: typeof value.catalog === 'string' ? value.catalog : null,
+                    db: typeof value.db === 'string' ? value.db : null,
+                    name: value.name,
+                },
+            ];
+        }
+        return [];
+    });
+}
+function declaredSchemas(driver, connection) {
+    const key = getDriverRegistration(driver)?.scopeConfigKey;
+    if (!key)
+        return [];
+    return [...new Set(stringArray(connection[key]))].sort();
+}
+function uniqueSortedTableRefs(refs) {
+    const byKey = new Map();
+    for (const ref of refs) {
+        byKey.set(tableRefKey(ref), ref);
+    }
+    return [...byKey.entries()]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([, ref]) => ref);
+}
+async function latestLiveDatabaseScanDir(projectDir, connectionId) {
+    const root = join(projectDir, 'raw-sources', connectionId, 'live-database');
+    let entries;
+    try {
+        entries = await readdir(root, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT')
+            return null;
+        throw error;
+    }
+    const syncDirs = entries
+        .filter((entry) => entry.isDirectory())
+        .map((entry) => entry.name)
+        .sort()
+        .reverse();
+    for (const syncDir of syncDirs) {
+        const absolute = join(root, syncDir);
+        try {
+            await access(join(absolute, 'connection.json'));
+            return absolute;
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}
+async function scannedTableRefs(projectDir, connectionId) {
+    const scanDir = await latestLiveDatabaseScanDir(projectDir, connectionId);
+    if (!scanDir) {
+        return { refs: [], catalogAvailable: false, warnings: [] };
+    }
+    try {
+        const tableFiles = await readLiveDatabaseTableFiles(scanDir);
+        return {
+            refs: uniqueSortedTableRefs(tableFiles.map(({ table }) => ({ catalog: table.catalog, db: table.db, name: table.name }))),
+            catalogAvailable: true,
+            warnings: [],
+        };
+    }
+    catch (error) {
+        return {
+            refs: [],
+            catalogAvailable: false,
+            warnings: [
+                `query_history_scope_floor_catalog_read_failed:live_database_scan:${error instanceof Error ? error.message : String(error)}`,
+            ],
+        };
+    }
+}
+async function listYamlFiles(root) {
+    try {
+        const entries = await readdir(root, { withFileTypes: true, recursive: true });
+        return entries
+            .filter((entry) => entry.isFile() && /\.ya?ml$/i.test(entry.name))
+            .map((entry) => relative(root, join(entry.parentPath, entry.name)).replace(/\\/g, '/'))
+            .sort();
+    }
+    catch (error) {
+        if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT')
+            return [];
+        throw error;
+    }
+}
+function refsFromManifest(content) {
+    const parsed = YAML.parse(content);
+    if (!isRecord(parsed) || !isRecord(parsed.tables))
+        return [];
+    return Object.values(parsed.tables).flatMap((entry) => {
+        if (!isRecord(entry) || typeof entry.table !== 'string')
+            return [];
+        const ref = parseDottedTableEntry(entry.table);
+        return ref ? [ref] : [];
+    });
+}
+function refsFromStandaloneSource(content) {
+    const parsed = YAML.parse(content);
+    if (!isRecord(parsed) || typeof parsed.table !== 'string')
+        return [];
+    const ref = parseDottedTableEntry(parsed.table);
+    return ref ? [ref] : [];
+}
+async function semanticTableRefs(projectDir, connectionId) {
+    const root = join(projectDir, 'semantic-layer', connectionId);
+    const files = await listYamlFiles(root);
+    const refs = [];
+    const warnings = [];
+    for (const file of files) {
+        try {
+            const content = await readFile(join(root, file), 'utf-8');
+            refs.push(...(file.startsWith('_schema/') ? refsFromManifest(content) : refsFromStandaloneSource(content)));
+        }
+        catch (error) {
+            warnings.push(`query_history_scope_floor_catalog_read_failed:${file}:${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    return { refs: uniqueSortedTableRefs(refs), warnings };
+}
+export async function resolveQueryHistoryScopeFloor(input) {
+    const explicitEnabledTables = [
+        ...tableRefsFromValues(input.storedQueryHistory.enabledTables),
+        ...tableRefsFromValues(input.connection.enabled_tables),
+    ];
+    const semanticTables = await semanticTableRefs(input.projectDir, input.connectionId);
+    const scannedTables = await scannedTableRefs(input.projectDir, input.connectionId);
+    const modeledTables = uniqueSortedTableRefs([
+        ...semanticTables.refs,
+        ...scannedTables.refs,
+        ...explicitEnabledTables,
+    ]);
+    const warnings = [...semanticTables.warnings, ...scannedTables.warnings];
+    if (explicitEnabledTables.length > 0) {
+        return {
+            enabledTables: explicitEnabledTables,
+            enabledTableKeys: tableRefSet(explicitEnabledTables),
+            enabledSchemas: [],
+            modeledTableCatalog: modeledTables,
+            floorDisabled: false,
+            warnings,
+        };
+    }
+    const explicitSchemas = stringArray(input.storedQueryHistory.enabledSchemas);
+    if (explicitSchemas.includes('*')) {
+        return {
+            enabledTables: [],
+            enabledTableKeys: null,
+            enabledSchemas: ['*'],
+            modeledTableCatalog: modeledTables,
+            floorDisabled: true,
+            warnings,
+        };
+    }
+    if (explicitSchemas.length > 0) {
+        if (!scannedTables.catalogAvailable || modeledTables.length === 0) {
+            return {
+                enabledTables: [],
+                enabledTableKeys: null,
+                enabledSchemas: ['*'],
+                modeledTableCatalog: modeledTables,
+                floorDisabled: true,
+                warnings: [...warnings, 'query_history_scope_floor_disabled:catalog_unavailable'],
+            };
+        }
+        return {
+            enabledTables: [],
+            enabledTableKeys: null,
+            enabledSchemas: [...new Set(explicitSchemas)].sort(),
+            modeledTableCatalog: modeledTables,
+            floorDisabled: false,
+            warnings,
+        };
+    }
+    const schemas = new Set(declaredSchemas(input.driver, input.connection));
+    for (const ref of semanticTables.refs) {
+        if (ref.db)
+            schemas.add(ref.db);
+    }
+    if (schemas.size > 0 && (!scannedTables.catalogAvailable || modeledTables.length === 0)) {
+        return {
+            enabledTables: [],
+            enabledTableKeys: null,
+            enabledSchemas: ['*'],
+            modeledTableCatalog: modeledTables,
+            floorDisabled: true,
+            warnings: [...warnings, 'query_history_scope_floor_disabled:catalog_unavailable'],
+        };
+    }
+    return {
+        enabledTables: [],
+        enabledTableKeys: null,
+        enabledSchemas: [...schemas].sort(),
+        modeledTableCatalog: modeledTables,
+        floorDisabled: false,
+        warnings,
+    };
+}

package/dist/context/ingest/adapters/historic-sql/scope-membership.d.ts

@@ -0,0 +1,8 @@
+import type { KtxTableRef } from '../../../scan/types.js';
+export interface QueryHistoryScopeMembershipConfig {
+    enabledTables: readonly KtxTableRef[];
+    enabledSchemas: readonly string[];
+}
+export declare function isQueryHistoryScopeFloorDisabled(config: QueryHistoryScopeMembershipConfig): boolean;
+export declare function shouldFailOpenQueryHistoryScope(config: QueryHistoryScopeMembershipConfig): boolean;
+export declare function includedQueryHistoryTableRefs(tablesTouched: readonly KtxTableRef[], config: QueryHistoryScopeMembershipConfig): KtxTableRef[];

package/dist/context/ingest/adapters/historic-sql/scope-membership.js

@@ -0,0 +1,29 @@
+import { tableRefKey, tableRefSet } from '../../../scan/table-ref.js';
+function schemaNameForRef(ref) {
+    return ref.db && ref.db.length > 0 ? ref.db : null;
+}
+function schemaNamesFromConfig(enabledSchemas) {
+    return new Set(enabledSchemas.filter((schema) => schema !== '*'));
+}
+export function isQueryHistoryScopeFloorDisabled(config) {
+    return config.enabledSchemas.includes('*');
+}
+export function shouldFailOpenQueryHistoryScope(config) {
+    return (config.enabledTables.length === 0 &&
+        !isQueryHistoryScopeFloorDisabled(config) &&
+        config.enabledSchemas.length === 0);
+}
+export function includedQueryHistoryTableRefs(tablesTouched, config) {
+    if (config.enabledTables.length > 0) {
+        const enabled = tableRefSet(config.enabledTables);
+        return tablesTouched.filter((ref) => enabled.has(tableRefKey(ref)));
+    }
+    if (isQueryHistoryScopeFloorDisabled(config) || shouldFailOpenQueryHistoryScope(config)) {
+        return [...tablesTouched];
+    }
+    const schemas = schemaNamesFromConfig(config.enabledSchemas);
+    return tablesTouched.filter((ref) => {
+        const schema = schemaNameForRef(ref);
+        return schema !== null && schemas.has(schema);
+    });
+}

package/dist/context/ingest/adapters/historic-sql/snowflake-query-history-reader.js

@@ -148,26 +148,75 @@ export class SnowflakeHistoricSqlQueryHistoryReader {
     }
     async *fetchAggregated(client, window, config) {
         const sql = `
+WITH filtered_queries AS (
+  SELECT
+    query_hash,
+    query_text,
+    user_name,
+    start_time,
+    total_elapsed_time,
+    execution_status,
+    rows_produced
+  FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
+  WHERE query_text IS NOT NULL
+    AND query_type IN ('SELECT', 'MERGE')
+    AND start_time >= ${timestampLiteral(window.start)}
+    AND start_time < ${timestampLiteral(window.end)}
+),
+template_stats AS (
+  SELECT
+    query_hash AS template_id,
+    MIN(query_text) AS canonical_sql,
+    COUNT(*) AS executions,
+    COUNT(DISTINCT user_name) AS distinct_users,
+    MIN(start_time) AS first_seen,
+    MAX(start_time) AS last_seen,
+    APPROX_PERCENTILE(total_elapsed_time, 0.50) AS p50_ms,
+    APPROX_PERCENTILE(total_elapsed_time, 0.95) AS p95_ms,
+    DIV0(COUNT_IF(execution_status != 'SUCCESS'), COUNT(*)) AS error_rate,
+    SUM(rows_produced) AS rows_produced
+  FROM filtered_queries
+  GROUP BY query_hash
+  HAVING COUNT(*) >= ${config.minExecutions}
+),
+template_users AS (
+  SELECT
+    query_hash AS template_id,
+    user_name AS user,
+    COUNT(*) AS executions,
+    MAX(start_time) AS last_seen
+  FROM filtered_queries
+  GROUP BY query_hash, user_name
+)
 SELECT
-  query_hash AS template_id,
-  MIN(query_text) AS canonical_sql,
-  COUNT(*) AS executions,
-  COUNT(DISTINCT user_name) AS distinct_users,
-  MIN(start_time) AS first_seen,
-  MAX(start_time) AS last_seen,
-  APPROX_PERCENTILE(total_elapsed_time, 0.50) AS p50_ms,
-  APPROX_PERCENTILE(total_elapsed_time, 0.95) AS p95_ms,
-  DIV0(COUNT_IF(execution_status != 'SUCCESS'), COUNT(*)) AS error_rate,
-  SUM(rows_produced) AS rows_produced,
-  ARRAY_AGG(OBJECT_CONSTRUCT('user', user_name, 'executions', 1)) WITHIN GROUP (ORDER BY start_time DESC)::string AS top_users
-FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
-WHERE query_text IS NOT NULL
-  AND query_type IN ('SELECT', 'MERGE')
-  AND start_time >= ${timestampLiteral(window.start)}
-  AND start_time < ${timestampLiteral(window.end)}
-GROUP BY query_hash
-HAVING COUNT(*) >= ${config.minExecutions}
-ORDER BY executions DESC`.trim();
+  stats.template_id,
+  stats.canonical_sql,
+  stats.executions,
+  stats.distinct_users,
+  stats.first_seen,
+  stats.last_seen,
+  stats.p50_ms,
+  stats.p95_ms,
+  stats.error_rate,
+  stats.rows_produced,
+  ARRAY_AGG(
+    OBJECT_CONSTRUCT('user', users.user, 'executions', users.executions)
+  ) WITHIN GROUP (ORDER BY users.executions DESC, users.last_seen DESC)::string AS top_users
+FROM template_stats AS stats
+JOIN template_users AS users
+  ON users.template_id = stats.template_id
+GROUP BY
+  stats.template_id,
+  stats.canonical_sql,
+  stats.executions,
+  stats.distinct_users,
+  stats.first_seen,
+  stats.last_seen,
+  stats.p50_ms,
+  stats.p95_ms,
+  stats.error_rate,
+  stats.rows_produced
+ORDER BY stats.executions DESC`.trim();
         const result = await queryClient(client).executeQuery(sql);
         if (result.error) {
             throw grantsError(result.error);