@kaelio/ktx 0.8.0 → 0.10.0

package/dist/context/ingest/adapters/historic-sql/stage-unified.js

@@ -1,8 +1,10 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
+import { tableRefKey } from '../../../scan/table-ref.js';
 import { bucketDistinctUsers, bucketErrorRate, bucketExecutions, bucketFrequency, bucketP95Runtime, bucketRecency, } from './buckets.js';
 import { splitHistoricSqlPatternInputs } from './pattern-inputs.js';
 import { compileHistoricSqlRedactionPatterns, redactHistoricSqlText, } from './redaction.js';
+import { includedQueryHistoryTableRefs, isQueryHistoryScopeFloorDisabled, shouldFailOpenQueryHistoryScope, } from './scope-membership.js';
 import { HISTORIC_SQL_SOURCE_KEY, aggregatedTemplateSchema, historicSqlUnifiedPullConfigSchema, } from './types.js';
 const TRIVIAL_SQL_RE = /^\s*SELECT\s+(1|NOW\(\)|CURRENT_TIMESTAMP|VERSION\(\))\s*;?\s*$/i;
 const NOISE_PREFIX_RE = /^\s*(SHOW|DESCRIBE|DESC|EXPLAIN|USE|SET)\b/i;
@@ -17,9 +19,22 @@ function compilePatterns(patterns) {
 function matchesAny(value, patterns) {
     return !!value && patterns.some((pattern) => pattern.test(value));
 }
+// ktx's own warehouse scan emits relationship- and column-profiling probes that land in
+// pg_stat_statements (relationship-validation, relationship-composite-candidates, and each
+// dialect's relationship value aggregation). They are ktx introspection, not genuine query
+// usage, so they must not be mined back as query history. The markers are ktx-owned
+// identifiers, stable across dialects.
+function isKtxScanProbe(sql) {
+    if (/\brelationship_profile_values\b/i.test(sql)) {
+        return true;
+    }
+    return /\bchild_values\b/i.test(sql) && /\bparent_values\b/i.test(sql);
+}
 function shouldDropBySql(sql, config) {
     if (NOISE_PREFIX_RE.test(sql) || SYSTEM_TABLE_RE.test(sql))
         return true;
+    if (isKtxScanProbe(sql))
+        return true;
     if (config.filters.dropTrivialProbes !== false && TRIVIAL_SQL_RE.test(sql))
         return true;
     return false;
@@ -32,8 +47,7 @@ function shouldDropByUsers(template, config) {
     const matchingExecutions = template.topUsers
         .filter((entry) => matchesAny(entry.user, patterns))
         .reduce((sum, entry) => sum + entry.executions, 0);
-    const allExecutions = template.topUsers.reduce((sum, entry) => sum + entry.executions, 0);
-    const serviceOnly = allExecutions > 0 && matchingExecutions >= allExecutions;
+    const serviceOnly = template.stats.executions > 0 && matchingExecutions >= template.stats.executions;
     return service.mode === 'exclude' ? serviceOnly : !serviceOnly;
 }
 function shouldDropByFailure(template, config) {
@@ -49,38 +63,8 @@ function shouldDropTemplate(template, config) {
         return true;
     return false;
 }
-function normalizeTableIdentifier(value) {
-    return value.trim().toLowerCase();
-}
-function unqualifiedTableIdentifier(value) {
-    const parts = normalizeTableIdentifier(value).split('.').filter(Boolean);
-    return parts.at(-1) ?? '';
-}
-function buildEnabledTableFilter(enabledTables) {
-    if (enabledTables.length === 0) {
-        return null;
-    }
-    const exact = new Set(enabledTables.map(normalizeTableIdentifier).filter((value) => value.length > 0));
-    const unqualifiedCounts = new Map();
-    for (const table of exact) {
-        const unqualified = unqualifiedTableIdentifier(table);
-        if (unqualified.length > 0) {
-            unqualifiedCounts.set(unqualified, (unqualifiedCounts.get(unqualified) ?? 0) + 1);
-        }
-    }
-    return {
-        exact,
-        uniqueUnqualified: new Set([...unqualifiedCounts.entries()]
-            .filter(([, count]) => count === 1)
-            .map(([table]) => table)),
-    };
-}
-function isEnabledTable(table, filter) {
-    if (!filter) {
-        return true;
-    }
-    const normalized = normalizeTableIdentifier(table);
-    return filter.exact.has(normalized) || filter.uniqueUnqualified.has(unqualifiedTableIdentifier(normalized));
+function displayTableRef(ref) {
+    return [ref.catalog, ref.db, ref.name].filter((part) => !!part && part.length > 0).join('.');
 }
 function historicSqlWindowDays(config) {
     return 'windowDays' in config ? config.windowDays : 90;
@@ -107,9 +91,10 @@ function recordJoin(acc, otherTable, columns, executions) {
         acc.observedJoins.set(otherTable, byColumns);
     }
 }
-function accumulatorFor(table) {
+function accumulatorFor(tableRef) {
     return {
-        table,
+        tableRef,
+        table: displayTableRef(tableRef),
         executions: 0,
         distinctUsers: 0,
         errorRateNumerator: 0,
@@ -138,8 +123,8 @@ function addTemplate(acc, parsed) {
         }
     }
     const joinColumns = parsed.columnsByClause.join ?? [];
-    for (const otherTable of parsed.tablesTouched.filter((table) => table !== acc.table)) {
-        recordJoin(acc, otherTable, joinColumns, executions);
+    for (const otherTable of parsed.tablesTouched.filter((table) => tableRefKey(table) !== tableRefKey(acc.tableRef))) {
+        recordJoin(acc, displayTableRef(otherTable), joinColumns, executions);
     }
     acc.topTemplates.push(parsed.template);
 }
@@ -170,6 +155,7 @@ function toStagedTable(acc, now) {
     }));
     return {
         table: acc.table,
+        tableRef: acc.tableRef,
         stats: {
             executionsBucket: bucketExecutions(acc.executions),
             distinctUsersBucket: bucketDistinctUsers(acc.distinctUsers),
@@ -188,7 +174,7 @@ function toPatternsInput(parsedTemplates) {
             .map(({ template, tablesTouched }) => ({
             id: template.templateId,
             canonicalSql: template.canonicalSql,
-            tablesTouched: [...tablesTouched].sort(),
+            tablesTouched: [...tablesTouched].sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right))),
             executionsBucket: bucketExecutions(template.stats.executions),
             distinctUsersBucket: bucketDistinctUsers(template.stats.distinctUsers),
             dialect: template.dialect,
@@ -198,7 +184,6 @@ function toPatternsInput(parsedTemplates) {
 }
 export async function stageHistoricSqlAggregatedSnapshot(input) {
     const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig);
-    const enabledTableFilter = buildEnabledTableFilter(config.enabledTables);
     const redactors = compileHistoricSqlRedactionPatterns(config.redactionPatterns);
     const now = input.now ?? new Date();
     const windowStart = new Date(now.getTime() - historicSqlWindowDays(config) * 24 * 60 * 60 * 1000);
@@ -212,8 +197,25 @@ export async function stageHistoricSqlAggregatedSnapshot(input) {
             snapshot.push(parsed);
         }
     }
-    const analysis = await input.sqlAnalysis.analyzeBatch(snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql })), config.dialect);
-    const warnings = [];
+    const analysisItems = snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql }));
+    const analysisOptions = config.modeledTableCatalog.length > 0 ? { catalog: { tables: config.modeledTableCatalog } } : undefined;
+    const warnings = [
+        ...config.scopeFloorWarnings,
+        ...(shouldFailOpenQueryHistoryScope(config) ? ['query_history_scope_floor_disabled:empty_modeled_scope'] : []),
+    ];
+    let scopeDisabledByQualificationFailure = false;
+    let analysis;
+    try {
+        analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, config.dialect, analysisOptions);
+    }
+    catch (error) {
+        if (!analysisOptions || config.enabledTables.length > 0 || isQueryHistoryScopeFloorDisabled(config)) {
+            throw error;
+        }
+        warnings.push('query_history_scope_floor_disabled:catalog_qualification_failed');
+        scopeDisabledByQualificationFailure = true;
+        analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, config.dialect, undefined);
+    }
     const parsedTemplates = [];
     for (const template of snapshot) {
         const parsed = analysis.get(template.templateId);
@@ -221,8 +223,12 @@ export async function stageHistoricSqlAggregatedSnapshot(input) {
             warnings.push(`parse_failed:${template.templateId}`);
             continue;
         }
-        const tablesTouched = [...new Set(parsed.tablesTouched)].filter((table) => table.length > 0).sort();
-        const includedTables = tablesTouched.filter((table) => isEnabledTable(table, enabledTableFilter));
+        const tablesTouched = [...new Map(parsed.tablesTouched.map((ref) => [tableRefKey(ref), ref])).values()]
+            .filter((ref) => ref.name.length > 0)
+            .sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right)));
+        const includedTables = scopeDisabledByQualificationFailure
+            ? [...tablesTouched]
+            : includedQueryHistoryTableRefs(tablesTouched, config);
         if (includedTables.length === 0) {
             continue;
         }
@@ -235,19 +241,20 @@ export async function stageHistoricSqlAggregatedSnapshot(input) {
     }
     const byTable = new Map();
     for (const parsed of parsedTemplates) {
-        for (const table of parsed.includedTables) {
-            const acc = byTable.get(table) ?? accumulatorFor(table);
+        for (const tableRef of parsed.includedTables) {
+            const key = tableRefKey(tableRef);
+            const acc = byTable.get(key) ?? accumulatorFor(tableRef);
             addTemplate(acc, parsed);
-            byTable.set(table, acc);
+            byTable.set(key, acc);
         }
     }
     await mkdir(input.stagedDir, { recursive: true });
-    for (const [table, acc] of [...byTable.entries()].sort(([left], [right]) => left.localeCompare(right))) {
-        await writeJson(input.stagedDir, `tables/${table}.json`, toStagedTable(acc, now));
+    for (const [, acc] of [...byTable.entries()].sort((left, right) => left[0].localeCompare(right[0]))) {
+        await writeJson(input.stagedDir, `tables/${acc.table}.json`, toStagedTable(acc, now));
     }
     const patternsInput = toPatternsInput(parsedTemplates);
     const patternInputSplit = splitHistoricSqlPatternInputs(patternsInput);
-    const allWarnings = [...warnings, ...patternInputSplit.warnings];
+    const allWarnings = [...new Set([...warnings, ...patternInputSplit.warnings])];
     await writeJson(input.stagedDir, 'patterns-input.json', patternInputSplit.auditInput);
     for (const shard of patternInputSplit.shards) {
         await writeJson(input.stagedDir, shard.path, shard.input);

package/dist/context/ingest/adapters/historic-sql/types.d.ts

@@ -9,7 +9,19 @@ declare const historicSqlDialectSchema: z.ZodEnum<{
 export type HistoricSqlDialect = z.infer<typeof historicSqlDialectSchema>;
 export declare const historicSqlUnifiedPullConfigSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     minExecutions: z.ZodDefault<z.ZodNumber>;
-    enabledTables: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    enabledTables: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        catalog: z.ZodNullable<z.ZodString>;
+        db: z.ZodNullable<z.ZodString>;
+        name: z.ZodString;
+    }, z.core.$strict>>>;
+    enabledSchemas: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    modeledTableCatalog: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        catalog: z.ZodNullable<z.ZodString>;
+        db: z.ZodNullable<z.ZodString>;
+        name: z.ZodString;
+        columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strict>>>;
+    scopeFloorWarnings: z.ZodDefault<z.ZodArray<z.ZodString>>;
     filters: z.ZodDefault<z.ZodObject<{
         serviceAccounts: z.ZodOptional<z.ZodObject<{
             patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
@@ -41,7 +53,19 @@ export declare const historicSqlUnifiedPullConfigSchema: z.ZodDiscriminatedUnion
     windowDays: z.ZodDefault<z.ZodNumber>;
 }, z.core.$strip>, z.ZodObject<{
     minExecutions: z.ZodDefault<z.ZodNumber>;
-    enabledTables: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    enabledTables: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        catalog: z.ZodNullable<z.ZodString>;
+        db: z.ZodNullable<z.ZodString>;
+        name: z.ZodString;
+    }, z.core.$strict>>>;
+    enabledSchemas: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    modeledTableCatalog: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        catalog: z.ZodNullable<z.ZodString>;
+        db: z.ZodNullable<z.ZodString>;
+        name: z.ZodString;
+        columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strict>>>;
+    scopeFloorWarnings: z.ZodDefault<z.ZodArray<z.ZodString>>;
     filters: z.ZodDefault<z.ZodObject<{
         serviceAccounts: z.ZodOptional<z.ZodObject<{
             patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
@@ -95,6 +119,11 @@ export declare const aggregatedTemplateSchema: z.ZodObject<{
 export type AggregatedTemplate = z.infer<typeof aggregatedTemplateSchema>;
 export declare const stagedTableInputSchema: z.ZodObject<{
     table: z.ZodString;
+    tableRef: z.ZodObject<{
+        catalog: z.ZodNullable<z.ZodString>;
+        db: z.ZodNullable<z.ZodString>;
+        name: z.ZodString;
+    }, z.core.$strict>;
     stats: z.ZodObject<{
         executionsBucket: z.ZodString;
         distinctUsersBucket: z.ZodString;
@@ -121,7 +150,11 @@ export declare const stagedPatternsInputSchema: z.ZodObject<{
     templates: z.ZodArray<z.ZodObject<{
         id: z.ZodString;
         canonicalSql: z.ZodString;
-        tablesTouched: z.ZodArray<z.ZodString>;
+        tablesTouched: z.ZodArray<z.ZodObject<{
+            catalog: z.ZodNullable<z.ZodString>;
+            db: z.ZodNullable<z.ZodString>;
+            name: z.ZodString;
+        }, z.core.$strict>>;
         executionsBucket: z.ZodString;
         distinctUsersBucket: z.ZodString;
         dialect: z.ZodEnum<{

package/dist/context/ingest/adapters/historic-sql/types.js

@@ -2,9 +2,20 @@ import { z } from 'zod';
 export const HISTORIC_SQL_SOURCE_KEY = 'historic-sql';
 const historicSqlDialectSchema = z.enum(['snowflake', 'bigquery', 'postgres']);
 const filterModeSchema = z.enum(['exclude', 'include', 'mark-only']);
+const ktxTableRefSchema = z.object({
+    catalog: z.string().nullable(),
+    db: z.string().nullable(),
+    name: z.string().min(1),
+}).strict();
+const ktxTableRefWithColumnsSchema = ktxTableRefSchema.extend({
+    columns: z.array(z.string().min(1)).optional(),
+}).strict();
 const historicSqlCommonPullConfigSchema = z.object({
     minExecutions: z.number().int().nonnegative().default(5),
-    enabledTables: z.array(z.string().min(1)).default([]),
+    enabledTables: z.array(ktxTableRefSchema).default([]),
+    enabledSchemas: z.array(z.string().min(1)).default([]),
+    modeledTableCatalog: z.array(ktxTableRefWithColumnsSchema).default([]),
+    scopeFloorWarnings: z.array(z.string()).default([]),
     filters: z.object({
         serviceAccounts: z.object({
             patterns: z.array(z.string()).default([]),
@@ -54,6 +65,7 @@ export const aggregatedTemplateSchema = z.object({
 });
 export const stagedTableInputSchema = z.object({
     table: z.string().min(1),
+    tableRef: ktxTableRefSchema,
     stats: z.object({
         executionsBucket: z.string(),
         distinctUsersBucket: z.string(),
@@ -77,7 +89,7 @@ export const stagedPatternsInputSchema = z.object({
     templates: z.array(z.object({
         id: z.string(),
         canonicalSql: z.string(),
-        tablesTouched: z.array(z.string()),
+        tablesTouched: z.array(ktxTableRefSchema),
         executionsBucket: z.string(),
         distinctUsersBucket: z.string(),
         dialect: historicSqlDialectSchema,

package/dist/context/ingest/context-candidates/curator-pagination.service.d.ts

@@ -31,11 +31,7 @@ export interface CuratorPaginationInput {
     buildUserPrompt: (input: CuratorPaginationPromptInput) => string;
     buildToolSet: (passNumber: number) => KtxRuntimeToolSet;
     getReconciliationActions: () => MemoryAction[];
-    onStepFinish?: (info: {
-        passNumber: number;
-        stepIndex: number;
-        stepBudget: number;
-    }) => void;
+    abortSignal?: AbortSignal;
 }
 interface CuratorPaginationResult extends ReconciliationOutcome {
     report: CuratorPaginationReport;

package/dist/context/ingest/context-candidates/curator-pagination.service.js

@@ -163,9 +163,7 @@ export class CuratorPaginationService {
             sourceKey: params.input.sourceKey,
             jobId: params.input.jobId,
             forceRun: params.forceRun,
-            onStepFinish: params.input.onStepFinish
-                ? ({ stepIndex, stepBudget }) => params.input.onStepFinish?.({ passNumber: params.passNumber, stepIndex, stepBudget })
-                : undefined,
+            abortSignal: params.input.abortSignal,
         });
     }
     batchSummary(items) {

package/dist/context/ingest/context-evidence/sqlite-context-evidence-store.d.ts

@@ -47,7 +47,7 @@ export declare class SqliteContextEvidenceStore implements ContextEvidenceIndexS
         assertion: string;
         rationale: string;
         actionHint: string;
-        status: "conflict" | "merged" | "pending" | "promoted" | "rejected";
+        status: "conflict" | "merged" | "rejected" | "pending" | "promoted";
         promotionScore: number;
         suggestedPageKey: string | null;
         evidenceRefs: JsonValue;

package/dist/context/ingest/final-gate-repair.d.ts

@@ -20,6 +20,7 @@ export interface RepairFinalGateFailureInput {
     repairKind: FinalGateRepairKind;
     maxAttempts?: number;
     stepBudget?: number;
+    abortSignal?: AbortSignal;
 }
 export declare function finalGateRepairPaths(input: {
     changedWikiPageKeys: string[];

package/dist/context/ingest/final-gate-repair.js

@@ -153,6 +153,7 @@ export async function repairFinalGateFailure(input) {
                 jobId: input.trace.context.jobId,
                 repairKind: input.repairKind,
             },
+            abortSignal: input.abortSignal,
         }));
         if (result.stopReason === 'error') {
             lastFailure = result.error?.message ?? 'gate repair agent loop errored';

package/dist/context/ingest/ingest-bundle.runner.d.ts

@@ -9,6 +9,9 @@ export declare class IngestBundleRunner {
     private readonly chainByConnection;
     constructor(deps: IngestBundleRunnerDeps);
     run(job: IngestBundleJob, ctx?: IngestJobContext): Promise<IngestBundleResult>;
+    private formatRateLimitWait;
+    private subscribeRateLimitGovernor;
+    private withRateLimitWorkSlot;
     /**
      * When profiling is enabled — via the `KTX_PROFILE_INGEST` env var or the
      * `ingest.profile` config setting — read the job's trace + tool transcripts