@kaelio/ktx 0.4.1 → 0.6.0

package/dist/context/ingest/adapters/historic-sql/connection-dialect.d.ts

@@ -0,0 +1,8 @@
+import type { HistoricSqlDialect } from './types.js';
+export declare function isQueryHistoryEnabled(connection: unknown): boolean;
+/**
+ * Resolves the query-history dialect for a connection. Returns null when
+ * query history is disabled, or when the connection's driver has no
+ * query-history reader.
+ */
+export declare function queryHistoryDialectForConnection(connection: unknown): HistoricSqlDialect | null;

package/dist/context/ingest/adapters/historic-sql/connection-dialect.js

@@ -0,0 +1,34 @@
+import { getDriverRegistration } from '../../../connections/drivers.js';
+const historicSqlDialects = ['postgres', 'bigquery', 'snowflake'];
+function recordOrNull(value) {
+    return value && typeof value === 'object' && !Array.isArray(value) ? value : null;
+}
+function queryHistoryRecord(connection) {
+    const conn = recordOrNull(connection);
+    const context = conn ? recordOrNull(conn.context) : null;
+    return context ? recordOrNull(context.queryHistory) : null;
+}
+function historicSqlDialectForDriver(driver) {
+    const dialect = historicSqlDialects.find((candidate) => candidate === driver);
+    if (!dialect) {
+        throw new Error(`Driver "${driver}" is marked as historic-SQL capable but has no HistoricSqlDialect mapping.`);
+    }
+    return dialect;
+}
+export function isQueryHistoryEnabled(connection) {
+    return queryHistoryRecord(connection)?.enabled === true;
+}
+/**
+ * Resolves the query-history dialect for a connection. Returns null when
+ * query history is disabled, or when the connection's driver has no
+ * query-history reader.
+ */
+export function queryHistoryDialectForConnection(connection) {
+    if (!isQueryHistoryEnabled(connection)) {
+        return null;
+    }
+    const conn = recordOrNull(connection);
+    const driver = String(conn?.driver ?? '').toLowerCase();
+    const registration = getDriverRegistration(driver);
+    return registration?.hasHistoricSqlReader ? historicSqlDialectForDriver(registration.driver) : null;
+}

package/dist/context/ingest/adapters/historic-sql/evidence-tool.d.ts

@@ -0,0 +1,41 @@
+interface EmitHistoricSqlEvidenceToolContext {
+    connectionId?: string | null;
+    session?: {
+        ingest?: {
+            runId: string;
+            sourceKey: string;
+        };
+        allowedRawPaths?: ReadonlySet<string>;
+        configService?: {
+            writeFile(path: string, content: string, author: string, authorEmail: string, commitMessage: string, options?: {
+                skipLock?: boolean;
+            }): Promise<unknown>;
+        };
+    };
+}
+export declare function createEmitHistoricSqlEvidenceTool(defaultContext?: EmitHistoricSqlEvidenceToolContext): import("ai").Tool<{
+    kind: "pattern" | "table_usage";
+    table?: string | undefined;
+    usage?: {
+        [x: string]: unknown;
+        narrative: string;
+        frequencyTier: "high" | "mid" | "low" | "unused";
+        commonFilters: string[];
+        commonJoins: {
+            table: string;
+            on: string[];
+        }[];
+        commonGroupBys?: string[] | undefined;
+        staleSince?: string | null | undefined;
+    } | undefined;
+    pattern?: {
+        slug: string;
+        title: string;
+        narrative: string;
+        definitionSql: string;
+        tablesInvolved: string[];
+        slRefs: string[];
+        constituentTemplateIds: string[];
+    } | undefined;
+}, string>;
+export {};

package/dist/context/ingest/adapters/historic-sql/evidence-tool.js

@@ -0,0 +1,91 @@
+import { tool } from 'ai';
+import { z } from 'zod';
+import { historicSqlEvidencePath, serializeHistoricSqlEvidence } from './evidence.js';
+import { patternOutputSchema, tableUsageOutputSchema } from './skill-schemas.js';
+const SYSTEM_AUTHOR = 'System User';
+const SYSTEM_EMAIL = 'system@example.com';
+const emitHistoricSqlEvidenceInputSchema = z
+    .object({
+    kind: z.enum(['table_usage', 'pattern']),
+    table: z.string().min(1).optional(),
+    usage: tableUsageOutputSchema.optional(),
+    pattern: patternOutputSchema.optional(),
+})
+    .superRefine((input, ctx) => {
+    if (input.kind === 'table_usage') {
+        if (!input.table) {
+            ctx.addIssue({
+                code: 'custom',
+                path: ['table'],
+                message: 'table is required when kind is table_usage',
+            });
+        }
+        if (!input.usage) {
+            ctx.addIssue({
+                code: 'custom',
+                path: ['usage'],
+                message: 'usage is required when kind is table_usage',
+            });
+        }
+    }
+    if (input.kind === 'pattern' && !input.pattern) {
+        ctx.addIssue({
+            code: 'custom',
+            path: ['pattern'],
+            message: 'pattern is required when kind is pattern',
+        });
+    }
+});
+function unitKeyForEvidence(input) {
+    if (input.kind === 'table_usage') {
+        return `historic-sql-table-${String(input.table).replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '')}`;
+    }
+    return `historic-sql-pattern-${String(input.pattern?.slug).replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '')}`;
+}
+function evidenceEnvelope(input, connectionId, rawPaths) {
+    if (input.kind === 'table_usage') {
+        if (!input.table || !input.usage) {
+            throw new Error('Invalid historic-SQL table usage evidence input.');
+        }
+        return {
+            kind: 'table_usage',
+            connectionId,
+            table: input.table,
+            rawPaths,
+            usage: input.usage,
+        };
+    }
+    if (!input.pattern) {
+        throw new Error('Invalid historic-SQL pattern evidence input.');
+    }
+    return {
+        kind: 'pattern',
+        connectionId,
+        rawPaths,
+        pattern: input.pattern,
+    };
+}
+export function createEmitHistoricSqlEvidenceTool(defaultContext) {
+    return tool({
+        description: 'Record typed historic-SQL evidence for deterministic projection. Use this instead of wiki_write, sl_write_source, sl_edit_source, or context_candidate_write during historic-SQL WorkUnits.',
+        inputSchema: emitHistoricSqlEvidenceInputSchema,
+        execute: async (input, options) => {
+            const context = options.experimental_context ?? defaultContext;
+            const ingest = context?.session?.ingest;
+            const configService = context?.session?.configService;
+            if (!ingest || ingest.sourceKey !== 'historic-sql' || !configService || !context?.connectionId) {
+                return 'Error: emit_historic_sql_evidence is only available during historic-sql ingest.';
+            }
+            const rawPaths = context.session?.allowedRawPaths ? [...context.session.allowedRawPaths].sort() : [];
+            if (rawPaths.length === 0) {
+                return 'Error: emit_historic_sql_evidence requires a WorkUnit context with at least one raw file.';
+            }
+            const unitKey = unitKeyForEvidence(input);
+            const evidence = evidenceEnvelope(input, context.connectionId, rawPaths);
+            const content = serializeHistoricSqlEvidence(evidence);
+            await configService.writeFile(historicSqlEvidencePath(ingest.runId, unitKey), content, SYSTEM_AUTHOR, SYSTEM_EMAIL, `Record historic-SQL evidence: ${unitKey}`, { skipLock: true });
+            const label = evidence.kind === 'table_usage' ? evidence.table : evidence.pattern.slug;
+            return `Recorded historic-SQL ${input.kind} evidence for ${label}.`;
+        },
+    });
+}

package/dist/context/ingest/adapters/historic-sql/evidence.d.ts

@@ -0,0 +1,77 @@
+import { z } from 'zod';
+/** @internal */
+export declare const historicSqlTableUsageEvidenceSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"table_usage">;
+    connectionId: z.ZodString;
+    table: z.ZodString;
+    rawPaths: z.ZodArray<z.ZodString>;
+    usage: z.ZodObject<{
+        narrative: z.ZodString;
+        frequencyTier: z.ZodEnum<{
+            high: "high";
+            mid: "mid";
+            low: "low";
+            unused: "unused";
+        }>;
+        commonFilters: z.ZodArray<z.ZodString>;
+        commonGroupBys: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        commonJoins: z.ZodArray<z.ZodObject<{
+            table: z.ZodString;
+            on: z.ZodArray<z.ZodString>;
+        }, z.core.$strip>>;
+        staleSince: z.ZodOptional<z.ZodNullable<z.ZodISODateTime>>;
+    }, z.core.$loose>;
+}, z.core.$strip>;
+/** @internal */
+export declare const historicSqlPatternEvidenceSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"pattern">;
+    connectionId: z.ZodString;
+    rawPaths: z.ZodArray<z.ZodString>;
+    pattern: z.ZodObject<{
+        slug: z.ZodString;
+        title: z.ZodString;
+        narrative: z.ZodString;
+        definitionSql: z.ZodString;
+        tablesInvolved: z.ZodArray<z.ZodString>;
+        slRefs: z.ZodArray<z.ZodString>;
+        constituentTemplateIds: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export declare const historicSqlEvidenceEnvelopeSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"table_usage">;
+    connectionId: z.ZodString;
+    table: z.ZodString;
+    rawPaths: z.ZodArray<z.ZodString>;
+    usage: z.ZodObject<{
+        narrative: z.ZodString;
+        frequencyTier: z.ZodEnum<{
+            high: "high";
+            mid: "mid";
+            low: "low";
+            unused: "unused";
+        }>;
+        commonFilters: z.ZodArray<z.ZodString>;
+        commonGroupBys: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        commonJoins: z.ZodArray<z.ZodObject<{
+            table: z.ZodString;
+            on: z.ZodArray<z.ZodString>;
+        }, z.core.$strip>>;
+        staleSince: z.ZodOptional<z.ZodNullable<z.ZodISODateTime>>;
+    }, z.core.$loose>;
+}, z.core.$strip>, z.ZodObject<{
+    kind: z.ZodLiteral<"pattern">;
+    connectionId: z.ZodString;
+    rawPaths: z.ZodArray<z.ZodString>;
+    pattern: z.ZodObject<{
+        slug: z.ZodString;
+        title: z.ZodString;
+        narrative: z.ZodString;
+        definitionSql: z.ZodString;
+        tablesInvolved: z.ZodArray<z.ZodString>;
+        slRefs: z.ZodArray<z.ZodString>;
+        constituentTemplateIds: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>;
+}, z.core.$strip>], "kind">;
+export type HistoricSqlEvidenceEnvelope = z.infer<typeof historicSqlEvidenceEnvelopeSchema>;
+export declare function historicSqlEvidencePath(runId: string, unitKey: string): string;
+export declare function serializeHistoricSqlEvidence(evidence: HistoricSqlEvidenceEnvelope): string;

package/dist/context/ingest/adapters/historic-sql/evidence.js

@@ -0,0 +1,34 @@
+import { z } from 'zod';
+import { patternOutputSchema, tableUsageOutputSchema } from './skill-schemas.js';
+function safeEvidenceSegment(value) {
+    const segment = value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '');
+    if (!segment) {
+        throw new Error(`Invalid historic-SQL evidence path segment: ${value}`);
+    }
+    return segment;
+}
+/** @internal */
+export const historicSqlTableUsageEvidenceSchema = z.object({
+    kind: z.literal('table_usage'),
+    connectionId: z.string().min(1),
+    table: z.string().min(1),
+    rawPaths: z.array(z.string().min(1)).min(1),
+    usage: tableUsageOutputSchema,
+});
+/** @internal */
+export const historicSqlPatternEvidenceSchema = z.object({
+    kind: z.literal('pattern'),
+    connectionId: z.string().min(1),
+    rawPaths: z.array(z.string().min(1)).min(1),
+    pattern: patternOutputSchema,
+});
+export const historicSqlEvidenceEnvelopeSchema = z.discriminatedUnion('kind', [
+    historicSqlTableUsageEvidenceSchema,
+    historicSqlPatternEvidenceSchema,
+]);
+export function historicSqlEvidencePath(runId, unitKey) {
+    return `.ktx/ingest-evidence/historic-sql/${safeEvidenceSegment(runId)}/${safeEvidenceSegment(unitKey)}.json`;
+}
+export function serializeHistoricSqlEvidence(evidence) {
+    return `${JSON.stringify(historicSqlEvidenceEnvelopeSchema.parse(evidence), null, 2)}\n`;
+}

package/dist/context/ingest/adapters/historic-sql/pattern-inputs.d.ts

@@ -0,0 +1,21 @@
+import type { StagedPatternsInput } from './types.js';
+/** @internal */
+export declare const HISTORIC_SQL_PATTERN_WORKUNIT_MAX_BYTES = 110000;
+interface HistoricSqlPatternInputShard {
+    path: string;
+    input: StagedPatternsInput;
+    byteLength: number;
+}
+export interface HistoricSqlPatternInputSplitResult {
+    auditInput: StagedPatternsInput;
+    shards: HistoricSqlPatternInputShard[];
+    warnings: string[];
+}
+export interface HistoricSqlPatternInputSplitOptions {
+    maxBytes?: number;
+}
+export declare function isHistoricSqlPatternInputShardPath(path: string): boolean;
+/** @internal */
+export declare function serializedStagedPatternsInputByteLength(input: StagedPatternsInput): number;
+export declare function splitHistoricSqlPatternInputs(input: StagedPatternsInput, options?: HistoricSqlPatternInputSplitOptions): HistoricSqlPatternInputSplitResult;
+export {};

package/dist/context/ingest/adapters/historic-sql/pattern-inputs.js

@@ -0,0 +1,68 @@
+import { Buffer } from 'node:buffer';
+const HISTORIC_SQL_PATTERN_WORKUNIT_DIR = 'patterns-input';
+/** @internal */
+export const HISTORIC_SQL_PATTERN_WORKUNIT_MAX_BYTES = 110_000;
+const HISTORIC_SQL_PATTERN_WORKUNIT_PATH_RE = /^patterns-input\/part-\d{4}\.json$/;
+export function isHistoricSqlPatternInputShardPath(path) {
+    return HISTORIC_SQL_PATTERN_WORKUNIT_PATH_RE.test(path);
+}
+function serializeStagedPatternsInput(input) {
+    return `${JSON.stringify(input, null, 2)}\n`;
+}
+/** @internal */
+export function serializedStagedPatternsInputByteLength(input) {
+    return Buffer.byteLength(serializeStagedPatternsInput(input), 'utf-8');
+}
+function sortedAuditTemplates(templates) {
+    return [...templates].sort((left, right) => left.id.localeCompare(right.id));
+}
+function sortedPatternCandidates(templates) {
+    return [...templates]
+        .filter((template) => template.tablesTouched.length >= 2)
+        .map((template) => ({ ...template, tablesTouched: [...template.tablesTouched].sort() }))
+        .sort((left, right) => {
+        const cardinality = right.tablesTouched.length - left.tablesTouched.length;
+        if (cardinality !== 0)
+            return cardinality;
+        const tableSignature = left.tablesTouched.join('\0').localeCompare(right.tablesTouched.join('\0'));
+        if (tableSignature !== 0)
+            return tableSignature;
+        return left.id.localeCompare(right.id);
+    });
+}
+function shardPath(index) {
+    return `${HISTORIC_SQL_PATTERN_WORKUNIT_DIR}/part-${String(index).padStart(4, '0')}.json`;
+}
+export function splitHistoricSqlPatternInputs(input, options = {}) {
+    const maxBytes = options.maxBytes ?? HISTORIC_SQL_PATTERN_WORKUNIT_MAX_BYTES;
+    const auditInput = { templates: sortedAuditTemplates(input.templates) };
+    const warnings = [];
+    const shards = [];
+    let current = [];
+    const flush = () => {
+        if (current.length === 0) {
+            return;
+        }
+        const shardInput = { templates: current };
+        shards.push({
+            path: shardPath(shards.length + 1),
+            input: shardInput,
+            byteLength: serializedStagedPatternsInputByteLength(shardInput),
+        });
+        current = [];
+    };
+    for (const template of sortedPatternCandidates(input.templates)) {
+        const singleInput = { templates: [template] };
+        if (serializedStagedPatternsInputByteLength(singleInput) > maxBytes) {
+            warnings.push(`patterns_input_template_too_large:${template.id}`);
+            continue;
+        }
+        const nextInput = { templates: [...current, template] };
+        if (current.length > 0 && serializedStagedPatternsInputByteLength(nextInput) > maxBytes) {
+            flush();
+        }
+        current.push(template);
+    }
+    flush();
+    return { auditInput, shards, warnings };
+}

package/dist/context/ingest/adapters/historic-sql/projection.d.ts

@@ -0,0 +1,24 @@
+import type { MemoryAction } from '../../../../context/memory/types.js';
+import type { FinalizationOverrideReplay } from '../../types.js';
+export interface HistoricSqlProjectionInput {
+    workdir: string;
+    connectionId: string;
+    syncId: string;
+    runId: string;
+    overrideReplay?: FinalizationOverrideReplay;
+}
+export interface HistoricSqlProjectionResult {
+    tableUsageMerged: number;
+    staleTablesMarked: number;
+    patternPagesWritten: number;
+    stalePatternPagesMarked: number;
+    archivedPatternPages: number;
+    touchedSources: Array<{
+        connectionId: string;
+        sourceName: string;
+    }>;
+    changedWikiPageKeys: string[];
+    actions: MemoryAction[];
+    warnings: string[];
+}
+export declare function projectHistoricSqlEvidence(input: HistoricSqlProjectionInput): Promise<HistoricSqlProjectionResult>;