@kaelio/ktx 0.4.1 → 0.5.0

package/dist/context/sl/tools/sl-validate.tool.js

@@ -0,0 +1,101 @@
+import { z } from 'zod';
+import { touchedSlSourceNamesForConnection } from '../../../context/tools/touched-sl-sources.js';
+import { BaseSemanticLayerTool, } from './base-semantic-layer.tool.js';
+import { slToolConnectionIdSchema } from './connection-id-schema.js';
+const slValidateInputSchema = z.object({
+    connectionId: slToolConnectionIdSchema.describe('Data source connection ID'),
+});
+/** @internal */
+export async function validateSemanticLayerEndpoint(connectionId, semanticLayerService) {
+    try {
+        return await semanticLayerService.validateSourcesForConnection(connectionId);
+    }
+    catch (e) {
+        return {
+            errors: [`Validation call failed: ${e instanceof Error ? e.message : String(e)}`],
+            warnings: [],
+        };
+    }
+}
+export class SlValidateTool extends BaseSemanticLayerTool {
+    name = 'sl_validate';
+    constructor(deps) {
+        super(deps);
+    }
+    get description() {
+        return `<purpose>
+Validate that all semantic layer sources for a connection form a consistent model.
+Checks: all join targets exist, grain is valid, no missing references.
+</purpose>
+
+<when_to_use>
+- After making edits with sl_write_source
+- Before querying, to ensure the model is healthy
+- When troubleshooting query failures
+</when_to_use>`;
+    }
+    get inputSchema() {
+        return slValidateInputSchema;
+    }
+    async call(input, context) {
+        const { connectionId } = input;
+        const semanticLayerService = context.session?.semanticLayerService ?? this.semanticLayerService;
+        const { sources } = await semanticLayerService.loadAllSources(connectionId);
+        if (sources.length === 0) {
+            return this.buildOutput(true, [], '(all)', {
+                validationErrors: ['No sources found for this connection.'],
+            });
+        }
+        let { errors, warnings } = await validateSemanticLayerEndpoint(connectionId, semanticLayerService);
+        const touched = context.session?.touchedSlSources;
+        if (touched && touched.size > 0) {
+            const touchedArr = touchedSlSourceNamesForConnection(touched, connectionId);
+            if (touchedArr.length > 0) {
+                errors = errors.filter((e) => touchedArr.some((n) => e.includes(n)));
+                warnings = warnings.filter((w) => touchedArr.some((n) => w.includes(n)));
+            }
+        }
+        const valid = errors.length === 0;
+        const parts = [];
+        parts.push(`**Semantic layer validation** for ${sources.length} source(s):`);
+        if (valid && warnings.length === 0) {
+            parts.push('All sources are valid. Join graph is consistent.');
+        }
+        else {
+            const summary = [];
+            if (errors.length > 0) {
+                summary.push(`${errors.length} error(s)`);
+            }
+            if (warnings.length > 0) {
+                summary.push(`${warnings.length} warning(s)`);
+            }
+            parts.push(`Found ${summary.join(' and ')}:`);
+            if (errors.length > 0) {
+                parts.push('', '**Errors:**');
+                for (const err of errors) {
+                    parts.push(`- ${err}`);
+                }
+            }
+            if (warnings.length > 0) {
+                parts.push('', '**Warnings:**');
+                for (const warn of warnings) {
+                    parts.push(`- ${warn}`);
+                }
+            }
+        }
+        // List sources summary
+        parts.push('\n**Sources:**');
+        for (const s of sources) {
+            parts.push(`- **${s.name}** (${s.sql ? 'sql' : 'table'}): ${s.columns.length} cols, ${s.measures.length} measures, ${s.joins.length} joins`);
+        }
+        return {
+            markdown: parts.join('\n'),
+            structured: {
+                success: valid,
+                sourceName: '(all)',
+                validationErrors: errors.length > 0 ? errors : undefined,
+                validationWarnings: warnings.length > 0 ? warnings : undefined,
+            },
+        };
+    }
+}

package/dist/context/sl/tools/sl-validate.tool.test.js

@@ -0,0 +1,54 @@
+import { describe, expect, it, vi } from 'vitest';
+import { createTouchedSlSources } from '../../../context/tools/touched-sl-sources.js';
+import { SlValidateTool, validateSemanticLayerEndpoint } from './sl-validate.tool.js';
+describe('validateSemanticLayerEndpoint', () => {
+    it('uses the connection warehouse dialect, not hardcoded postgres', async () => {
+        const serviceMock = {
+            validateSourcesForConnection: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
+        };
+        await validateSemanticLayerEndpoint('conn-1', serviceMock);
+        expect(serviceMock.validateSourcesForConnection).toHaveBeenCalledWith('conn-1');
+    });
+    it('short-circuits when there are no validatable sources', async () => {
+        const serviceMock = {
+            validateSourcesForConnection: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
+        };
+        const result = await validateSemanticLayerEndpoint('conn-1', serviceMock);
+        expect(result).toEqual({ errors: [], warnings: [] });
+    });
+});
+describe('SlValidateTool — session-aware touched-set filtering', () => {
+    it('when session present, only returns errors/warnings that mention touched sources', async () => {
+        const sources = [
+            { name: 'orders', table: 'x.orders', grain: ['id'], columns: [], joins: [], measures: [] },
+            { name: 'customers', table: 'x.customers', grain: ['id'], columns: [], joins: [], measures: [] },
+        ];
+        const serviceMock = {
+            loadAllSources: vi.fn().mockResolvedValue({ sources, loadErrors: [] }),
+            validateSourcesForConnection: vi.fn().mockResolvedValue({
+                errors: ['orders: missing join target', 'customers: invalid grain'],
+                warnings: ['orders: disconnected-components warning'],
+            }),
+        };
+        const tool = new SlValidateTool({
+            semanticLayerService: serviceMock,
+            slSearchService: {},
+            authorResolver: { resolve: vi.fn() },
+        });
+        const session = {
+            connectionId: 'conn-1',
+            isWorktreeScoped: true,
+            preHead: null,
+            touchedSlSources: createTouchedSlSources([{ connectionId: 'conn-1', sourceName: 'orders' }]),
+            actions: [],
+            semanticLayerService: serviceMock,
+            wikiService: {},
+            configService: {},
+            gitService: {},
+        };
+        const context = { sourceId: 's', messageId: 'm', userId: 'u', session };
+        const result = await tool.call({ connectionId: 'conn-1' }, context);
+        expect(result.structured.validationErrors).toEqual(['orders: missing join target']);
+        expect(result.structured.validationWarnings).toEqual(['orders: disconnected-components warning']);
+    });
+});

package/dist/context/sl/tools/sl-warehouse-validation.d.ts

@@ -0,0 +1,32 @@
+import type { GitService } from '../../../context/core/git.service.js';
+import type { KtxFileStorePort } from '../../../context/core/file-store.js';
+import type { SlConnectionCatalogPort, SlSourcesIndexPort } from '../ports.js';
+import { SemanticLayerService } from '../semantic-layer.service.js';
+export interface SlValidationDeps {
+    semanticLayerService: SemanticLayerService;
+    connections: SlConnectionCatalogPort;
+    configService: KtxFileStorePort;
+    gitService: GitService;
+    slSourcesRepository: SlSourcesIndexPort;
+    probeRowCount: number;
+}
+/** @internal */
+export interface SourceValidationResult {
+    errors: string[];
+    warnings: string[];
+}
+/**
+ * Validate one SL source end-to-end: YAML parse, Zod schema, duplicate-measure detection,
+ * warehouse dry-run (`SELECT * FROM (sql) LIMIT 1` — forces runtime policy enforcement).
+ *
+ * Returns errors and hint-style warnings. An empty errors array means the YAML is
+ * structurally valid AND the warehouse can execute a probe against its embedded sql.
+ */
+/** @internal */
+export declare function validateSingleSource(deps: SlValidationDeps, connectionId: string, sourceName: string): Promise<SourceValidationResult>;
+/**
+ * Restore `sourceName` to the content it had at `preHead`, or delete it if it didn't
+ * exist then. Used by sl_rollback (agent-driven) and the pre-squash revert gate
+ * (automatic). Returns a short human-readable description of what happened.
+ */
+export declare function revertSourceToPreHead(deps: SlValidationDeps, connectionId: string, preHead: string | null, sourceName: string): Promise<string>;

package/dist/context/sl/tools/sl-warehouse-validation.js

@@ -0,0 +1,246 @@
+import YAML from 'yaml';
+import { SYSTEM_GIT_AUTHOR } from '../../../context/tools/authors.js';
+import { sourceOverlaySchema } from '../schemas.js';
+import { SemanticLayerService } from '../semantic-layer.service.js';
+import { sourceDefinitionSchema } from './base-semantic-layer.tool.js';
+const slSourcePath = (connectionId, sourceName) => `semantic-layer/${connectionId}/${sourceName}.yaml`;
+function resolveDialect(warehouse) {
+    if (!warehouse) {
+        return null;
+    }
+    return SemanticLayerService.mapDialect(warehouse);
+}
+function wrapWithZeroRowQuery(sql, dialect) {
+    if (dialect === 'tsql') {
+        return `SELECT TOP 0 * FROM (${sql}) AS _discovery`;
+    }
+    return `SELECT * FROM (${sql}) AS _discovery LIMIT 0`;
+}
+function wrapWithSingleRowQuery(sql, dialect) {
+    if (dialect === 'tsql') {
+        return `SELECT TOP 1 * FROM (${sql}) AS _base`;
+    }
+    return `SELECT * FROM (${sql}) AS _base LIMIT 1`;
+}
+/**
+ * Validate one SL source end-to-end: YAML parse, Zod schema, duplicate-measure detection,
+ * warehouse dry-run (`SELECT * FROM (sql) LIMIT 1` — forces runtime policy enforcement).
+ *
+ * Returns errors and hint-style warnings. An empty errors array means the YAML is
+ * structurally valid AND the warehouse can execute a probe against its embedded sql.
+ */
+/** @internal */
+export async function validateSingleSource(deps, connectionId, sourceName) {
+    const errors = [];
+    const warnings = [];
+    let content;
+    try {
+        const result = await deps.semanticLayerService.readSourceFile(connectionId, sourceName);
+        content = result.content;
+    }
+    catch {
+        errors.push(`${sourceName}.yaml: file not found`);
+        return { errors, warnings };
+    }
+    let parsed;
+    try {
+        parsed = YAML.parse(content);
+    }
+    catch (e) {
+        errors.push(`${sourceName}.yaml: invalid YAML — ${e instanceof Error ? e.message : String(e)}`);
+        return { errors, warnings };
+    }
+    if (!parsed || typeof parsed !== 'object') {
+        errors.push(`${sourceName}.yaml: top-level content is not an object`);
+        return { errors, warnings };
+    }
+    const isOverlay = !parsed.table && !parsed.sql;
+    if (!isOverlay) {
+        const isManifestBacked = await deps.semanticLayerService.isManifestBacked(connectionId, sourceName);
+        if (isManifestBacked) {
+            errors.push(`${sourceName}.yaml: standalone source shadows an existing manifest entry — ` +
+                `writing it as-is drops the manifest's columns and joins. ` +
+                `Remove "sql:", "table:", "grain:", and base-table "columns:" and keep only ` +
+                `"name:" plus overlay fields such as "measures:", "segments:", "descriptions:", ` +
+                `"joins:", "column_overrides:", or computed-only "columns:" to write an overlay ` +
+                `that inherits the manifest schema. Call sl_read_source to inspect the existing source first.`);
+            return { errors, warnings };
+        }
+    }
+    const schema = isOverlay ? sourceOverlaySchema : sourceDefinitionSchema;
+    const result = schema.safeParse(parsed);
+    if (!result.success) {
+        const issues = result.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ');
+        errors.push(`${sourceName}.yaml: schema — ${issues}`);
+        const errorPaths = new Set(result.error.issues.map((i) => String(i.path[0])));
+        if (errorPaths.has('joins')) {
+            warnings.push(`${sourceName}.yaml: hint — join format: {to, on: 'local_col = TARGET.col', relationship: 'many_to_one|one_to_many|one_to_one'}`);
+        }
+        if (errorPaths.has('columns')) {
+            warnings.push(`${sourceName}.yaml: hint — overlay columns must be computed: {name, expr, type}. Use column_overrides for manifest column descriptions or metadata.`);
+        }
+        if (errorPaths.has('measures')) {
+            warnings.push(`${sourceName}.yaml: hint — measure format: {name, expr, description (optional), filter (optional)}`);
+        }
+        return { errors, warnings };
+    }
+    if (!isOverlay && 'table' in result.data && result.data.table) {
+        errors.push(...(await deps.semanticLayerService.validatePhysicalTableReferences(connectionId, [
+            result.data,
+        ])));
+    }
+    const measures = parsed.measures ?? [];
+    const seenMeasures = new Set();
+    for (const m of measures) {
+        if (seenMeasures.has(m.name)) {
+            errors.push(`${sourceName}.yaml: duplicate measure name "${m.name}"`);
+        }
+        seenMeasures.add(m.name);
+    }
+    let warehouse = null;
+    try {
+        const connection = await deps.connections.getConnectionById(connectionId);
+        warehouse = connection?.connectionType ?? null;
+    }
+    catch {
+        warehouse = null;
+    }
+    if (typeof parsed.sql === 'string' && parsed.sql.trim().length > 0) {
+        const innerSql = parsed.sql.trim().replace(/;+\s*$/, '');
+        const probeRowCount = deps.probeRowCount;
+        const dialect = resolveDialect(warehouse);
+        let probeSql;
+        if (dialect) {
+            probeSql =
+                probeRowCount === 0 ? wrapWithZeroRowQuery(innerSql, dialect) : wrapWithSingleRowQuery(innerSql, dialect);
+        }
+        else {
+            probeSql = `SELECT * FROM (${innerSql}) AS _probe LIMIT ${probeRowCount}`;
+        }
+        const sourceColumns = (parsed.columns ?? [])
+            .map((c) => ({ name: c.name ?? '', type: c.type ?? '' }))
+            .filter((c) => c.name);
+        try {
+            const probe = await deps.connections.executeQuery(connectionId, probeSql);
+            const actual = new Set((probe.headers ?? []).map((h) => h.toLowerCase()));
+            const missing = sourceColumns.map((c) => c.name).filter((n) => !actual.has(n.toLowerCase()));
+            if (missing.length > 0) {
+                errors.push(`${sourceName}.yaml: declared columns absent from sql result — ${missing.join(', ')} (warehouse returned: ${[...actual].slice(0, 10).join(', ')}${actual.size > 10 ? ', …' : ''})`);
+            }
+        }
+        catch (e) {
+            errors.push(formatProbeError({
+                sourceName,
+                measureName: null,
+                probeSql,
+                warehouse,
+                sourceColumns,
+                error: e,
+                headline: 'embedded sql dry-run failed',
+            }));
+        }
+    }
+    else if (isOverlay) {
+        const measureErrors = await probeOverlayMeasures(deps, connectionId, sourceName, warehouse);
+        errors.push(...measureErrors);
+    }
+    return { errors, warnings };
+}
+function formatProbeError(args) {
+    const { sourceName, measureName, probeSql, warehouse, sourceColumns, error, headline } = args;
+    const errMsg = error instanceof Error ? error.message : String(error);
+    const refColumns = sourceColumns.filter((c) => referencesColumn(probeSql, c.name));
+    const lines = [
+        measureName ? `${sourceName}.yaml: measure "${measureName}" ${headline}.` : `${sourceName}.yaml: ${headline}.`,
+    ];
+    if (warehouse) {
+        lines.push(`  Warehouse: ${warehouse}`);
+    }
+    lines.push(`  Probe SQL: ${probeSql}`);
+    if (refColumns.length > 0) {
+        lines.push(`  Referenced columns: ${refColumns.map((c) => `${c.name} (${c.type || '?'})`).join(', ')}`);
+    }
+    lines.push(`  Error: ${errMsg}`);
+    return lines.join('\n');
+}
+function referencesColumn(sql, columnName) {
+    if (!columnName) {
+        return false;
+    }
+    const escaped = columnName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    return new RegExp(`\\b${escaped}\\b`).test(sql);
+}
+async function probeOverlayMeasures(deps, connectionId, sourceName, warehouse) {
+    const errors = [];
+    let composed;
+    try {
+        const { sources: all, loadErrors } = await deps.semanticLayerService.loadAllSources(connectionId);
+        errors.push(...loadErrors);
+        composed = all.find((s) => s.name === sourceName);
+    }
+    catch (e) {
+        errors.push(`${sourceName}.yaml: failed to load composed source for probe — ${e instanceof Error ? e.message : String(e)}`);
+        return errors;
+    }
+    if (!composed?.table || composed.measures.length === 0) {
+        return errors;
+    }
+    const sourceColumns = (composed.columns ?? [])
+        .map((c) => ({ name: c.name ?? '', type: c.type ?? '' }))
+        .filter((c) => c.name);
+    for (const measure of composed.measures) {
+        const measureRef = `${sourceName}.${measure.name}`;
+        let probeSql = `<composed via semantic-layer engine for ${measureRef}>`;
+        try {
+            const result = await deps.semanticLayerService.executeQuery(connectionId, {
+                measures: [measureRef],
+                dimensions: [],
+                filters: [],
+                limit: 1,
+            });
+            probeSql = result.sql ?? probeSql;
+        }
+        catch (e) {
+            errors.push(formatProbeError({
+                sourceName,
+                measureName: measure.name,
+                probeSql,
+                warehouse,
+                sourceColumns,
+                error: e,
+                headline: 'dry-run failed',
+            }));
+        }
+    }
+    return errors;
+}
+/**
+ * Restore `sourceName` to the content it had at `preHead`, or delete it if it didn't
+ * exist then. Used by sl_rollback (agent-driven) and the pre-squash revert gate
+ * (automatic). Returns a short human-readable description of what happened.
+ */
+export async function revertSourceToPreHead(deps, connectionId, preHead, sourceName) {
+    const relPath = slSourcePath(connectionId, sourceName);
+    let preContent = null;
+    if (preHead) {
+        try {
+            preContent = await deps.gitService.getFileAtCommit(relPath, preHead);
+        }
+        catch {
+            preContent = null;
+        }
+    }
+    if (preContent !== null) {
+        await deps.configService.writeFile(relPath, preContent, SYSTEM_GIT_AUTHOR.name, SYSTEM_GIT_AUTHOR.email, `Revert SL source to pre-session state: ${sourceName}`, { skipLock: true });
+        return 'restored to pre-session content';
+    }
+    try {
+        await deps.configService.deleteFile(relPath, SYSTEM_GIT_AUTHOR.name, SYSTEM_GIT_AUTHOR.email, `Drop SL source (not present at session start): ${sourceName}`, { skipLock: true });
+        await deps.slSourcesRepository.deleteByConnectionAndName(connectionId, sourceName);
+        return 'deleted (did not exist at session start)';
+    }
+    catch {
+        await deps.slSourcesRepository.deleteByConnectionAndName(connectionId, sourceName);
+        return 'no-op (already absent)';
+    }
+}