@kaelio/ktx 0.13.1 → 0.14.0

package/dist/context/ingest/artifact-gates.js

@@ -1,5 +1,8 @@
 import { findMissingWikiRefs } from '../wiki/wiki-ref-validation.js';
-import { findInvalidWikiBodyRefs } from './wiki-body-refs.js';
+import { findInvalidWikiBodyRefIssues } from './wiki-body-refs.js';
+function normalizeRawPath(path) {
+    return path.replace(/\\/g, '/').replace(/^\/+/, '');
+}
 function parseSlRef(ref) {
     const withoutConnection = ref.includes('/') ? ref.slice(ref.indexOf('/') + 1) : ref;
     const connectionId = ref.includes('/') ? ref.slice(0, ref.indexOf('/')) : null;
@@ -14,7 +17,7 @@ function slEntityNames(source) {
     ]);
 }
 async function validateWikiSlRefs(input) {
-    const errors = [];
+    const findings = [];
     const sourcesByConnection = new Map();
     for (const connectionId of input.connectionIds) {
         const { sources } = await input.semanticLayerService.loadAllSources(connectionId);
@@ -36,18 +39,32 @@ async function validateWikiSlRefs(input) {
                 }
             }
             if (!source) {
-                errors.push(`${pageKey}: unknown sl_refs entry ${ref}`);
+                findings.push({
+                    kind: 'missing_wiki_sl_ref',
+                    pageKey,
+                    ref,
+                    sourceName: parsed.sourceName,
+                    entityName: parsed.entityName,
+                    message: `${pageKey}: unknown sl_refs entry ${ref}`,
+                });
                 continue;
             }
             if (parsed.entityName && !slEntityNames(source).has(parsed.entityName)) {
-                errors.push(`${pageKey}: unknown sl_refs entity ${ref}`);
+                findings.push({
+                    kind: 'missing_wiki_sl_ref',
+                    pageKey,
+                    ref,
+                    sourceName: parsed.sourceName,
+                    entityName: parsed.entityName,
+                    message: `${pageKey}: unknown sl_refs entity ${ref}`,
+                });
             }
         }
     }
-    return errors;
+    return findings;
 }
 async function validateWikiRefs(input) {
-    const dangling = [];
+    const findings = [];
     for (const pageKey of input.changedWikiPageKeys) {
         const page = await input.wikiService.readPage('GLOBAL', null, pageKey);
         if (!page) {
@@ -62,28 +79,77 @@ async function validateWikiRefs(input) {
             content: page.content,
         });
         for (const missingRef of missingRefs) {
-            dangling.push(`${pageKey} -> ${missingRef}`);
+            findings.push({
+                kind: 'missing_wiki_ref',
+                pageKey,
+                targetPageKey: missingRef,
+                message: `${pageKey} -> ${missingRef}`,
+            });
+        }
+    }
+    return findings;
+}
+export function formatFinalArtifactGateFindings(findings) {
+    const errors = findings.map((finding) => {
+        if (finding.kind === 'invalid_source') {
+            return `semantic-layer validation failed for ${finding.connectionId}:${finding.sourceName}: ${finding.errors.join('; ')}`;
+        }
+        if (finding.kind === 'missing_wiki_ref') {
+            return `wiki reference targets missing page: ${finding.message}`;
+        }
+        return finding.message;
+    });
+    return `final artifact gates failed:\n${errors.join('\n')}`;
+}
+export function isFinalArtifactGateFindingPruneable(finding) {
+    switch (finding.kind) {
+        case 'invalid_source':
+        case 'missing_join_target':
+        case 'missing_wiki_ref':
+        case 'missing_wiki_sl_ref':
+        case 'missing_wiki_body_sl_entity':
+        case 'missing_wiki_body_sl_source':
+        case 'missing_wiki_body_table':
+            return true;
+        default: {
+            const exhaustive = finding;
+            return exhaustive;
         }
     }
-    return dangling;
 }
 export async function validateFinalIngestArtifacts(input) {
     // Join-neighbor expansion happens inside validateTouchedSources so work-unit
     // validation and this gate check the same set — a source that passes one
     // passes the other.
     const validation = await input.validateTouchedSources(input.touchedSlSources);
-    const errors = validation.invalidSources.map((invalid) => `semantic-layer validation failed for ${invalid.source}: ${invalid.errors.join('; ')}`);
-    errors.push(...(await validateWikiSlRefs(input)));
-    const danglingWikiRefs = await validateWikiRefs(input);
-    if (danglingWikiRefs.length > 0) {
-        errors.push(`wiki references target missing page(s): ${danglingWikiRefs.join(', ')}`);
+    const findings = [];
+    for (const invalid of validation.invalidSources) {
+        const [connectionId = '', sourceName = ''] = invalid.source.split(':', 2);
+        const issues = invalid.issues ?? invalid.errors.map((message) => ({ kind: 'source_validation', message }));
+        const sourceErrors = issues.filter((issue) => issue.kind === 'source_validation').map((issue) => issue.message);
+        if (sourceErrors.length > 0) {
+            findings.push({ kind: 'invalid_source', connectionId, sourceName, errors: sourceErrors });
+        }
+        for (const issue of issues) {
+            if (issue.kind === 'missing_join_target') {
+                findings.push({
+                    kind: 'missing_join_target',
+                    ownerConnectionId: connectionId,
+                    ownerSourceName: sourceName,
+                    targetSourceName: issue.targetSourceName,
+                    message: issue.message,
+                });
+            }
+        }
     }
+    findings.push(...(await validateWikiSlRefs(input)));
+    findings.push(...(await validateWikiRefs(input)));
     for (const pageKey of input.changedWikiPageKeys) {
         const page = await input.wikiService.readPage('GLOBAL', null, pageKey);
         if (!page) {
             continue;
         }
-        errors.push(...(await findInvalidWikiBodyRefs({
+        findings.push(...(await findInvalidWikiBodyRefIssues({
             pageKey,
             body: page.content,
             visibleConnectionIds: input.connectionIds,
@@ -94,13 +160,14 @@ export async function validateFinalIngestArtifacts(input) {
             tableExists: input.tableExists,
         })));
     }
-    if (errors.length > 0) {
-        throw new Error(`final artifact gates failed:\n${errors.join('\n')}`);
-    }
+    return { ok: findings.length === 0, findings };
 }
 export function validateProvenanceRawPaths(input) {
+    const currentRawPaths = new Set([...input.currentRawPaths].map(normalizeRawPath));
+    const deletedRawPaths = new Set([...input.deletedRawPaths].map(normalizeRawPath));
     for (const row of input.rows) {
-        if (!input.currentRawPaths.has(row.rawPath) && !input.deletedRawPaths.has(row.rawPath)) {
+        const rawPath = normalizeRawPath(row.rawPath);
+        if (!currentRawPaths.has(rawPath) && !deletedRawPaths.has(rawPath)) {
             throw new Error(`provenance row references raw path outside this snapshot: ${row.rawPath}`);
         }
     }

package/dist/context/ingest/final-gate-prune.d.ts

@@ -0,0 +1,35 @@
+import type { KtxFileStorePort } from '../core/file-store.js';
+import type { KnowledgeWikiService } from '../wiki/knowledge-wiki.service.js';
+import type { FinalArtifactGateFinding } from './artifact-gates.js';
+import type { IngestTraceWriter } from './ingest-trace.js';
+type FinalGatePrunedReferenceKind = 'join' | 'wiki_ref' | 'wiki_sl_ref' | 'wiki_body_ref';
+type SemanticLayerFileStore = Pick<KtxFileStorePort, 'readFile' | 'writeFile' | 'deleteFile' | 'listFiles'>;
+export interface FinalGatePrunedReference {
+    kind: FinalGatePrunedReferenceKind;
+    artifact: string;
+    removedRef: string;
+    absentTarget: string;
+}
+export interface FinalGateDroppedSource {
+    connectionId: string;
+    sourceName: string;
+    reason: string;
+}
+export interface FinalGatePruneResult {
+    prunedReferences: FinalGatePrunedReference[];
+    droppedSources: FinalGateDroppedSource[];
+}
+interface PruneInput {
+    workdir: string;
+    semanticLayerFiles: SemanticLayerFileStore;
+    findings: FinalArtifactGateFinding[];
+    droppedSources: FinalGateDroppedSource[];
+    trace: IngestTraceWriter;
+    author: {
+        name: string;
+        email: string;
+    };
+    wikiService?: KnowledgeWikiService;
+}
+export declare function pruneFinalGateFindings(input: PruneInput): Promise<FinalGatePruneResult>;
+export {};

package/dist/context/ingest/final-gate-prune.js

@@ -0,0 +1,229 @@
+import YAML from 'yaml';
+import { listSlSourceFiles, resolveSlSourceFile, slSourceNameForFile } from '../sl/source-files.js';
+async function resolveYamlSource(fileStore, connectionId, sourceName) {
+    const file = await resolveSlSourceFile(fileStore, connectionId, sourceName);
+    if (!file) {
+        return null;
+    }
+    const parsed = YAML.parse(file.content);
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        throw new Error(`${file.path}: expected semantic-layer source YAML object`);
+    }
+    return { path: file.path, source: parsed };
+}
+async function writeYamlSource(input) {
+    await input.fileStore.writeFile(input.path, YAML.stringify(input.source, { indent: 2, lineWidth: 0, version: '1.1' }), input.author.name, input.author.email, `Prune dangling joins from ${input.path}`, { skipLock: true });
+}
+function removeInlineToken(content, rawToken) {
+    return content.replaceAll(`\`${rawToken}\``, '').replace(/[ \t]+([.,;:!?])/g, '$1');
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function removeWikiRefToken(content, targetPageKey) {
+    const pattern = new RegExp(`\\[\\[\\s*${escapeRegExp(targetPageKey)}(?:\\|[^\\]\\n]+)?\\s*\\]\\]`, 'g');
+    return content.replace(pattern, '').replace(/[ \t]+([.,;:!?])/g, '$1');
+}
+function wikiBodyAbsentTarget(finding) {
+    if (finding.kind === 'missing_wiki_body_table') {
+        return finding.tableRef;
+    }
+    if (finding.kind === 'missing_wiki_body_sl_source') {
+        return finding.sourceName;
+    }
+    if (finding.kind === 'missing_wiki_body_sl_entity') {
+        return `${finding.sourceName}.${finding.entityName}`;
+    }
+    return '';
+}
+/** Remove every join whose target matches `shouldRemove`, write the file back, and
+ *  emit one pruned-reference record per distinct removed target. */
+async function pruneJoinsFromSource(input) {
+    if (!Array.isArray(input.resolved.source.joins)) {
+        return [];
+    }
+    const removed = new Set();
+    const nextJoins = input.resolved.source.joins.filter((entry) => {
+        const to = entry && typeof entry === 'object' && 'to' in entry ? entry.to : undefined;
+        if (typeof to === 'string' && input.shouldRemove(to)) {
+            removed.add(to);
+            return false;
+        }
+        return true;
+    });
+    if (removed.size === 0) {
+        return [];
+    }
+    input.resolved.source.joins = nextJoins;
+    await writeYamlSource({
+        fileStore: input.fileStore,
+        path: input.resolved.path,
+        source: input.resolved.source,
+        author: input.author,
+    });
+    const records = [];
+    for (const target of removed) {
+        const record = {
+            kind: 'join',
+            artifact: `semantic-layer/${input.connectionId}/${input.ownerSourceName}`,
+            removedRef: target,
+            absentTarget: target,
+        };
+        records.push(record);
+        await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
+    }
+    return records;
+}
+export async function pruneFinalGateFindings(input) {
+    const droppedSources = [...input.droppedSources];
+    const prunedReferences = [];
+    const droppedKey = new Set(droppedSources.map((source) => `${source.connectionId}:${source.sourceName}`));
+    for (const finding of input.findings) {
+        if (finding.kind !== 'invalid_source') {
+            continue;
+        }
+        const key = `${finding.connectionId}:${finding.sourceName}`;
+        if (droppedKey.has(key)) {
+            continue;
+        }
+        const file = await resolveSlSourceFile(input.semanticLayerFiles, finding.connectionId, finding.sourceName);
+        if (!file) {
+            continue;
+        }
+        const deleted = await input.semanticLayerFiles.deleteFile(file.path, input.author.name, input.author.email, `Drop invalid source ${finding.connectionId}:${finding.sourceName}`, { skipLock: true });
+        if (!deleted) {
+            continue;
+        }
+        const dropped = {
+            connectionId: finding.connectionId,
+            sourceName: finding.sourceName,
+            reason: finding.errors.join('; '),
+        };
+        droppedSources.push(dropped);
+        droppedKey.add(key);
+        await input.trace.event('info', 'final_gates', 'final_gate_source_dropped', dropped);
+    }
+    // A dropped node can leave a join dangling on any owner — including sources
+    // untouched by this run, which the touched-scoped gate (and the confirm gate
+    // after it) never revisit. Prune those edges directly (D5), or the committed
+    // orphan join breaks every SL query on the connection.
+    const droppedByConnection = new Map();
+    for (const dropped of droppedSources) {
+        const names = droppedByConnection.get(dropped.connectionId) ?? new Set();
+        names.add(dropped.sourceName);
+        droppedByConnection.set(dropped.connectionId, names);
+    }
+    for (const [connectionId, droppedNames] of droppedByConnection) {
+        for (const file of await listSlSourceFiles(input.semanticLayerFiles, connectionId)) {
+            let parsed;
+            try {
+                parsed = YAML.parse(file.content);
+            }
+            catch {
+                continue;
+            }
+            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+                continue;
+            }
+            prunedReferences.push(...(await pruneJoinsFromSource({
+                fileStore: input.semanticLayerFiles,
+                connectionId,
+                ownerSourceName: slSourceNameForFile(file.path, file.content),
+                resolved: { path: file.path, source: parsed },
+                shouldRemove: (target) => droppedNames.has(target),
+                author: input.author,
+                trace: input.trace,
+            })));
+        }
+    }
+    for (const finding of input.findings) {
+        if (finding.kind !== 'missing_join_target') {
+            continue;
+        }
+        const resolved = await resolveYamlSource(input.semanticLayerFiles, finding.ownerConnectionId, finding.ownerSourceName);
+        if (!resolved) {
+            continue;
+        }
+        prunedReferences.push(...(await pruneJoinsFromSource({
+            fileStore: input.semanticLayerFiles,
+            connectionId: finding.ownerConnectionId,
+            ownerSourceName: finding.ownerSourceName,
+            resolved,
+            shouldRemove: (target) => target === finding.targetSourceName,
+            author: input.author,
+            trace: input.trace,
+        })));
+    }
+    const wikiFindings = input.findings.filter((finding) => finding.kind === 'missing_wiki_ref' ||
+        finding.kind === 'missing_wiki_sl_ref' ||
+        finding.kind === 'missing_wiki_body_sl_source' ||
+        finding.kind === 'missing_wiki_body_sl_entity' ||
+        finding.kind === 'missing_wiki_body_table');
+    const pageKeys = [...new Set(wikiFindings.map((finding) => finding.pageKey))].sort();
+    for (const pageKey of pageKeys) {
+        const page = input.wikiService ? await input.wikiService.readPage('GLOBAL', null, pageKey) : null;
+        if (!page) {
+            continue;
+        }
+        const frontmatter = { ...page.frontmatter };
+        let content = page.content;
+        let changed = false;
+        for (const finding of wikiFindings.filter((candidate) => candidate.pageKey === pageKey)) {
+            if (finding.kind === 'missing_wiki_ref') {
+                const refs = Array.isArray(frontmatter.refs) ? frontmatter.refs.filter((ref) => ref !== finding.targetPageKey) : [];
+                const nextContent = removeWikiRefToken(content, finding.targetPageKey);
+                if ((Array.isArray(frontmatter.refs) && refs.length !== frontmatter.refs.length) || nextContent !== content) {
+                    if (Array.isArray(frontmatter.refs)) {
+                        frontmatter.refs = refs;
+                    }
+                    content = nextContent;
+                    changed = true;
+                    const record = {
+                        kind: 'wiki_ref',
+                        artifact: `wiki/global/${pageKey}`,
+                        removedRef: finding.targetPageKey,
+                        absentTarget: finding.targetPageKey,
+                    };
+                    prunedReferences.push(record);
+                    await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
+                }
+            }
+            else if (finding.kind === 'missing_wiki_sl_ref') {
+                const slRefs = Array.isArray(frontmatter.sl_refs)
+                    ? frontmatter.sl_refs.filter((ref) => ref !== finding.ref)
+                    : [];
+                if (Array.isArray(frontmatter.sl_refs) && slRefs.length !== frontmatter.sl_refs.length) {
+                    frontmatter.sl_refs = slRefs;
+                    changed = true;
+                    const record = {
+                        kind: 'wiki_sl_ref',
+                        artifact: `wiki/global/${pageKey}`,
+                        removedRef: finding.ref,
+                        absentTarget: finding.sourceName,
+                    };
+                    prunedReferences.push(record);
+                    await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
+                }
+            }
+            else {
+                const nextContent = removeInlineToken(content, finding.rawToken);
+                if (nextContent !== content) {
+                    content = nextContent;
+                    changed = true;
+                    const record = {
+                        kind: 'wiki_body_ref',
+                        artifact: `wiki/global/${pageKey}`,
+                        removedRef: finding.rawToken,
+                        absentTarget: wikiBodyAbsentTarget(finding),
+                    };
+                    prunedReferences.push(record);
+                    await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
+                }
+            }
+        }
+        if (changed && input.wikiService) {
+            await input.wikiService.writePage('GLOBAL', null, pageKey, frontmatter, content, input.author.name, input.author.email, `Prune dangling refs from ${pageKey}`, { skipLock: true });
+        }
+    }
+    return { prunedReferences, droppedSources };
+}

package/dist/context/ingest/ingest-bundle.runner.d.ts

@@ -8,6 +8,8 @@ export declare class IngestBundleRunner {
     private readonly logger;
     private readonly chainByConnection;
     constructor(deps: IngestBundleRunnerDeps);
+    private cachedWorkUnitOutcome;
+    private saveSuccessfulWorkUnitCache;
     run(job: IngestBundleJob, ctx?: IngestJobContext): Promise<IngestBundleResult>;
     private formatRateLimitWait;
     private subscribeRateLimitGovernor;
@@ -61,6 +63,7 @@ export declare class IngestBundleRunner {
     private uniqueWikiPageKeys;
     private uniqueTouchedSlSources;
     private removedWikiPageKeysFromActions;
+    private markFinalGateDroppedSourceWorkUnits;
     private finalGateActionOrigins;
     private wikiPageKeysForFinalGates;
     private runWorkUnitInWorktree;