@kaelio/ktx 0.13.1 → 0.15.0

package/dist/context/ingest/final-gate-prune.js

@@ -0,0 +1,229 @@
+import YAML from 'yaml';
+import { listSlSourceFiles, resolveSlSourceFile, slSourceNameForFile } from '../sl/source-files.js';
+async function resolveYamlSource(fileStore, connectionId, sourceName) {
+    const file = await resolveSlSourceFile(fileStore, connectionId, sourceName);
+    if (!file) {
+        return null;
+    }
+    const parsed = YAML.parse(file.content);
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        throw new Error(`${file.path}: expected semantic-layer source YAML object`);
+    }
+    return { path: file.path, source: parsed };
+}
+async function writeYamlSource(input) {
+    await input.fileStore.writeFile(input.path, YAML.stringify(input.source, { indent: 2, lineWidth: 0, version: '1.1' }), input.author.name, input.author.email, `Prune dangling joins from ${input.path}`, { skipLock: true });
+}
+function removeInlineToken(content, rawToken) {
+    return content.replaceAll(`\`${rawToken}\``, '').replace(/[ \t]+([.,;:!?])/g, '$1');
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function removeWikiRefToken(content, targetPageKey) {
+    const pattern = new RegExp(`\\[\\[\\s*${escapeRegExp(targetPageKey)}(?:\\|[^\\]\\n]+)?\\s*\\]\\]`, 'g');
+    return content.replace(pattern, '').replace(/[ \t]+([.,;:!?])/g, '$1');
+}
+function wikiBodyAbsentTarget(finding) {
+    if (finding.kind === 'missing_wiki_body_table') {
+        return finding.tableRef;
+    }
+    if (finding.kind === 'missing_wiki_body_sl_source') {
+        return finding.sourceName;
+    }
+    if (finding.kind === 'missing_wiki_body_sl_entity') {
+        return `${finding.sourceName}.${finding.entityName}`;
+    }
+    return '';
+}
+/** Remove every join whose target matches `shouldRemove`, write the file back, and
+ *  emit one pruned-reference record per distinct removed target. */
+async function pruneJoinsFromSource(input) {
+    if (!Array.isArray(input.resolved.source.joins)) {
+        return [];
+    }
+    const removed = new Set();
+    const nextJoins = input.resolved.source.joins.filter((entry) => {
+        const to = entry && typeof entry === 'object' && 'to' in entry ? entry.to : undefined;
+        if (typeof to === 'string' && input.shouldRemove(to)) {
+            removed.add(to);
+            return false;
+        }
+        return true;
+    });
+    if (removed.size === 0) {
+        return [];
+    }
+    input.resolved.source.joins = nextJoins;
+    await writeYamlSource({
+        fileStore: input.fileStore,
+        path: input.resolved.path,
+        source: input.resolved.source,
+        author: input.author,
+    });
+    const records = [];
+    for (const target of removed) {
+        const record = {
+            kind: 'join',
+            artifact: `semantic-layer/${input.connectionId}/${input.ownerSourceName}`,
+            removedRef: target,
+            absentTarget: target,
+        };
+        records.push(record);
+        await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
+    }
+    return records;
+}
+export async function pruneFinalGateFindings(input) {
+    const droppedSources = [...input.droppedSources];
+    const prunedReferences = [];
+    const droppedKey = new Set(droppedSources.map((source) => `${source.connectionId}:${source.sourceName}`));
+    for (const finding of input.findings) {
+        if (finding.kind !== 'invalid_source') {
+            continue;
+        }
+        const key = `${finding.connectionId}:${finding.sourceName}`;
+        if (droppedKey.has(key)) {
+            continue;
+        }
+        const file = await resolveSlSourceFile(input.semanticLayerFiles, finding.connectionId, finding.sourceName);
+        if (!file) {
+            continue;
+        }
+        const deleted = await input.semanticLayerFiles.deleteFile(file.path, input.author.name, input.author.email, `Drop invalid source ${finding.connectionId}:${finding.sourceName}`, { skipLock: true });
+        if (!deleted) {
+            continue;
+        }
+        const dropped = {
+            connectionId: finding.connectionId,
+            sourceName: finding.sourceName,
+            reason: finding.errors.join('; '),
+        };
+        droppedSources.push(dropped);
+        droppedKey.add(key);
+        await input.trace.event('info', 'final_gates', 'final_gate_source_dropped', dropped);
+    }
+    // A dropped node can leave a join dangling on any owner — including sources
+    // untouched by this run, which the touched-scoped gate (and the confirm gate
+    // after it) never revisit. Prune those edges directly (D5), or the committed
+    // orphan join breaks every SL query on the connection.
+    const droppedByConnection = new Map();
+    for (const dropped of droppedSources) {
+        const names = droppedByConnection.get(dropped.connectionId) ?? new Set();
+        names.add(dropped.sourceName);
+        droppedByConnection.set(dropped.connectionId, names);
+    }
+    for (const [connectionId, droppedNames] of droppedByConnection) {
+        for (const file of await listSlSourceFiles(input.semanticLayerFiles, connectionId)) {
+            let parsed;
+            try {
+                parsed = YAML.parse(file.content);
+            }
+            catch {
+                continue;
+            }
+            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+                continue;
+            }
+            prunedReferences.push(...(await pruneJoinsFromSource({
+                fileStore: input.semanticLayerFiles,
+                connectionId,
+                ownerSourceName: slSourceNameForFile(file.path, file.content),
+                resolved: { path: file.path, source: parsed },
+                shouldRemove: (target) => droppedNames.has(target),
+                author: input.author,
+                trace: input.trace,
+            })));
+        }
+    }
+    for (const finding of input.findings) {
+        if (finding.kind !== 'missing_join_target') {
+            continue;
+        }
+        const resolved = await resolveYamlSource(input.semanticLayerFiles, finding.ownerConnectionId, finding.ownerSourceName);
+        if (!resolved) {
+            continue;
+        }
+        prunedReferences.push(...(await pruneJoinsFromSource({
+            fileStore: input.semanticLayerFiles,
+            connectionId: finding.ownerConnectionId,
+            ownerSourceName: finding.ownerSourceName,
+            resolved,
+            shouldRemove: (target) => target === finding.targetSourceName,
+            author: input.author,
+            trace: input.trace,
+        })));
+    }
+    const wikiFindings = input.findings.filter((finding) => finding.kind === 'missing_wiki_ref' ||
+        finding.kind === 'missing_wiki_sl_ref' ||
+        finding.kind === 'missing_wiki_body_sl_source' ||
+        finding.kind === 'missing_wiki_body_sl_entity' ||
+        finding.kind === 'missing_wiki_body_table');
+    const pageKeys = [...new Set(wikiFindings.map((finding) => finding.pageKey))].sort();
+    for (const pageKey of pageKeys) {
+        const page = input.wikiService ? await input.wikiService.readPage('GLOBAL', null, pageKey) : null;
+        if (!page) {
+            continue;
+        }
+        const frontmatter = { ...page.frontmatter };
+        let content = page.content;
+        let changed = false;
+        for (const finding of wikiFindings.filter((candidate) => candidate.pageKey === pageKey)) {
+            if (finding.kind === 'missing_wiki_ref') {
+                const refs = Array.isArray(frontmatter.refs) ? frontmatter.refs.filter((ref) => ref !== finding.targetPageKey) : [];
+                const nextContent = removeWikiRefToken(content, finding.targetPageKey);
+                if ((Array.isArray(frontmatter.refs) && refs.length !== frontmatter.refs.length) || nextContent !== content) {
+                    if (Array.isArray(frontmatter.refs)) {
+                        frontmatter.refs = refs;
+                    }
+                    content = nextContent;
+                    changed = true;
+                    const record = {
+                        kind: 'wiki_ref',
+                        artifact: `wiki/global/${pageKey}`,
+                        removedRef: finding.targetPageKey,
+                        absentTarget: finding.targetPageKey,
+                    };
+                    prunedReferences.push(record);
+                    await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
+                }
+            }
+            else if (finding.kind === 'missing_wiki_sl_ref') {
+                const slRefs = Array.isArray(frontmatter.sl_refs)
+                    ? frontmatter.sl_refs.filter((ref) => ref !== finding.ref)
+                    : [];
+                if (Array.isArray(frontmatter.sl_refs) && slRefs.length !== frontmatter.sl_refs.length) {
+                    frontmatter.sl_refs = slRefs;
+                    changed = true;
+                    const record = {
+                        kind: 'wiki_sl_ref',
+                        artifact: `wiki/global/${pageKey}`,
+                        removedRef: finding.ref,
+                        absentTarget: finding.sourceName,
+                    };
+                    prunedReferences.push(record);
+                    await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
+                }
+            }
+            else {
+                const nextContent = removeInlineToken(content, finding.rawToken);
+                if (nextContent !== content) {
+                    content = nextContent;
+                    changed = true;
+                    const record = {
+                        kind: 'wiki_body_ref',
+                        artifact: `wiki/global/${pageKey}`,
+                        removedRef: finding.rawToken,
+                        absentTarget: wikiBodyAbsentTarget(finding),
+                    };
+                    prunedReferences.push(record);
+                    await input.trace.event('info', 'final_gates', 'final_gate_reference_pruned', record);
+                }
+            }
+        }
+        if (changed && input.wikiService) {
+            await input.wikiService.writePage('GLOBAL', null, pageKey, frontmatter, content, input.author.name, input.author.email, `Prune dangling refs from ${pageKey}`, { skipLock: true });
+        }
+    }
+    return { prunedReferences, droppedSources };
+}

package/dist/context/ingest/ingest-bundle.runner.d.ts

@@ -8,6 +8,8 @@ export declare class IngestBundleRunner {
     private readonly logger;
     private readonly chainByConnection;
     constructor(deps: IngestBundleRunnerDeps);
+    private cachedWorkUnitOutcome;
+    private saveSuccessfulWorkUnitCache;
     run(job: IngestBundleJob, ctx?: IngestJobContext): Promise<IngestBundleResult>;
     private formatRateLimitWait;
     private subscribeRateLimitGovernor;
@@ -61,6 +63,7 @@ export declare class IngestBundleRunner {
     private uniqueWikiPageKeys;
     private uniqueTouchedSlSources;
     private removedWikiPageKeysFromActions;
+    private markFinalGateDroppedSourceWorkUnits;
     private finalGateActionOrigins;
     private wikiPageKeysForFinalGates;
     private runWorkUnitInWorktree;