@kaelio/ktx 0.1.0-rc.6 → 0.1.0

package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js

@@ -11,14 +11,13 @@ import { NOTION_DEFAULT_MAX_KNOWLEDGE_CREATES_PER_RUN } from './adapters/notion/
 import { validateFinalIngestArtifacts, validateProvenanceRawPaths } from './artifact-gates.js';
 import { selectRelevantCanonicalPins } from './canonical-pins.js';
 import { finalGateRepairPaths, repairFinalGateFailure } from './final-gate-repair.js';
-import { compareFinalizationDeclarations, deriveFinalizationTouchedSources, deriveFinalizationWikiPageKeys, } from './finalization-scope.js';
 import { FileIngestTraceWriter, ingestTracePathForJob, traceTimed } from './ingest-trace.js';
 import { integrateWorkUnitPatch } from './isolated-diff/patch-integrator.js';
 import { resolveTextualConflict } from './isolated-diff/textual-conflict-resolver.js';
 import { runIsolatedWorkUnit } from './isolated-diff/work-unit-executor.js';
 import { sanitizeMemoryFlowError } from './memory-flow/live-buffer.js';
 import { buildSyncId, rawSourcesDirForSync } from './raw-sources-paths.js';
-import { buildStageIndexFromReportBody, } from './reports.js';
+import { buildStageIndexFromReportBody, postProcessorSavedMemoryCounts, } from './reports.js';
 import { buildReconcileSystemPrompt, buildReconcileToolSet, buildReconcileUserPrompt, } from './stages/build-reconcile-context.js';
 import { buildWuSystemPrompt, buildWuToolSet, buildWuUserPrompt } from './stages/build-wu-context.js';
 import { stageRawFilesStage1 } from './stages/stage-1-stage-raw-files.js';
@@ -287,15 +286,6 @@ export class IngestBundleRunner {
         }
         return false;
     }
-    async loadSourcesByConnection(workdir, connectionIds) {
-        const service = this.deps.semanticLayerService.forWorktree(workdir);
-        const result = new Map();
-        for (const connectionId of connectionIds) {
-            const { sources } = await service.loadAllSources(connectionId);
-            result.set(connectionId, sources);
-        }
-        return result;
-    }
     resolveContextCuratorBudget(bundleRef, stageIndex) {
         const rawConfig = bundleRef.kind === 'scheduled_pull' && bundleRef.config && typeof bundleRef.config === 'object'
             ? bundleRef.config
@@ -384,15 +374,6 @@ export class IngestBundleRunner {
                 });
             }
         });
-        input.finalizationActions.forEach((action, actionIndex) => {
-            for (const rawPath of action.rawPaths ?? []) {
-                pushActionProvenance(rawPath, action, {
-                    source: 'finalization_action',
-                    actionIndex,
-                    action,
-                });
-            }
-        });
         (input.stageIndex.artifactResolutions ?? []).forEach((resolution, resolutionIndex) => {
             const hash = input.currentHashes.get(resolution.rawPath) ?? '';
             pushRow({
@@ -431,29 +412,6 @@ export class IngestBundleRunner {
         }
         return { rows, diagnostics };
     }
-    partitionFinalizationActionsForProvenance(input) {
-        const defensible = new Set([
-            ...input.currentRawPaths,
-            ...input.currentEvictionRawPaths,
-            ...input.overrideEvictionRawPaths,
-        ]);
-        const actions = [];
-        const exclusions = [];
-        for (const action of input.actions) {
-            const rawPaths = action.rawPaths ?? [];
-            if (rawPaths.length === 0) {
-                exclusions.push({ action, reason: 'missing_raw_paths' });
-                continue;
-            }
-            const invalidRawPaths = rawPaths.filter((rawPath) => !defensible.has(rawPath)).sort();
-            if (invalidRawPaths.length > 0) {
-                exclusions.push({ action, reason: 'raw_path_not_defensible', invalidRawPaths });
-                continue;
-            }
-            actions.push(action);
-        }
-        return { actions, exclusions };
-    }
     toReportProvenanceRows(rows) {
         return rows.map(({ rawPath, artifactKind, artifactKey, actionType, targetConnectionId }) => ({
             rawPath,
@@ -739,7 +697,6 @@ export class IngestBundleRunner {
         let latestEvictionInputs = [];
         let latestUnresolvedCards = [];
         let latestReportProvenanceRows = [];
-        let latestFinalizationOutcome;
         let activeFailureDetails;
         let latestIsolatedDiffSummary;
         await trace.event('info', 'run', 'ingest_started', {
@@ -891,7 +848,7 @@ export class IngestBundleRunner {
                 let unresolvedCards;
                 let sourceContextReport;
                 let parseArtifacts;
-                let finalizationOutcome;
+                let postProcessorOutcome;
                 let wikiSlRefRepairResult = null;
                 let reconcileNotes = [];
                 let triageResult = null;
@@ -1534,187 +1491,55 @@ export class IngestBundleRunner {
                     artifactResolutionCount: stageIndex.artifactResolutions?.length ?? 0,
                 });
                 await stage4?.updateProgress(1.0, reconcileOutcome.skipped ? 'No reconciliation needed' : 'Reconciled');
-                const preFinalizationSha = await sessionWorktree.git.revParseHead();
-                const preFinalizationSourcesByConnection = await this.loadSourcesByConnection(sessionWorktree.workdir, slConnectionIds);
-                let finalizationActions = [];
-                let finalizationTouchedPaths = [];
-                let finalizationTouchedSources = [];
-                let finalizationChangedWikiPageKeys = [];
-                let finalizationSha = null;
-                activePhase = 'finalization';
-                if (adapter.finalize) {
-                    const stageFinalization = ctx?.startPhase(0.04);
-                    emitStageProgress('finalization', 87, 'Running deterministic finalization');
-                    await stageFinalization?.updateProgress(0.0, 'Running deterministic finalization');
-                    await runTrace.event('debug', 'finalization', 'finalization_started', { sourceKey: job.sourceKey });
-                    const result = await adapter.finalize({
-                        connectionId: job.connectionId,
-                        sourceKey: job.sourceKey,
-                        syncId,
-                        jobId: job.jobId,
-                        runId: createdRunRow.id,
-                        stagedDir,
-                        workdir: sessionWorktree.workdir,
-                        ...(overrideReport ? {} : { parseArtifacts }),
-                        stageIndex,
-                        workUnitOutcomes,
-                        reconciliationActions: reconcileActions,
-                        ...(overrideReport
-                            ? {
-                                overrideReplay: {
-                                    priorJobId: overrideReport.jobId,
-                                    priorRunId: overrideReport.runId,
-                                    priorSyncId: overrideReport.body.syncId,
-                                    evictionRawPaths: overrideReport.body.evictionInputs,
-                                },
-                            }
-                            : {}),
-                    });
-                    if (result.errors.length > 0) {
-                        finalizationOutcome = {
+                const postProcessor = this.deps.postProcessors?.[job.sourceKey];
+                activePhase = 'post_processor';
+                if (postProcessor) {
+                    const stagePostProcessor = ctx?.startPhase(0.04);
+                    emitStageProgress('post_processor', 87, 'Running deterministic imports');
+                    await stagePostProcessor?.updateProgress(0.0, 'Running deterministic imports');
+                    try {
+                        const result = await traceTimed(runTrace, 'post_processor', 'post_processor', { sourceKey: job.sourceKey }, () => postProcessor.run({
+                            connectionId: job.connectionId,
                             sourceKey: job.sourceKey,
-                            status: 'failed',
-                            commitSha: null,
-                            touchedPaths: [],
-                            declaredTouchedSources: result.touchedSources,
-                            derivedTouchedSources: [],
-                            declaredChangedWikiPageKeys: result.changedWikiPageKeys,
-                            derivedChangedWikiPageKeys: [],
-                            mismatches: [],
+                            syncId,
+                            jobId: job.jobId,
+                            runId: createdRunRow.id,
+                            workdir: sessionWorktree.workdir,
+                            parseArtifacts,
+                        }));
+                        postProcessorOutcome = {
+                            sourceKey: job.sourceKey,
+                            status: result.errors.length > 0 && result.touchedSources.length === 0 ? 'failed' : 'success',
                             result: result.result,
                             errors: result.errors,
                             warnings: result.warnings,
-                            actions: result.actions ?? [],
-                            provenanceExclusions: [],
+                            touchedSources: result.touchedSources,
                         };
-                        latestFinalizationOutcome = finalizationOutcome;
-                        await runTrace.event('error', 'finalization', 'finalization_failed', {
-                            sourceKey: job.sourceKey,
-                            errors: result.errors,
-                            warnings: result.warnings,
-                        });
-                        throw new Error(`deterministic finalization failed: ${result.errors.join('; ')}`);
+                        emitStageProgress('post_processor', 88, 'Deterministic imports complete');
+                        await stagePostProcessor?.updateProgress(1.0, 'Deterministic imports complete');
                     }
-                    const changedBeforeFinalization = new Set([
-                        ...projectionTouchedPaths,
-                        ...workUnitOutcomes.flatMap((outcome) => outcome.patchTouchedPaths ?? []),
-                        ...(preReconciliationSha && preFinalizationSha !== preReconciliationSha
-                            ? (await sessionWorktree.git.diffNameStatus(preReconciliationSha, preFinalizationSha)).map((entry) => entry.path)
-                            : []),
-                    ]);
-                    finalizationTouchedPaths = await sessionWorktree.git.changedPaths();
-                    const overlapping = finalizationTouchedPaths.filter((path) => changedBeforeFinalization.has(path));
-                    if (overlapping.length > 0) {
-                        await runTrace.event('error', 'finalization', 'finalization_failed', {
-                            sourceKey: job.sourceKey,
-                            reason: 'path_overlap',
-                            overlappingPaths: overlapping.sort(),
-                        });
-                        throw new Error(`finalization modified path(s) already changed earlier in this run: ${overlapping.sort().join(', ')}`);
-                    }
-                    const finalizationCommit = finalizationTouchedPaths.length > 0
-                        ? await sessionWorktree.git.commitFiles(finalizationTouchedPaths, `ingest(${job.sourceKey}): deterministic finalization syncId=${syncId}`, this.deps.storage.systemGitAuthor.name, this.deps.storage.systemGitAuthor.email)
-                        : await sessionWorktree.git.commitStaged(`ingest(${job.sourceKey}): deterministic finalization syncId=${syncId}`, this.deps.storage.systemGitAuthor.name, this.deps.storage.systemGitAuthor.email);
-                    finalizationSha = finalizationCommit.created ? finalizationCommit.commitHash : null;
-                    const postFinalizationSha = await sessionWorktree.git.revParseHead();
-                    finalizationTouchedPaths =
-                        preFinalizationSha !== postFinalizationSha
-                            ? (await sessionWorktree.git.diffNameStatus(preFinalizationSha, postFinalizationSha)).map((entry) => entry.path)
-                            : [];
-                    const changedConnectionIds = [
-                        ...new Set([
-                            ...slConnectionIds,
-                            ...finalizationTouchedPaths
-                                .filter((path) => path.startsWith('semantic-layer/'))
-                                .map((path) => path.split('/')[1])
-                                .filter((connectionId) => Boolean(connectionId)),
-                        ]),
-                    ].sort();
-                    const postFinalizationSourcesByConnection = await this.loadSourcesByConnection(sessionWorktree.workdir, changedConnectionIds);
-                    const scope = await deriveFinalizationTouchedSources({
-                        changedPaths: finalizationTouchedPaths,
-                        beforeSourcesByConnection: preFinalizationSourcesByConnection,
-                        afterSourcesByConnection: postFinalizationSourcesByConnection,
-                    });
-                    if (scope.unresolvedPaths.length > 0) {
-                        await runTrace.event('error', 'finalization', 'finalization_failed', {
-                            sourceKey: job.sourceKey,
-                            reason: 'unresolved_semantic_layer_paths',
-                            unresolvedPaths: scope.unresolvedPaths,
-                        });
-                        throw new Error(`could not resolve finalization semantic-layer path(s): ${scope.unresolvedPaths.join(', ')}`);
-                    }
-                    finalizationTouchedSources = scope.touchedSources;
-                    finalizationChangedWikiPageKeys = deriveFinalizationWikiPageKeys(finalizationTouchedPaths);
-                    const mismatches = compareFinalizationDeclarations({
-                        declaredTouchedSources: result.touchedSources,
-                        derivedTouchedSources: finalizationTouchedSources,
-                        declaredChangedWikiPageKeys: result.changedWikiPageKeys,
-                        derivedChangedWikiPageKeys: finalizationChangedWikiPageKeys,
-                    });
-                    if (mismatches.length > 0) {
-                        finalizationOutcome = {
+                    catch (error) {
+                        postProcessorOutcome = {
                             sourceKey: job.sourceKey,
                             status: 'failed',
-                            commitSha: finalizationSha,
-                            touchedPaths: finalizationTouchedPaths,
-                            declaredTouchedSources: result.touchedSources,
-                            derivedTouchedSources: finalizationTouchedSources,
-                            declaredChangedWikiPageKeys: result.changedWikiPageKeys,
-                            derivedChangedWikiPageKeys: finalizationChangedWikiPageKeys,
-                            mismatches,
-                            result: result.result,
-                            errors: ['finalization touched artifact declaration mismatch'],
-                            warnings: result.warnings,
-                            actions: result.actions ?? [],
-                            provenanceExclusions: [],
+                            errors: [error instanceof Error ? error.message : String(error)],
+                            warnings: [],
+                            touchedSources: [],
                         };
-                        latestFinalizationOutcome = finalizationOutcome;
-                        await runTrace.event('error', 'finalization', 'finalization_failed', {
-                            sourceKey: job.sourceKey,
-                            reason: 'declaration_mismatch',
-                            mismatches,
-                        });
-                        throw new Error(`finalization touched artifact declaration mismatch: ${mismatches
-                            .map((mismatch) => `${mismatch.direction}:${mismatch.artifactKind}:${mismatch.key}`)
-                            .join(', ')}`);
+                        await this.deps.runs.markFailed(runRow.id);
+                        throw error;
                     }
-                    finalizationActions = result.actions ?? [];
-                    finalizationOutcome = {
-                        sourceKey: job.sourceKey,
-                        status: 'success',
-                        commitSha: finalizationSha,
-                        touchedPaths: finalizationTouchedPaths,
-                        declaredTouchedSources: result.touchedSources,
-                        derivedTouchedSources: finalizationTouchedSources,
-                        declaredChangedWikiPageKeys: result.changedWikiPageKeys,
-                        derivedChangedWikiPageKeys: finalizationChangedWikiPageKeys,
-                        mismatches,
-                        result: result.result,
-                        errors: [],
-                        warnings: result.warnings,
-                        actions: finalizationActions,
-                        provenanceExclusions: [],
-                    };
-                    latestFinalizationOutcome = finalizationOutcome;
-                    emitStageProgress('finalization', 88, 'Deterministic finalization complete');
-                    await stageFinalization?.updateProgress(1.0, 'Deterministic finalization complete');
-                    await runTrace.event('debug', 'finalization', 'finalization_committed', {
-                        sourceKey: job.sourceKey,
-                        commitSha: finalizationSha,
-                        touchedPaths: finalizationTouchedPaths,
-                        touchedSources: finalizationTouchedSources,
-                        changedWikiPageKeys: finalizationChangedWikiPageKeys,
-                        warnings: result.warnings,
-                    });
-                }
-                else {
-                    await runTrace.event('debug', 'finalization', 'finalization_skipped', { sourceKey: job.sourceKey });
                 }
+                await runTrace.event('debug', 'post_processor', 'post_processor_finished', {
+                    sourceKey: job.sourceKey,
+                    status: postProcessorOutcome?.status ?? 'skipped',
+                    touchedSources: postProcessorOutcome?.touchedSources ?? [],
+                    warnings: postProcessorOutcome?.warnings ?? [],
+                });
                 const repairConnectionIds = [
                     ...new Set([
                         ...slConnectionIds,
-                        ...finalizationTouchedSources.map((source) => source.connectionId),
+                        ...(postProcessorOutcome?.touchedSources ?? []).map((source) => source.connectionId),
                     ]),
                 ].sort();
                 activePhase = 'wiki_sl_ref_repair';
@@ -1741,7 +1566,6 @@ export class IngestBundleRunner {
                         .flatMap((outcome) => outcome.patchTouchedPaths ?? [])
                         .flatMap((path) => this.wikiPageKeysFromPaths([path])),
                     ...this.wikiPageKeysFromActions(reconcileActions),
-                    ...finalizationChangedWikiPageKeys,
                     ...postReconciliationPaths.flatMap((path) => this.wikiPageKeysFromPaths([path])),
                     ...wikiSlRefRepairResult.repairs.filter((repair) => repair.scope === 'GLOBAL').map((repair) => repair.pageKey),
                 ]);
@@ -1750,7 +1574,7 @@ export class IngestBundleRunner {
                     ...workUnitOutcomes.flatMap((outcome) => outcome.touchedSlSources),
                     ...this.touchedSlSourcesFromActions(reconcileActions, job.connectionId),
                     ...this.touchedSlSourcesFromPaths(postReconciliationPaths),
-                    ...finalizationTouchedSources,
+                    ...(postProcessorOutcome?.touchedSources ?? []),
                 ]);
                 const finalWikiGateScope = await this.wikiPageKeysForFinalGates({
                     wikiService: this.deps.wikiService.forWorktree(sessionWorktree.workdir),
@@ -1763,7 +1587,7 @@ export class IngestBundleRunner {
                     ...projectionTouchedPaths,
                     ...workUnitOutcomes.flatMap((outcome) => outcome.patchTouchedPaths ?? []),
                     ...postReconciliationPaths,
-                    ...finalizationTouchedPaths,
+                    ...(postProcessorOutcome?.touchedSources ?? []).map((source) => `semantic-layer/${source.connectionId}/${source.sourceName}.yaml`),
                 ];
                 const targetPolicyTraceData = {
                     allowedTargetConnectionIds: slConnectionIds,
@@ -1891,23 +1715,12 @@ export class IngestBundleRunner {
                 latestArtifactResolutions = stageIndex.artifactResolutions ?? [];
                 latestEvictionInputs = eviction?.deletedRawPaths ?? [];
                 latestUnresolvedCards = unresolvedCards ?? [];
-                const finalizationProvenance = this.partitionFinalizationActionsForProvenance({
-                    actions: finalizationActions,
-                    currentRawPaths: new Set(currentHashes.keys()),
-                    currentEvictionRawPaths: new Set(stageIndex.evictionsApplied.map((entry) => entry.rawPath)),
-                    overrideEvictionRawPaths: new Set(overrideReport?.body.evictionInputs ?? []),
-                });
-                if (finalizationOutcome) {
-                    finalizationOutcome.provenanceExclusions = finalizationProvenance.exclusions;
-                    latestFinalizationOutcome = finalizationOutcome;
-                }
                 const provenancePlan = this.buildProvenancePlan({
                     job,
                     syncId,
                     currentHashes,
                     stageIndex,
                     reconcileActions,
-                    finalizationActions: finalizationProvenance.actions,
                 });
                 const provenanceRows = provenancePlan.rows;
                 const currentRawPaths = new Set(currentHashes.keys());
@@ -1956,15 +1769,13 @@ export class IngestBundleRunner {
                     commitSha,
                     touchedPaths: mergeResult.touchedPaths,
                 });
-                const memoryFlowSavedActions = stageIndex.workUnits
-                    .flatMap((wu) => wu.actions)
-                    .concat(reconcileActions)
-                    .concat(finalizationActions);
+                const memoryFlowSavedActions = stageIndex.workUnits.flatMap((wu) => wu.actions).concat(reconcileActions);
+                const postProcessorMemoryCounts = postProcessorSavedMemoryCounts(postProcessorOutcome);
                 memoryFlow?.emit({
                     type: 'saved',
                     commitSha,
-                    wikiCount: countMemoryFlowActions(memoryFlowSavedActions, 'wiki'),
-                    slCount: countMemoryFlowActions(memoryFlowSavedActions, 'sl'),
+                    wikiCount: countMemoryFlowActions(memoryFlowSavedActions, 'wiki') + postProcessorMemoryCounts.wikiCount,
+                    slCount: countMemoryFlowActions(memoryFlowSavedActions, 'sl') + postProcessorMemoryCounts.slCount,
                 });
                 await stage6?.updateProgress(1.0, commitSha ? `Saved changes (${commitSha.slice(0, 8)})` : 'No changes to save');
                 // Sync the shared `knowledge` index from the squashed diff in a single
@@ -1981,7 +1792,7 @@ export class IngestBundleRunner {
                         ...new Set(memoryFlowSavedActions
                             .filter((action) => action.target === 'sl')
                             .map((action) => actionTargetConnectionId(action, job.connectionId))
-                            .concat(finalizationTouchedSources.map((source) => source.connectionId))),
+                            .concat((postProcessorOutcome?.touchedSources ?? []).map((source) => source.connectionId))),
                     ].sort();
                     for (const connectionId of touchedConnections) {
                         try {
@@ -2062,7 +1873,7 @@ export class IngestBundleRunner {
                     overrideOf: overrideReport?.jobId ?? null,
                     provenanceRows: reportProvenanceRows,
                     toolTranscripts: reportToolTranscripts,
-                    finalization: finalizationOutcome,
+                    postProcessor: postProcessorOutcome,
                     wikiSlRefRepairs: wikiSlRefRepairResult.repairs,
                     wikiSlRefRepairWarnings: wikiSlRefRepairResult.warnings,
                     ...(reportMemoryFlow ? { memoryFlow: reportMemoryFlow } : {}),
@@ -2220,7 +2031,6 @@ export class IngestBundleRunner {
                         artifactResolutions: latestArtifactResolutions,
                         evictionInputs: latestEvictionInputs,
                         reconciliationActions: latestReconciliationActions,
-                        finalization: latestFinalizationOutcome,
                         evictionDecisions: [],
                         unresolvedCards: latestUnresolvedCards,
                         supersededBy: null,