@kaelio/ktx 0.0.0 → 0.1.0-rc.1

package/node_modules/@ktx/context/dist/ingest/context-candidates/context-candidate-carryforward.service.js

@@ -0,0 +1,131 @@
+import { createHash } from 'node:crypto';
+import { noopLogger } from '../../core/index.js';
+export class ContextCandidateCarryforwardService {
+    deps;
+    logger;
+    constructor(deps) {
+        this.deps = deps;
+        this.logger = deps.logger ?? noopLogger;
+    }
+    async carryForward(args) {
+        const candidates = await this.deps.store.listBudgetExhaustedCandidatesForCarryForward({
+            connectionId: args.connectionId,
+            sourceKey: args.sourceKey,
+            currentRunId: args.runId,
+        });
+        const chunks = await this.deps.store.listCurrentRunEvidenceChunksForCarryForward(args.runId);
+        const chunksByStableKey = new Map(chunks.map((chunk) => [chunk.stableCitationKey, chunk]));
+        const allowStaleEvidence = this.deps.settings.reExamineBudgetExhaustedOnRerun;
+        let carriedForward = 0;
+        let skippedNotReemitted = 0;
+        let remappedEvidenceRefs = 0;
+        let staleEvidenceRefs = 0;
+        for (const candidate of candidates) {
+            const remap = this.remapEvidence(candidate, chunksByStableKey);
+            if (remap.remappedCount === 0 && !allowStaleEvidence) {
+                skippedNotReemitted += 1;
+                continue;
+            }
+            await this.deps.store.insertCandidate({
+                runId: args.runId,
+                connectionId: args.connectionId,
+                sourceKey: args.sourceKey,
+                candidateKey: candidate.candidateKey,
+                topic: candidate.topic,
+                assertion: candidate.assertion,
+                rationale: candidate.rationale,
+                evidenceChunkIds: remap.evidenceChunkIds,
+                evidenceRefs: remap.evidenceRefs,
+                suggestedPageKey: candidate.suggestedPageKey,
+                actionHint: candidate.actionHint,
+                durabilityScore: candidate.durabilityScore,
+                authorityScore: candidate.authorityScore,
+                reuseScore: candidate.reuseScore,
+                noveltyScore: candidate.noveltyScore,
+                riskScore: candidate.riskScore,
+                promotionScore: candidate.promotionScore,
+                status: 'pending',
+                rejectionReason: null,
+                lane: candidate.lane,
+                embedding: null,
+            });
+            carriedForward += 1;
+            remappedEvidenceRefs += remap.remappedCount;
+            staleEvidenceRefs += remap.staleCount;
+        }
+        const warnings = this.buildWarnings({ carriedForward, skippedNotReemitted, staleEvidenceRefs });
+        if (carriedForward > 0 || skippedNotReemitted > 0) {
+            this.logger.log(`Budget carryforward: considered ${candidates.length}, carried ${carriedForward}, skipped ${skippedNotReemitted}`);
+        }
+        return {
+            considered: candidates.length,
+            carriedForward,
+            skippedNotReemitted,
+            remappedEvidenceRefs,
+            staleEvidenceRefs,
+            warnings,
+        };
+    }
+    remapEvidence(candidate, chunksByStableKey) {
+        const refs = Array.isArray(candidate.evidenceRefs) ? candidate.evidenceRefs : [];
+        const remappedRefs = [];
+        const remappedChunkIds = [];
+        for (const ref of refs) {
+            const stableKey = this.stableCitationKey(ref);
+            const currentChunk = stableKey ? chunksByStableKey.get(stableKey) : undefined;
+            if (!currentChunk) {
+                continue;
+            }
+            remappedChunkIds.push(currentChunk.chunkId);
+            remappedRefs.push(this.currentEvidenceRef(currentChunk));
+        }
+        if (remappedRefs.length > 0) {
+            return {
+                evidenceChunkIds: [...new Set(remappedChunkIds)],
+                evidenceRefs: remappedRefs,
+                remappedCount: remappedRefs.length,
+                staleCount: 0,
+            };
+        }
+        return {
+            evidenceChunkIds: candidate.evidenceChunkIds,
+            evidenceRefs: candidate.evidenceRefs,
+            remappedCount: 0,
+            staleCount: refs.length,
+        };
+    }
+    stableCitationKey(ref) {
+        if (!ref || typeof ref !== 'object' || Array.isArray(ref)) {
+            return null;
+        }
+        const value = ref.stableCitationKey;
+        return typeof value === 'string' ? value : null;
+    }
+    currentEvidenceRef(chunk) {
+        return {
+            chunkId: chunk.chunkId,
+            stableCitationKey: chunk.stableCitationKey,
+            syncId: chunk.syncId,
+            rawPath: chunk.rawPath,
+            title: chunk.title,
+            path: chunk.path,
+            url: chunk.url,
+            lastEditedAt: chunk.lastEditedAt?.toISOString() ?? null,
+            snippetHash: createHash('sha256').update(chunk.content).digest('hex'),
+            citation: chunk.citation,
+        };
+    }
+    buildWarnings(params) {
+        const warnings = [];
+        if (params.carriedForward > 0) {
+            warnings.push(`Re-examined ${params.carriedForward} prior budget-exhausted context candidate${params.carriedForward === 1 ? '' : 's'}.`);
+        }
+        if (params.skippedNotReemitted > 0) {
+            warnings.push(`Skipped ${params.skippedNotReemitted} budget-exhausted context candidate${params.skippedNotReemitted === 1 ? '' : 's'} because its evidence was not re-emitted in this run.`);
+        }
+        if (params.staleEvidenceRefs > 0) {
+            warnings.push(`Carried ${params.staleEvidenceRefs} budget-exhausted evidence ref${params.staleEvidenceRefs === 1 ? '' : 's'} without a current-run chunk remap.`);
+        }
+        return warnings;
+    }
+}

package/node_modules/@ktx/context/dist/ingest/context-candidates/context-candidate-carryforward.service.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/context-candidates/context-candidate-carryforward.service.test.js

@@ -0,0 +1,161 @@
+import { createHash } from 'node:crypto';
+import { describe, expect, it, vi } from 'vitest';
+import { ContextCandidateCarryforwardService } from './context-candidate-carryforward.service.js';
+function candidate(overrides = {}) {
+    return {
+        sourceRunId: 'prior-run-1',
+        candidateKey: 'budget-revenue-policy',
+        topic: 'Revenue policy',
+        assertion: 'Booked revenue excludes refunds.',
+        rationale: 'The finance handbook states this reusable rule.',
+        evidenceChunkIds: ['prior-chunk-1'],
+        evidenceRefs: [
+            {
+                chunkId: 'prior-chunk-1',
+                stableCitationKey: 'notion:page-1:revenue-policy',
+                syncId: 'sync-prior',
+                rawPath: 'pages/page-1/page.md',
+            },
+        ],
+        suggestedPageKey: 'revenue-policy',
+        actionHint: 'create',
+        durabilityScore: 3,
+        authorityScore: 3,
+        reuseScore: 3,
+        noveltyScore: 2,
+        riskScore: 1,
+        promotionScore: 10,
+        lane: 'full',
+        ...overrides,
+    };
+}
+function chunk(overrides = {}) {
+    return {
+        chunkId: 'current-chunk-1',
+        stableCitationKey: 'notion:page-1:revenue-policy',
+        syncId: 'sync-current',
+        rawPath: 'pages/page-1/page.md',
+        title: 'Revenue Policy',
+        path: 'Company / Revenue Policy',
+        url: 'https://notion.example/page-1',
+        lastEditedAt: new Date('2026-04-30T12:00:00.000Z'),
+        citation: {
+            source: 'notion',
+            pageId: 'page-1',
+            syncId: 'sync-current',
+            rawPath: 'pages/page-1/page.md',
+        },
+        content: 'Booked revenue excludes refunds and test accounts.',
+        ...overrides,
+    };
+}
+function buildHarness(reExamineBudgetExhaustedOnRerun) {
+    const store = {
+        listPendingCandidatesForDedup: vi.fn(),
+        updateCandidateEmbedding: vi.fn(),
+        markCandidatesAsMergedToCluster: vi.fn(),
+        listBudgetExhaustedCandidatesForCarryForward: vi.fn(),
+        listCurrentRunEvidenceChunksForCarryForward: vi.fn(),
+        insertCandidate: vi.fn().mockResolvedValue({ id: 'new-candidate-1' }),
+    };
+    const service = new ContextCandidateCarryforwardService({
+        store: store,
+        settings: { reExamineBudgetExhaustedOnRerun },
+    });
+    return { service, store };
+}
+describe('ContextCandidateCarryforwardService', () => {
+    it('carries a prior budget-exhausted candidate with remapped current evidence', async () => {
+        const { service, store } = buildHarness(true);
+        store.listBudgetExhaustedCandidatesForCarryForward.mockResolvedValueOnce([candidate()]);
+        store.listCurrentRunEvidenceChunksForCarryForward.mockResolvedValueOnce([chunk()]);
+        const result = await service.carryForward({
+            runId: 'current-run-1',
+            connectionId: 'connection-1',
+            sourceKey: 'notion',
+        });
+        expect(result).toMatchObject({
+            considered: 1,
+            carriedForward: 1,
+            skippedNotReemitted: 0,
+            remappedEvidenceRefs: 1,
+            staleEvidenceRefs: 0,
+        });
+        expect(store.insertCandidate).toHaveBeenCalledWith(expect.objectContaining({
+            runId: 'current-run-1',
+            connectionId: 'connection-1',
+            sourceKey: 'notion',
+            candidateKey: 'budget-revenue-policy',
+            evidenceChunkIds: ['current-chunk-1'],
+            evidenceRefs: [
+                expect.objectContaining({
+                    chunkId: 'current-chunk-1',
+                    stableCitationKey: 'notion:page-1:revenue-policy',
+                    syncId: 'sync-current',
+                    snippetHash: createHash('sha256')
+                        .update('Booked revenue excludes refunds and test accounts.')
+                        .digest('hex'),
+                }),
+            ],
+            status: 'pending',
+            rejectionReason: null,
+            lane: 'full',
+            embedding: null,
+        }));
+        expect(result.warnings).toEqual(['Re-examined 1 prior budget-exhausted context candidate.']);
+    });
+    it('skips stale prior candidates when config requires current evidence re-emission', async () => {
+        const { service, store } = buildHarness(false);
+        store.listBudgetExhaustedCandidatesForCarryForward.mockResolvedValueOnce([candidate()]);
+        store.listCurrentRunEvidenceChunksForCarryForward.mockResolvedValueOnce([]);
+        const result = await service.carryForward({
+            runId: 'current-run-1',
+            connectionId: 'connection-1',
+            sourceKey: 'notion',
+        });
+        expect(result).toMatchObject({
+            considered: 1,
+            carriedForward: 0,
+            skippedNotReemitted: 1,
+            remappedEvidenceRefs: 0,
+            staleEvidenceRefs: 0,
+        });
+        expect(store.insertCandidate).not.toHaveBeenCalled();
+        expect(result.warnings).toEqual([
+            'Skipped 1 budget-exhausted context candidate because its evidence was not re-emitted in this run.',
+        ]);
+    });
+    it('carries stale prior evidence when reExamineBudgetExhaustedOnRerun is enabled', async () => {
+        const { service, store } = buildHarness(true);
+        store.listBudgetExhaustedCandidatesForCarryForward.mockResolvedValueOnce([candidate()]);
+        store.listCurrentRunEvidenceChunksForCarryForward.mockResolvedValueOnce([]);
+        const result = await service.carryForward({
+            runId: 'current-run-1',
+            connectionId: 'connection-1',
+            sourceKey: 'notion',
+        });
+        expect(result).toMatchObject({
+            considered: 1,
+            carriedForward: 1,
+            skippedNotReemitted: 0,
+            remappedEvidenceRefs: 0,
+            staleEvidenceRefs: 1,
+        });
+        expect(store.insertCandidate).toHaveBeenCalledWith(expect.objectContaining({
+            runId: 'current-run-1',
+            evidenceChunkIds: ['prior-chunk-1'],
+            evidenceRefs: [
+                {
+                    chunkId: 'prior-chunk-1',
+                    stableCitationKey: 'notion:page-1:revenue-policy',
+                    syncId: 'sync-prior',
+                    rawPath: 'pages/page-1/page.md',
+                },
+            ],
+        }));
+        expect(result.warnings).toEqual([
+            'Re-examined 1 prior budget-exhausted context candidate.',
+            'Carried 1 budget-exhausted evidence ref without a current-run chunk remap.',
+        ]);
+    });
+});

package/node_modules/@ktx/context/dist/ingest/context-candidates/curator-pagination.service.d.ts

@@ -0,0 +1,64 @@
+import type { KtxModelRole } from '@ktx/llm';
+import { type KtxLogger } from '../../core/index.js';
+import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../llm/index.js';
+import type { MemoryAction } from '../../memory/index.js';
+import type { ContextCandidateForDedup, CuratorPaginationPort, CuratorPaginationReport } from '../ports.js';
+import type { ReconcileCandidateForPrompt, ReconcileCandidateSummary, ReconcilePromptRunState } from '../stages/build-reconcile-context.js';
+import { type ReconciliationOutcome } from '../stages/stage-4-reconciliation.js';
+import type { StageIndex } from '../stages/stage-index.types.js';
+import type { EvictionUnit } from '../types.js';
+import type { ContextCandidateStorePort } from './store.js';
+import type { CuratorPaginationSettings } from './types.js';
+interface CuratorPaginationBudget {
+    creates: number;
+    updates: number;
+}
+interface CuratorPaginationPromptInput {
+    summary: ReconcileCandidateSummary;
+    items: ReconcileCandidateForPrompt[];
+    runState: ReconcilePromptRunState;
+}
+export interface CuratorPaginationInput {
+    runId: string;
+    sourceKey: string;
+    jobId: string;
+    stageIndex: StageIndex;
+    evictionUnit: EvictionUnit | undefined;
+    representatives: ContextCandidateForDedup[];
+    initialBudget: CuratorPaginationBudget;
+    modelRole: KtxModelRole;
+    buildSystemPrompt: () => string;
+    buildUserPrompt: (input: CuratorPaginationPromptInput) => string;
+    buildToolSet: (passNumber: number) => KtxRuntimeToolSet;
+    getReconciliationActions: () => MemoryAction[];
+    onStepFinish?: (info: {
+        passNumber: number;
+        stepIndex: number;
+        stepBudget: number;
+    }) => void;
+}
+interface CuratorPaginationResult extends ReconciliationOutcome {
+    report: CuratorPaginationReport;
+    warnings: string[];
+}
+export interface CuratorPaginationServiceDeps {
+    store: ContextCandidateStorePort;
+    agentRunner: AgentRunnerPort;
+    settings: CuratorPaginationSettings;
+    logger?: KtxLogger;
+}
+export declare class CuratorPaginationService implements CuratorPaginationPort {
+    private readonly deps;
+    private readonly logger;
+    constructor(deps: CuratorPaginationServiceDeps);
+    reconcile(input: CuratorPaginationInput): Promise<CuratorPaginationResult>;
+    private runPass;
+    private batchSummary;
+    private hasNonCandidateReconcileWork;
+    private consumeBudget;
+    private toWikiRefs;
+    private reportFromVerdicts;
+    private emptyReport;
+    private result;
+}
+export {};

package/node_modules/@ktx/context/dist/ingest/context-candidates/curator-pagination.service.js

@@ -0,0 +1,240 @@
+import { noopLogger } from '../../core/index.js';
+import { runReconciliationStage4 } from '../stages/stage-4-reconciliation.js';
+export class CuratorPaginationService {
+    deps;
+    logger;
+    constructor(deps) {
+        this.deps = deps;
+        this.logger = deps.logger ?? noopLogger;
+    }
+    async reconcile(input) {
+        const config = this.deps.settings;
+        const representatives = [...input.representatives];
+        const allCandidateKeys = representatives.map((candidate) => candidate.candidateKey);
+        const budget = {
+            creates: Math.max(0, input.initialBudget.creates),
+            updates: Math.max(0, input.initialBudget.updates),
+        };
+        const previouslyPromotedInRun = [];
+        const warnings = [];
+        let passNumber = 0;
+        let topicsExamined = 0;
+        let budgetExhausted = budget.creates === 0 && budget.updates === 0;
+        let stopReason;
+        let error;
+        let actionCursor = input.getReconciliationActions().length;
+        if (representatives.length === 0 && !this.hasNonCandidateReconcileWork(input.stageIndex, input.evictionUnit)) {
+            return this.result({
+                skipped: true,
+                stopReason,
+                error,
+                report: this.emptyReport(),
+                warnings,
+            });
+        }
+        if (representatives.length === 0) {
+            passNumber = 1;
+            const outcome = await this.runPass({
+                input,
+                candidates: [],
+                passNumber,
+                maxPasses: config.maxPasses,
+                budget,
+                previouslyPromotedInRun,
+                forceRun: false,
+            });
+            stopReason = outcome.stopReason;
+            error = outcome.error;
+            return this.result({
+                skipped: outcome.skipped,
+                stopReason,
+                error,
+                report: this.emptyReport({ passesRun: outcome.skipped ? 0 : 1 }),
+                warnings,
+            });
+        }
+        const queue = [...representatives];
+        while (queue.length > 0 && passNumber < config.maxPasses) {
+            if (budget.creates === 0 && budget.updates === 0) {
+                budgetExhausted = true;
+                await this.deps.store.markPendingCandidatesByReason({
+                    runId: input.runId,
+                    candidateKeys: queue.map((candidate) => candidate.candidateKey),
+                    rejectionReason: 'exceeded_run_budget',
+                });
+                queue.length = 0;
+                break;
+            }
+            const batch = queue.splice(0, config.batchSize);
+            const batchKeys = batch.map((candidate) => candidate.candidateKey);
+            passNumber += 1;
+            topicsExamined += batch.length;
+            const outcome = await this.runPass({
+                input,
+                candidates: batch,
+                passNumber,
+                maxPasses: config.maxPasses,
+                budget,
+                previouslyPromotedInRun,
+                forceRun: true,
+            });
+            stopReason = outcome.stopReason;
+            error = outcome.error;
+            const actions = input.getReconciliationActions();
+            const newWikiActions = actions
+                .slice(actionCursor)
+                .filter((action) => action.target === 'wiki' && (action.type === 'created' || action.type === 'updated'));
+            actionCursor = actions.length;
+            this.consumeBudget(budget, newWikiActions);
+            previouslyPromotedInRun.push(...this.toWikiRefs(newWikiActions));
+            if (outcome.stopReason === 'error' || outcome.error) {
+                const message = `Curator pass ${passNumber} failed: ${outcome.error?.message ?? outcome.stopReason ?? 'unknown error'}`;
+                warnings.push(message);
+                this.logger.warn(message);
+                await this.deps.store.markPendingCandidatesByReason({
+                    runId: input.runId,
+                    candidateKeys: batchKeys,
+                    rejectionReason: 'curator_pass_error',
+                });
+                continue;
+            }
+            if (budget.creates === 0 && budget.updates === 0) {
+                budgetExhausted = true;
+                await this.deps.store.markPendingCandidatesByReason({
+                    runId: input.runId,
+                    candidateKeys: [...batchKeys, ...queue.map((candidate) => candidate.candidateKey)],
+                    rejectionReason: 'exceeded_run_budget',
+                });
+                queue.length = 0;
+                break;
+            }
+        }
+        if (queue.length > 0) {
+            await this.deps.store.markPendingCandidatesByReason({
+                runId: input.runId,
+                candidateKeys: queue.map((candidate) => candidate.candidateKey),
+                rejectionReason: 'exceeded_curator_passes',
+            });
+        }
+        await this.deps.store.markPendingCandidatesByReason({
+            runId: input.runId,
+            candidateKeys: allCandidateKeys,
+            rejectionReason: 'exceeded_curator_passes',
+        });
+        const verdicts = await this.deps.store.summarizeCandidateVerdicts(input.runId, allCandidateKeys);
+        const report = this.reportFromVerdicts({
+            passesRun: passNumber,
+            topicsExamined,
+            budgetExhausted,
+            verdicts,
+        });
+        this.logger.log(`Curator: ${report.passesRun} passes, ${report.topicsExamined} topics examined, ${report.topicsByVerdict.promoted} promoted`);
+        return this.result({
+            skipped: false,
+            stopReason,
+            error,
+            report,
+            warnings,
+        });
+    }
+    async runPass(params) {
+        const config = this.deps.settings;
+        const candidateKeys = params.candidates.map((candidate) => candidate.candidateKey);
+        const items = await this.deps.store.listCandidatesForPromptByKeys(params.input.runId, candidateKeys);
+        const summary = this.batchSummary(items);
+        return runReconciliationStage4({
+            stageIndex: params.input.stageIndex,
+            evictionUnit: params.input.evictionUnit,
+            agentRunner: this.deps.agentRunner,
+            buildSystemPrompt: () => params.input.buildSystemPrompt(),
+            buildUserPrompt: () => params.input.buildUserPrompt({
+                summary,
+                items,
+                runState: {
+                    passNumber: params.passNumber,
+                    maxPasses: params.maxPasses,
+                    budgetRemaining: { ...params.budget },
+                    previouslyPromotedInRun: [...params.previouslyPromotedInRun],
+                },
+            }),
+            buildToolSet: () => params.input.buildToolSet(params.passNumber),
+            modelRole: params.input.modelRole,
+            stepBudget: config.stepBudgetPerPass,
+            sourceKey: params.input.sourceKey,
+            jobId: params.input.jobId,
+            forceRun: params.forceRun,
+            onStepFinish: params.input.onStepFinish
+                ? ({ stepIndex, stepBudget }) => params.input.onStepFinish?.({ passNumber: params.passNumber, stepIndex, stepBudget })
+                : undefined,
+        });
+    }
+    batchSummary(items) {
+        return items.reduce((summary, item) => {
+            summary.total += 1;
+            if (item.status === 'pending') {
+                summary.pending += 1;
+            }
+            else if (item.status === 'promoted') {
+                summary.promoted += 1;
+            }
+            else if (item.status === 'merged') {
+                summary.merged += 1;
+            }
+            else if (item.status === 'rejected') {
+                summary.rejected += 1;
+            }
+            else if (item.status === 'conflict') {
+                summary.conflict += 1;
+            }
+            return summary;
+        }, { total: 0, pending: 0, promoted: 0, merged: 0, rejected: 0, conflict: 0 });
+    }
+    hasNonCandidateReconcileWork(stageIndex, evictionUnit) {
+        return stageIndex.workUnits.some((wu) => wu.actions.length > 0) || !!evictionUnit?.deletedRawPaths.length;
+    }
+    consumeBudget(budget, actions) {
+        const creates = actions.filter((action) => action.type === 'created').length;
+        const updates = actions.filter((action) => action.type === 'updated').length;
+        budget.creates = Math.max(0, budget.creates - creates);
+        budget.updates = Math.max(0, budget.updates - updates);
+    }
+    toWikiRefs(actions) {
+        return actions.map((action) => ({
+            pageKey: action.key,
+            action: action.type,
+            summary: action.detail,
+        }));
+    }
+    reportFromVerdicts(params) {
+        return {
+            passesRun: params.passesRun,
+            topicsExamined: params.topicsExamined,
+            topicsByVerdict: {
+                promoted: params.verdicts.promoted,
+                merged: params.verdicts.merged,
+                rejected: params.verdicts.rejected,
+                conflict: params.verdicts.conflict,
+            },
+            topicsRejectedByReason: params.verdicts.rejectedByReason,
+            budgetExhausted: params.budgetExhausted,
+        };
+    }
+    emptyReport(overrides = {}) {
+        return {
+            passesRun: 0,
+            topicsExamined: 0,
+            topicsByVerdict: {
+                promoted: 0,
+                merged: 0,
+                rejected: 0,
+                conflict: 0,
+            },
+            topicsRejectedByReason: {},
+            budgetExhausted: false,
+            ...overrides,
+        };
+    }
+    result(result) {
+        return result;
+    }
+}

package/node_modules/@ktx/context/dist/ingest/context-candidates/curator-pagination.service.test.d.ts

@@ -0,0 +1 @@
+export {};