@kaelio/ktx 0.4.1 → 0.6.0

package/dist/context/ingest/context-candidates/context-candidate-carryforward.service.js

@@ -0,0 +1,131 @@
+import { createHash } from 'node:crypto';
+import { noopLogger } from '../../../context/core/config.js';
+export class ContextCandidateCarryforwardService {
+    deps;
+    logger;
+    constructor(deps) {
+        this.deps = deps;
+        this.logger = deps.logger ?? noopLogger;
+    }
+    async carryForward(args) {
+        const candidates = await this.deps.store.listBudgetExhaustedCandidatesForCarryForward({
+            connectionId: args.connectionId,
+            sourceKey: args.sourceKey,
+            currentRunId: args.runId,
+        });
+        const chunks = await this.deps.store.listCurrentRunEvidenceChunksForCarryForward(args.runId);
+        const chunksByStableKey = new Map(chunks.map((chunk) => [chunk.stableCitationKey, chunk]));
+        const allowStaleEvidence = this.deps.settings.reExamineBudgetExhaustedOnRerun;
+        let carriedForward = 0;
+        let skippedNotReemitted = 0;
+        let remappedEvidenceRefs = 0;
+        let staleEvidenceRefs = 0;
+        for (const candidate of candidates) {
+            const remap = this.remapEvidence(candidate, chunksByStableKey);
+            if (remap.remappedCount === 0 && !allowStaleEvidence) {
+                skippedNotReemitted += 1;
+                continue;
+            }
+            await this.deps.store.insertCandidate({
+                runId: args.runId,
+                connectionId: args.connectionId,
+                sourceKey: args.sourceKey,
+                candidateKey: candidate.candidateKey,
+                topic: candidate.topic,
+                assertion: candidate.assertion,
+                rationale: candidate.rationale,
+                evidenceChunkIds: remap.evidenceChunkIds,
+                evidenceRefs: remap.evidenceRefs,
+                suggestedPageKey: candidate.suggestedPageKey,
+                actionHint: candidate.actionHint,
+                durabilityScore: candidate.durabilityScore,
+                authorityScore: candidate.authorityScore,
+                reuseScore: candidate.reuseScore,
+                noveltyScore: candidate.noveltyScore,
+                riskScore: candidate.riskScore,
+                promotionScore: candidate.promotionScore,
+                status: 'pending',
+                rejectionReason: null,
+                lane: candidate.lane,
+                embedding: null,
+            });
+            carriedForward += 1;
+            remappedEvidenceRefs += remap.remappedCount;
+            staleEvidenceRefs += remap.staleCount;
+        }
+        const warnings = this.buildWarnings({ carriedForward, skippedNotReemitted, staleEvidenceRefs });
+        if (carriedForward > 0 || skippedNotReemitted > 0) {
+            this.logger.log(`Budget carryforward: considered ${candidates.length}, carried ${carriedForward}, skipped ${skippedNotReemitted}`);
+        }
+        return {
+            considered: candidates.length,
+            carriedForward,
+            skippedNotReemitted,
+            remappedEvidenceRefs,
+            staleEvidenceRefs,
+            warnings,
+        };
+    }
+    remapEvidence(candidate, chunksByStableKey) {
+        const refs = Array.isArray(candidate.evidenceRefs) ? candidate.evidenceRefs : [];
+        const remappedRefs = [];
+        const remappedChunkIds = [];
+        for (const ref of refs) {
+            const stableKey = this.stableCitationKey(ref);
+            const currentChunk = stableKey ? chunksByStableKey.get(stableKey) : undefined;
+            if (!currentChunk) {
+                continue;
+            }
+            remappedChunkIds.push(currentChunk.chunkId);
+            remappedRefs.push(this.currentEvidenceRef(currentChunk));
+        }
+        if (remappedRefs.length > 0) {
+            return {
+                evidenceChunkIds: [...new Set(remappedChunkIds)],
+                evidenceRefs: remappedRefs,
+                remappedCount: remappedRefs.length,
+                staleCount: 0,
+            };
+        }
+        return {
+            evidenceChunkIds: candidate.evidenceChunkIds,
+            evidenceRefs: candidate.evidenceRefs,
+            remappedCount: 0,
+            staleCount: refs.length,
+        };
+    }
+    stableCitationKey(ref) {
+        if (!ref || typeof ref !== 'object' || Array.isArray(ref)) {
+            return null;
+        }
+        const value = ref.stableCitationKey;
+        return typeof value === 'string' ? value : null;
+    }
+    currentEvidenceRef(chunk) {
+        return {
+            chunkId: chunk.chunkId,
+            stableCitationKey: chunk.stableCitationKey,
+            syncId: chunk.syncId,
+            rawPath: chunk.rawPath,
+            title: chunk.title,
+            path: chunk.path,
+            url: chunk.url,
+            lastEditedAt: chunk.lastEditedAt?.toISOString() ?? null,
+            snippetHash: createHash('sha256').update(chunk.content).digest('hex'),
+            citation: chunk.citation,
+        };
+    }
+    buildWarnings(params) {
+        const warnings = [];
+        if (params.carriedForward > 0) {
+            warnings.push(`Re-examined ${params.carriedForward} prior budget-exhausted context candidate${params.carriedForward === 1 ? '' : 's'}.`);
+        }
+        if (params.skippedNotReemitted > 0) {
+            warnings.push(`Skipped ${params.skippedNotReemitted} budget-exhausted context candidate${params.skippedNotReemitted === 1 ? '' : 's'} because its evidence was not re-emitted in this run.`);
+        }
+        if (params.staleEvidenceRefs > 0) {
+            warnings.push(`Carried ${params.staleEvidenceRefs} budget-exhausted evidence ref${params.staleEvidenceRefs === 1 ? '' : 's'} without a current-run chunk remap.`);
+        }
+        return warnings;
+    }
+}

package/dist/context/ingest/context-candidates/curator-pagination.service.d.ts

@@ -0,0 +1,64 @@
+import type { KtxModelRole } from '../../../llm/types.js';
+import { type KtxLogger } from '../../../context/core/config.js';
+import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js';
+import type { MemoryAction } from '../../../context/memory/types.js';
+import type { ContextCandidateForDedup, CuratorPaginationPort, CuratorPaginationReport } from '../ports.js';
+import type { ReconcileCandidateForPrompt, ReconcileCandidateSummary, ReconcilePromptRunState } from '../stages/build-reconcile-context.js';
+import { type ReconciliationOutcome } from '../stages/stage-4-reconciliation.js';
+import type { StageIndex } from '../stages/stage-index.types.js';
+import type { EvictionUnit } from '../types.js';
+import type { ContextCandidateStorePort } from './store.js';
+import type { CuratorPaginationSettings } from './types.js';
+interface CuratorPaginationBudget {
+    creates: number;
+    updates: number;
+}
+interface CuratorPaginationPromptInput {
+    summary: ReconcileCandidateSummary;
+    items: ReconcileCandidateForPrompt[];
+    runState: ReconcilePromptRunState;
+}
+export interface CuratorPaginationInput {
+    runId: string;
+    sourceKey: string;
+    jobId: string;
+    stageIndex: StageIndex;
+    evictionUnit: EvictionUnit | undefined;
+    representatives: ContextCandidateForDedup[];
+    initialBudget: CuratorPaginationBudget;
+    modelRole: KtxModelRole;
+    buildSystemPrompt: () => string;
+    buildUserPrompt: (input: CuratorPaginationPromptInput) => string;
+    buildToolSet: (passNumber: number) => KtxRuntimeToolSet;
+    getReconciliationActions: () => MemoryAction[];
+    onStepFinish?: (info: {
+        passNumber: number;
+        stepIndex: number;
+        stepBudget: number;
+    }) => void;
+}
+interface CuratorPaginationResult extends ReconciliationOutcome {
+    report: CuratorPaginationReport;
+    warnings: string[];
+}
+export interface CuratorPaginationServiceDeps {
+    store: ContextCandidateStorePort;
+    agentRunner: AgentRunnerPort;
+    settings: CuratorPaginationSettings;
+    logger?: KtxLogger;
+}
+export declare class CuratorPaginationService implements CuratorPaginationPort {
+    private readonly deps;
+    private readonly logger;
+    constructor(deps: CuratorPaginationServiceDeps);
+    reconcile(input: CuratorPaginationInput): Promise<CuratorPaginationResult>;
+    private runPass;
+    private batchSummary;
+    private hasNonCandidateReconcileWork;
+    private consumeBudget;
+    private toWikiRefs;
+    private reportFromVerdicts;
+    private emptyReport;
+    private result;
+}
+export {};

package/dist/context/ingest/context-candidates/curator-pagination.service.js

@@ -0,0 +1,240 @@
+import { noopLogger } from '../../../context/core/config.js';
+import { runReconciliationStage4 } from '../stages/stage-4-reconciliation.js';
+export class CuratorPaginationService {
+    deps;
+    logger;
+    constructor(deps) {
+        this.deps = deps;
+        this.logger = deps.logger ?? noopLogger;
+    }
+    async reconcile(input) {
+        const config = this.deps.settings;
+        const representatives = [...input.representatives];
+        const allCandidateKeys = representatives.map((candidate) => candidate.candidateKey);
+        const budget = {
+            creates: Math.max(0, input.initialBudget.creates),
+            updates: Math.max(0, input.initialBudget.updates),
+        };
+        const previouslyPromotedInRun = [];
+        const warnings = [];
+        let passNumber = 0;
+        let topicsExamined = 0;
+        let budgetExhausted = budget.creates === 0 && budget.updates === 0;
+        let stopReason;
+        let error;
+        let actionCursor = input.getReconciliationActions().length;
+        if (representatives.length === 0 && !this.hasNonCandidateReconcileWork(input.stageIndex, input.evictionUnit)) {
+            return this.result({
+                skipped: true,
+                stopReason,
+                error,
+                report: this.emptyReport(),
+                warnings,
+            });
+        }
+        if (representatives.length === 0) {
+            passNumber = 1;
+            const outcome = await this.runPass({
+                input,
+                candidates: [],
+                passNumber,
+                maxPasses: config.maxPasses,
+                budget,
+                previouslyPromotedInRun,
+                forceRun: false,
+            });
+            stopReason = outcome.stopReason;
+            error = outcome.error;
+            return this.result({
+                skipped: outcome.skipped,
+                stopReason,
+                error,
+                report: this.emptyReport({ passesRun: outcome.skipped ? 0 : 1 }),
+                warnings,
+            });
+        }
+        const queue = [...representatives];
+        while (queue.length > 0 && passNumber < config.maxPasses) {
+            if (budget.creates === 0 && budget.updates === 0) {
+                budgetExhausted = true;
+                await this.deps.store.markPendingCandidatesByReason({
+                    runId: input.runId,
+                    candidateKeys: queue.map((candidate) => candidate.candidateKey),
+                    rejectionReason: 'exceeded_run_budget',
+                });
+                queue.length = 0;
+                break;
+            }
+            const batch = queue.splice(0, config.batchSize);
+            const batchKeys = batch.map((candidate) => candidate.candidateKey);
+            passNumber += 1;
+            topicsExamined += batch.length;
+            const outcome = await this.runPass({
+                input,
+                candidates: batch,
+                passNumber,
+                maxPasses: config.maxPasses,
+                budget,
+                previouslyPromotedInRun,
+                forceRun: true,
+            });
+            stopReason = outcome.stopReason;
+            error = outcome.error;
+            const actions = input.getReconciliationActions();
+            const newWikiActions = actions
+                .slice(actionCursor)
+                .filter((action) => action.target === 'wiki' && (action.type === 'created' || action.type === 'updated'));
+            actionCursor = actions.length;
+            this.consumeBudget(budget, newWikiActions);
+            previouslyPromotedInRun.push(...this.toWikiRefs(newWikiActions));
+            if (outcome.stopReason === 'error' || outcome.error) {
+                const message = `Curator pass ${passNumber} failed: ${outcome.error?.message ?? outcome.stopReason ?? 'unknown error'}`;
+                warnings.push(message);
+                this.logger.warn(message);
+                await this.deps.store.markPendingCandidatesByReason({
+                    runId: input.runId,
+                    candidateKeys: batchKeys,
+                    rejectionReason: 'curator_pass_error',
+                });
+                continue;
+            }
+            if (budget.creates === 0 && budget.updates === 0) {
+                budgetExhausted = true;
+                await this.deps.store.markPendingCandidatesByReason({
+                    runId: input.runId,
+                    candidateKeys: [...batchKeys, ...queue.map((candidate) => candidate.candidateKey)],
+                    rejectionReason: 'exceeded_run_budget',
+                });
+                queue.length = 0;
+                break;
+            }
+        }
+        if (queue.length > 0) {
+            await this.deps.store.markPendingCandidatesByReason({
+                runId: input.runId,
+                candidateKeys: queue.map((candidate) => candidate.candidateKey),
+                rejectionReason: 'exceeded_curator_passes',
+            });
+        }
+        await this.deps.store.markPendingCandidatesByReason({
+            runId: input.runId,
+            candidateKeys: allCandidateKeys,
+            rejectionReason: 'exceeded_curator_passes',
+        });
+        const verdicts = await this.deps.store.summarizeCandidateVerdicts(input.runId, allCandidateKeys);
+        const report = this.reportFromVerdicts({
+            passesRun: passNumber,
+            topicsExamined,
+            budgetExhausted,
+            verdicts,
+        });
+        this.logger.log(`Curator: ${report.passesRun} passes, ${report.topicsExamined} topics examined, ${report.topicsByVerdict.promoted} promoted`);
+        return this.result({
+            skipped: false,
+            stopReason,
+            error,
+            report,
+            warnings,
+        });
+    }
+    async runPass(params) {
+        const config = this.deps.settings;
+        const candidateKeys = params.candidates.map((candidate) => candidate.candidateKey);
+        const items = await this.deps.store.listCandidatesForPromptByKeys(params.input.runId, candidateKeys);
+        const summary = this.batchSummary(items);
+        return runReconciliationStage4({
+            stageIndex: params.input.stageIndex,
+            evictionUnit: params.input.evictionUnit,
+            agentRunner: this.deps.agentRunner,
+            buildSystemPrompt: () => params.input.buildSystemPrompt(),
+            buildUserPrompt: () => params.input.buildUserPrompt({
+                summary,
+                items,
+                runState: {
+                    passNumber: params.passNumber,
+                    maxPasses: params.maxPasses,
+                    budgetRemaining: { ...params.budget },
+                    previouslyPromotedInRun: [...params.previouslyPromotedInRun],
+                },
+            }),
+            buildToolSet: () => params.input.buildToolSet(params.passNumber),
+            modelRole: params.input.modelRole,
+            stepBudget: config.stepBudgetPerPass,
+            sourceKey: params.input.sourceKey,
+            jobId: params.input.jobId,
+            forceRun: params.forceRun,
+            onStepFinish: params.input.onStepFinish
+                ? ({ stepIndex, stepBudget }) => params.input.onStepFinish?.({ passNumber: params.passNumber, stepIndex, stepBudget })
+                : undefined,
+        });
+    }
+    batchSummary(items) {
+        return items.reduce((summary, item) => {
+            summary.total += 1;
+            if (item.status === 'pending') {
+                summary.pending += 1;
+            }
+            else if (item.status === 'promoted') {
+                summary.promoted += 1;
+            }
+            else if (item.status === 'merged') {
+                summary.merged += 1;
+            }
+            else if (item.status === 'rejected') {
+                summary.rejected += 1;
+            }
+            else if (item.status === 'conflict') {
+                summary.conflict += 1;
+            }
+            return summary;
+        }, { total: 0, pending: 0, promoted: 0, merged: 0, rejected: 0, conflict: 0 });
+    }
+    hasNonCandidateReconcileWork(stageIndex, evictionUnit) {
+        return stageIndex.workUnits.some((wu) => wu.actions.length > 0) || !!evictionUnit?.deletedRawPaths.length;
+    }
+    consumeBudget(budget, actions) {
+        const creates = actions.filter((action) => action.type === 'created').length;
+        const updates = actions.filter((action) => action.type === 'updated').length;
+        budget.creates = Math.max(0, budget.creates - creates);
+        budget.updates = Math.max(0, budget.updates - updates);
+    }
+    toWikiRefs(actions) {
+        return actions.map((action) => ({
+            pageKey: action.key,
+            action: action.type,
+            summary: action.detail,
+        }));
+    }
+    reportFromVerdicts(params) {
+        return {
+            passesRun: params.passesRun,
+            topicsExamined: params.topicsExamined,
+            topicsByVerdict: {
+                promoted: params.verdicts.promoted,
+                merged: params.verdicts.merged,
+                rejected: params.verdicts.rejected,
+                conflict: params.verdicts.conflict,
+            },
+            topicsRejectedByReason: params.verdicts.rejectedByReason,
+            budgetExhausted: params.budgetExhausted,
+        };
+    }
+    emptyReport(overrides = {}) {
+        return {
+            passesRun: 0,
+            topicsExamined: 0,
+            topicsByVerdict: {
+                promoted: 0,
+                merged: 0,
+                rejected: 0,
+                conflict: 0,
+            },
+            topicsRejectedByReason: {},
+            budgetExhausted: false,
+            ...overrides,
+        };
+    }
+    result(result) {
+        return result;
+    }
+}

package/dist/context/ingest/context-candidates/types.d.ts

@@ -0,0 +1,93 @@
+import type { JsonValue } from '../ports.js';
+type ContextCandidateActionHint = 'create' | 'update' | 'merge' | 'conflict' | 'skip';
+type ContextCandidateStatus = 'pending' | 'promoted' | 'merged' | 'rejected' | 'conflict';
+export type ContextCandidateRejectionReason = 'low_score' | 'duplicates_existing_wiki' | 'not_durable' | 'conflict_unresolved' | 'exceeded_run_budget' | 'exceeded_curator_passes' | 'curator_pass_error';
+type ContextCandidateLane = 'light' | 'full' | null;
+type ContextCandidateScoreAggregation = 'max' | 'mean' | 'sum';
+export interface ContextCandidateVerdictSummary {
+    pending: number;
+    promoted: number;
+    merged: number;
+    rejected: number;
+    conflict: number;
+    rejectedByReason: Record<string, number>;
+}
+export interface CuratorPaginationSettings {
+    batchSize: number;
+    maxPasses: number;
+    stepBudgetPerPass: number;
+}
+export interface InsertContextCandidateInput {
+    runId: string;
+    connectionId: string;
+    sourceKey: string;
+    candidateKey: string;
+    topic: string;
+    assertion: string;
+    rationale: string;
+    evidenceChunkIds: string[];
+    evidenceRefs: JsonValue;
+    suggestedPageKey: string | null;
+    actionHint: ContextCandidateActionHint;
+    durabilityScore: number;
+    authorityScore: number;
+    reuseScore: number;
+    noveltyScore: number;
+    riskScore: number;
+    promotionScore: number;
+    status: ContextCandidateStatus;
+    rejectionReason: string | null;
+    lane?: ContextCandidateLane;
+    embedding?: number[] | null;
+}
+export interface MarkContextCandidateClusterInput {
+    representativeId: string;
+    memberIds: string[];
+    evidenceChunkIds: string[];
+    evidenceRefs: JsonValue;
+    promotionScore: number;
+}
+export interface BudgetExhaustedCandidateForCarryForward {
+    sourceRunId: string;
+    candidateKey: string;
+    topic: string;
+    assertion: string;
+    rationale: string;
+    evidenceChunkIds: string[];
+    evidenceRefs: JsonValue;
+    suggestedPageKey: string | null;
+    actionHint: ContextCandidateActionHint;
+    durabilityScore: number;
+    authorityScore: number;
+    reuseScore: number;
+    noveltyScore: number;
+    riskScore: number;
+    promotionScore: number;
+    lane: ContextCandidateLane;
+}
+export interface CurrentRunEvidenceChunkForCarryForward {
+    chunkId: string;
+    stableCitationKey: string;
+    syncId: string;
+    rawPath: string;
+    title: string;
+    path: string;
+    url: string | null;
+    lastEditedAt: Date | null;
+    citation: JsonValue;
+    content: string;
+}
+export interface ContextCandidateEmbeddingPort {
+    maxBatchSize: number;
+    computeEmbedding(text: string): Promise<number[]>;
+    computeEmbeddingsBulk(texts: string[]): Promise<number[][]>;
+}
+export interface CandidateDedupSettings {
+    enabled: boolean;
+    topicSimilarityThreshold: number;
+    scoreAggregation: ContextCandidateScoreAggregation;
+}
+export interface ContextCandidateCarryforwardSettings {
+    reExamineBudgetExhaustedOnRerun: boolean;
+}
+export {};

package/dist/context/ingest/context-evidence/context-evidence-index.service.d.ts

@@ -0,0 +1,53 @@
+import { type KtxLogger } from '../../../context/core/config.js';
+import type { DiffSet } from '../types.js';
+import type { ContextEvidenceIndexStorePort } from './store.js';
+import type { ContextEvidenceEmbeddingPort, ContextEvidenceIndexSummary } from './types.js';
+interface IndexStagedDirArgs {
+    stagedDir: string;
+    runId: string;
+    connectionId: string;
+    sourceKey: string;
+    syncId: string;
+    diffSet: DiffSet;
+    currentHashes: Map<string, string>;
+    forceRebuild?: boolean;
+}
+interface PublishSyncArgs {
+    connectionId: string;
+    sourceKey: string;
+    syncId: string;
+    diffSet: DiffSet;
+}
+interface ContextEvidenceIndexServiceDeps {
+    store: ContextEvidenceIndexStorePort;
+    embeddings: ContextEvidenceEmbeddingPort;
+    logger?: Pick<KtxLogger, 'warn'>;
+}
+export declare class ContextEvidenceIndexService {
+    private readonly store;
+    private readonly embeddings;
+    private readonly logger;
+    constructor(deps: ContextEvidenceIndexServiceDeps);
+    indexStagedDir(args: IndexStagedDirArgs): Promise<ContextEvidenceIndexSummary>;
+    publishSync(args: PublishSyncArgs): Promise<{
+        documentsPublished: number;
+        documentsDeleted: number;
+    }>;
+    private collectDocuments;
+    private findFiles;
+    private indexableDocumentPaths;
+    private shouldIndexDocument;
+    private deletedMarkdownPaths;
+    private buildChunks;
+    private splitLargeChunks;
+    private buildSearchText;
+    private computeEmbeddings;
+    private toRawPath;
+    private headingLevelKey;
+    private slug;
+    private normalizeHeadingPath;
+    private buildStableCitationKey;
+    private estimateTokens;
+    private sha256;
+}
+export {};