@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/node_modules/@ktx/context/dist/ingest/context-candidates/index.js

@@ -0,0 +1,4 @@
+export { CandidateDedupService } from './candidate-dedup.service.js';
+export { ContextCandidateCarryforwardService } from './context-candidate-carryforward.service.js';
+export { CuratorPaginationService } from './curator-pagination.service.js';
+export { buildContextCandidateEmbeddingText } from './embedding-text.js';

package/node_modules/@ktx/context/dist/ingest/context-candidates/store.d.ts

@@ -0,0 +1,24 @@
+import type { ContextCandidateForDedup } from '../ports.js';
+import type { ReconcileCandidateForPrompt } from '../stages/build-reconcile-context.js';
+import type { BudgetExhaustedCandidateForCarryForward, ContextCandidateRejectionReason, ContextCandidateVerdictSummary, CurrentRunEvidenceChunkForCarryForward, InsertContextCandidateInput, MarkContextCandidateClusterInput } from './types.js';
+export interface ContextCandidateStorePort {
+    listPendingCandidatesForDedup(runId: string): Promise<ContextCandidateForDedup[]>;
+    updateCandidateEmbedding(candidateId: string, embedding: number[]): Promise<void>;
+    markCandidatesAsMergedToCluster(params: MarkContextCandidateClusterInput): Promise<void>;
+    listBudgetExhaustedCandidatesForCarryForward(params: {
+        connectionId: string;
+        sourceKey: string;
+        currentRunId: string;
+    }): Promise<BudgetExhaustedCandidateForCarryForward[]>;
+    listCurrentRunEvidenceChunksForCarryForward(runId: string): Promise<CurrentRunEvidenceChunkForCarryForward[]>;
+    insertCandidate(params: InsertContextCandidateInput): Promise<{
+        id: string;
+    }>;
+    listCandidatesForPromptByKeys(runId: string, candidateKeys: string[]): Promise<ReconcileCandidateForPrompt[]>;
+    markPendingCandidatesByReason(params: {
+        runId: string;
+        candidateKeys: string[];
+        rejectionReason: ContextCandidateRejectionReason;
+    }): Promise<number>;
+    summarizeCandidateVerdicts(runId: string, candidateKeys: string[]): Promise<ContextCandidateVerdictSummary>;
+}

package/node_modules/@ktx/context/dist/ingest/context-candidates/store.js

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/context-candidates/store.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/context-candidates/store.test.js

@@ -0,0 +1,67 @@
+import { describe, expect, it, vi } from 'vitest';
+const candidate = {
+    id: 'candidate-1',
+    candidateKey: 'revenue-policy',
+    topic: 'Revenue policy',
+    assertion: 'Booked revenue excludes refunds.',
+    promotionScore: 10,
+    createdAt: new Date('2026-04-30T10:00:00.000Z'),
+    evidenceChunkIds: ['chunk-1'],
+    evidenceRefs: [{ stableCitationKey: 'notion:page-1:revenue' }],
+    embedding: '[1,0,0]',
+    lane: 'full',
+};
+const insert = {
+    runId: 'run-1',
+    connectionId: 'connection-1',
+    sourceKey: 'notion',
+    candidateKey: 'revenue-policy',
+    topic: 'Revenue policy',
+    assertion: 'Booked revenue excludes refunds.',
+    rationale: 'Finance handbook says this.',
+    evidenceChunkIds: ['chunk-1'],
+    evidenceRefs: [{ stableCitationKey: 'notion:page-1:revenue' }],
+    suggestedPageKey: 'revenue-policy',
+    actionHint: 'create',
+    durabilityScore: 3,
+    authorityScore: 3,
+    reuseScore: 3,
+    noveltyScore: 2,
+    riskScore: 1,
+    promotionScore: 10,
+    status: 'pending',
+    rejectionReason: null,
+    lane: 'full',
+    embedding: null,
+};
+describe('ContextCandidateStorePort', () => {
+    it('describes the persistence operations required by candidate services', async () => {
+        const store = {
+            listPendingCandidatesForDedup: vi.fn().mockResolvedValue([candidate]),
+            updateCandidateEmbedding: vi.fn().mockResolvedValue(undefined),
+            markCandidatesAsMergedToCluster: vi.fn().mockResolvedValue(undefined),
+            listBudgetExhaustedCandidatesForCarryForward: vi.fn().mockResolvedValue([]),
+            listCurrentRunEvidenceChunksForCarryForward: vi.fn().mockResolvedValue([]),
+            insertCandidate: vi.fn().mockResolvedValue({ id: 'candidate-2' }),
+            listCandidatesForPromptByKeys: vi.fn().mockResolvedValue([]),
+            markPendingCandidatesByReason: vi.fn().mockResolvedValue(0),
+            summarizeCandidateVerdicts: vi.fn().mockResolvedValue({
+                pending: 0,
+                promoted: 0,
+                merged: 0,
+                rejected: 0,
+                conflict: 0,
+                rejectedByReason: {},
+            }),
+        };
+        await expect(store.listPendingCandidatesForDedup('run-1')).resolves.toEqual([candidate]);
+        await expect(store.insertCandidate(insert)).resolves.toEqual({ id: 'candidate-2' });
+        await expect(store.markCandidatesAsMergedToCluster({
+            representativeId: 'candidate-1',
+            memberIds: ['candidate-3'],
+            evidenceChunkIds: ['chunk-1', 'chunk-3'],
+            evidenceRefs: [{ stableCitationKey: 'notion:page-1:revenue' }],
+            promotionScore: 10,
+        })).resolves.toBeUndefined();
+    });
+});

package/node_modules/@ktx/context/dist/ingest/context-candidates/types.d.ts

@@ -0,0 +1,103 @@
+import type { JsonValue } from '../ports.js';
+export type ContextCandidateActionHint = 'create' | 'update' | 'merge' | 'conflict' | 'skip';
+export type ContextCandidateStatus = 'pending' | 'promoted' | 'merged' | 'rejected' | 'conflict';
+export type ContextCandidateRejectionReason = 'low_score' | 'duplicates_existing_wiki' | 'not_durable' | 'conflict_unresolved' | 'exceeded_run_budget' | 'exceeded_curator_passes' | 'curator_pass_error';
+export type ContextCandidateLane = 'light' | 'full' | null;
+export type ContextCandidateScoreAggregation = 'max' | 'mean' | 'sum';
+export interface ContextCandidateForPrompt {
+    candidateKey: string;
+    topic: string;
+    assertion: string;
+    rationale: string;
+    actionHint: string;
+    status: string;
+    promotionScore: number;
+    suggestedPageKey: string | null;
+    evidenceRefs: JsonValue;
+}
+export interface ContextCandidateVerdictSummary {
+    pending: number;
+    promoted: number;
+    merged: number;
+    rejected: number;
+    conflict: number;
+    rejectedByReason: Record<string, number>;
+}
+export interface CuratorPaginationSettings {
+    batchSize: number;
+    maxPasses: number;
+    stepBudgetPerPass: number;
+}
+export interface InsertContextCandidateInput {
+    runId: string;
+    connectionId: string;
+    sourceKey: string;
+    candidateKey: string;
+    topic: string;
+    assertion: string;
+    rationale: string;
+    evidenceChunkIds: string[];
+    evidenceRefs: JsonValue;
+    suggestedPageKey: string | null;
+    actionHint: ContextCandidateActionHint;
+    durabilityScore: number;
+    authorityScore: number;
+    reuseScore: number;
+    noveltyScore: number;
+    riskScore: number;
+    promotionScore: number;
+    status: ContextCandidateStatus;
+    rejectionReason: string | null;
+    lane?: ContextCandidateLane;
+    embedding?: number[] | null;
+}
+export interface MarkContextCandidateClusterInput {
+    representativeId: string;
+    memberIds: string[];
+    evidenceChunkIds: string[];
+    evidenceRefs: JsonValue;
+    promotionScore: number;
+}
+export interface BudgetExhaustedCandidateForCarryForward {
+    sourceRunId: string;
+    candidateKey: string;
+    topic: string;
+    assertion: string;
+    rationale: string;
+    evidenceChunkIds: string[];
+    evidenceRefs: JsonValue;
+    suggestedPageKey: string | null;
+    actionHint: ContextCandidateActionHint;
+    durabilityScore: number;
+    authorityScore: number;
+    reuseScore: number;
+    noveltyScore: number;
+    riskScore: number;
+    promotionScore: number;
+    lane: ContextCandidateLane;
+}
+export interface CurrentRunEvidenceChunkForCarryForward {
+    chunkId: string;
+    stableCitationKey: string;
+    syncId: string;
+    rawPath: string;
+    title: string;
+    path: string;
+    url: string | null;
+    lastEditedAt: Date | null;
+    citation: JsonValue;
+    content: string;
+}
+export interface ContextCandidateEmbeddingPort {
+    maxBatchSize: number;
+    computeEmbedding(text: string): Promise<number[]>;
+    computeEmbeddingsBulk(texts: string[]): Promise<number[][]>;
+}
+export interface CandidateDedupSettings {
+    enabled: boolean;
+    topicSimilarityThreshold: number;
+    scoreAggregation: ContextCandidateScoreAggregation;
+}
+export interface ContextCandidateCarryforwardSettings {
+    reExamineBudgetExhaustedOnRerun: boolean;
+}

package/node_modules/@ktx/context/dist/ingest/context-candidates/types.js

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/context-evidence/context-evidence-index.service.d.ts

@@ -0,0 +1,53 @@
+import { type KtxLogger } from '../../core/index.js';
+import type { DiffSet } from '../types.js';
+import type { ContextEvidenceIndexStorePort } from './store.js';
+import type { ContextEvidenceEmbeddingPort, ContextEvidenceIndexSummary } from './types.js';
+interface IndexStagedDirArgs {
+    stagedDir: string;
+    runId: string;
+    connectionId: string;
+    sourceKey: string;
+    syncId: string;
+    diffSet: DiffSet;
+    currentHashes: Map<string, string>;
+    forceRebuild?: boolean;
+}
+interface PublishSyncArgs {
+    connectionId: string;
+    sourceKey: string;
+    syncId: string;
+    diffSet: DiffSet;
+}
+interface ContextEvidenceIndexServiceDeps {
+    store: ContextEvidenceIndexStorePort;
+    embeddings: ContextEvidenceEmbeddingPort;
+    logger?: Pick<KtxLogger, 'warn'>;
+}
+export declare class ContextEvidenceIndexService {
+    private readonly store;
+    private readonly embeddings;
+    private readonly logger;
+    constructor(deps: ContextEvidenceIndexServiceDeps);
+    indexStagedDir(args: IndexStagedDirArgs): Promise<ContextEvidenceIndexSummary>;
+    publishSync(args: PublishSyncArgs): Promise<{
+        documentsPublished: number;
+        documentsDeleted: number;
+    }>;
+    private collectDocuments;
+    private findFiles;
+    private indexableDocumentPaths;
+    private shouldIndexDocument;
+    private deletedMarkdownPaths;
+    private buildChunks;
+    private splitLargeChunks;
+    private buildSearchText;
+    private computeEmbeddings;
+    private toRawPath;
+    private headingLevelKey;
+    private slug;
+    private normalizeHeadingPath;
+    private buildStableCitationKey;
+    private estimateTokens;
+    private sha256;
+}
+export {};

package/node_modules/@ktx/context/dist/ingest/context-evidence/context-evidence-index.service.js

@@ -0,0 +1,303 @@
+import { createHash } from 'node:crypto';
+import { readdir, readFile } from 'node:fs/promises';
+import { basename, dirname, join, relative } from 'node:path';
+import { noopLogger } from '../../core/index.js';
+export class ContextEvidenceIndexService {
+    store;
+    embeddings;
+    logger;
+    constructor(deps) {
+        this.store = deps.store;
+        this.embeddings = deps.embeddings;
+        this.logger = deps.logger ?? noopLogger;
+    }
+    async indexStagedDir(args) {
+        const warnings = [];
+        const documents = await this.collectDocuments(args.stagedDir, warnings);
+        const indexablePaths = this.indexableDocumentPaths(args.diffSet);
+        let documentsIndexed = 0;
+        let chunksIndexed = 0;
+        let embeddingFailures = 0;
+        for (const staged of documents) {
+            if (!args.forceRebuild && !this.shouldIndexDocument(args.stagedDir, staged, indexablePaths)) {
+                continue;
+            }
+            const externalId = staged.metadata.id;
+            const title = staged.metadata.title;
+            const path = staged.metadata.path ?? title;
+            if (!externalId || !title || !path) {
+                warnings.push(`Skipped ${staged.metadataPath}: metadata requires id, title, and path`);
+                continue;
+            }
+            const rawPath = this.toRawPath(args.stagedDir, staged.markdownPath);
+            const contentHash = args.currentHashes.get(rawPath) ?? this.sha256(staged.markdown);
+            const document = await this.store.upsertDocument({
+                runId: args.runId,
+                connectionId: args.connectionId,
+                sourceKey: args.sourceKey,
+                externalId,
+                externalParentId: staged.metadata.parentId ?? null,
+                databaseId: staged.metadata.databaseId ?? null,
+                dataSourceId: staged.metadata.dataSourceId ?? null,
+                title,
+                path,
+                url: staged.metadata.url ?? null,
+                objectType: staged.metadata.objectType ?? 'page',
+                lastEditedAt: staged.metadata.lastEditedAt ? new Date(staged.metadata.lastEditedAt) : null,
+                lastEditedBy: staged.metadata.lastEditedBy ?? null,
+                rawPath,
+                syncId: args.syncId,
+                contentHash,
+                publishState: 'pending',
+                metadata: {
+                    metadataPath: this.toRawPath(args.stagedDir, staged.metadataPath),
+                    ...(staged.linksPath && staged.links
+                        ? { linksPath: this.toRawPath(args.stagedDir, staged.linksPath), links: staged.links }
+                        : {}),
+                    properties: staged.metadata.properties ?? {},
+                },
+            });
+            const chunks = this.buildChunks(staged.markdown, title);
+            const searchTexts = chunks.map((chunk) => this.buildSearchText(staged, chunk));
+            const embeddings = await this.computeEmbeddings(searchTexts);
+            if (embeddings.failed) {
+                embeddingFailures += 1;
+            }
+            const headingPathOccurrences = new Map();
+            const replaceChunks = chunks.map((chunk, ordinal) => {
+                const headingLeaf = chunk.headingPath[chunk.headingPath.length - 1] ?? title;
+                const headingSlug = this.slug(headingLeaf);
+                const normalizedHeadingPath = this.normalizeHeadingPath(chunk.headingPath);
+                const occurrence = (headingPathOccurrences.get(normalizedHeadingPath) ?? 0) + 1;
+                headingPathOccurrences.set(normalizedHeadingPath, occurrence);
+                return {
+                    chunkKey: `${this.headingLevelKey(chunk.headingPath)}:${headingSlug}:${String(ordinal).padStart(4, '0')}`,
+                    headingPath: chunk.headingPath,
+                    ordinal,
+                    content: chunk.content,
+                    searchText: searchTexts[ordinal],
+                    embedding: embeddings.values[ordinal] ?? null,
+                    tokenCount: this.estimateTokens(chunk.content),
+                    citation: {
+                        source: args.sourceKey,
+                        pageId: externalId,
+                        title,
+                        path,
+                        url: staged.metadata.url ?? null,
+                        lastEditedAt: staged.metadata.lastEditedAt ?? null,
+                        syncId: args.syncId,
+                        rawPath,
+                    },
+                    stableCitationKey: this.buildStableCitationKey(args.sourceKey, externalId, headingSlug, normalizedHeadingPath, occurrence),
+                    syncId: args.syncId,
+                    contentHash: this.sha256(chunk.content),
+                };
+            });
+            await this.store.replaceChunks(document.id, replaceChunks);
+            documentsIndexed += 1;
+            chunksIndexed += replaceChunks.length;
+        }
+        const deletedMarkdownPaths = this.deletedMarkdownPaths(args.diffSet);
+        const documentsDeleted = await this.store.countPublishedDocumentsByRawPaths(args.connectionId, args.sourceKey, deletedMarkdownPaths);
+        return { documentsIndexed, chunksIndexed, documentsDeleted, embeddingFailures, warnings };
+    }
+    async publishSync(args) {
+        return this.store.publishSync(args.connectionId, args.sourceKey, args.syncId, this.deletedMarkdownPaths(args.diffSet));
+    }
+    async collectDocuments(stagedDir, warnings) {
+        const metadataPaths = await this.findFiles(stagedDir, 'metadata.json');
+        const documents = [];
+        for (const metadataPath of metadataPaths) {
+            const markdownPath = join(dirname(metadataPath), 'page.md');
+            let markdown;
+            try {
+                markdown = await readFile(markdownPath, 'utf-8');
+            }
+            catch {
+                continue;
+            }
+            try {
+                const metadata = JSON.parse(await readFile(metadataPath, 'utf-8'));
+                const linksPath = join(dirname(metadataPath), 'links.json');
+                let links;
+                try {
+                    const parsedLinks = JSON.parse(await readFile(linksPath, 'utf-8'));
+                    if (parsedLinks && typeof parsedLinks === 'object' && !Array.isArray(parsedLinks)) {
+                        links = parsedLinks;
+                    }
+                }
+                catch {
+                    // links.json is optional.
+                }
+                documents.push({
+                    metadataPath,
+                    markdownPath,
+                    linksPath: links ? linksPath : undefined,
+                    metadata,
+                    links,
+                    markdown,
+                });
+            }
+            catch (error) {
+                warnings.push(`Skipped ${relative(stagedDir, metadataPath)}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        return documents;
+    }
+    async findFiles(root, fileName) {
+        const entries = await readdir(root, { withFileTypes: true, recursive: true });
+        return entries
+            .filter((entry) => entry.isFile() && entry.name === fileName)
+            .map((entry) => join(entry.parentPath, entry.name))
+            .sort();
+    }
+    indexableDocumentPaths(diffSet) {
+        return new Set([...diffSet.added, ...diffSet.modified]);
+    }
+    shouldIndexDocument(stagedDir, staged, indexablePaths) {
+        return (indexablePaths.has(this.toRawPath(stagedDir, staged.markdownPath)) ||
+            indexablePaths.has(this.toRawPath(stagedDir, staged.metadataPath)) ||
+            (staged.linksPath ? indexablePaths.has(this.toRawPath(stagedDir, staged.linksPath)) : false));
+    }
+    deletedMarkdownPaths(diffSet) {
+        return diffSet.deleted.filter((path) => basename(path) === 'page.md');
+    }
+    buildChunks(markdown, title) {
+        const lines = markdown.split(/\r?\n/);
+        const chunks = [];
+        let headingPath = [title];
+        let currentLines = [];
+        const flush = () => {
+            const content = currentLines.join('\n').trim();
+            if (content) {
+                chunks.push({ headingPath: [...headingPath], content });
+            }
+            currentLines = [];
+        };
+        for (const line of lines) {
+            const match = /^(#{1,6})\s+(.+)$/.exec(line);
+            if (match) {
+                flush();
+                const level = match[1].length;
+                const heading = match[2].trim();
+                headingPath = level === 1 ? [heading] : [...headingPath.slice(0, level - 1), heading];
+                continue;
+            }
+            currentLines.push(line);
+        }
+        flush();
+        if (chunks.length === 0) {
+            const content = markdown.trim();
+            return content ? [{ headingPath: [title], content }] : [];
+        }
+        return this.splitLargeChunks(chunks);
+    }
+    splitLargeChunks(chunks) {
+        const maxChars = 4800;
+        const out = [];
+        const pushBounded = (chunk, content) => {
+            for (let start = 0; start < content.length; start += maxChars) {
+                const part = content.slice(start, start + maxChars).trim();
+                if (part) {
+                    out.push({ ...chunk, content: part });
+                }
+            }
+        };
+        for (const chunk of chunks) {
+            if (chunk.content.length <= maxChars) {
+                out.push(chunk);
+                continue;
+            }
+            const paragraphs = chunk.content.split(/\n{2,}/);
+            let current = '';
+            for (const paragraph of paragraphs) {
+                if (paragraph.length > maxChars) {
+                    if (current) {
+                        out.push({ ...chunk, content: current });
+                        current = '';
+                    }
+                    pushBounded(chunk, paragraph);
+                    continue;
+                }
+                const next = current ? `${current}\n\n${paragraph}` : paragraph;
+                if (next.length > maxChars && current) {
+                    out.push({ ...chunk, content: current });
+                    current = paragraph;
+                }
+                else {
+                    current = next;
+                }
+            }
+            if (current) {
+                out.push({ ...chunk, content: current });
+            }
+        }
+        return out;
+    }
+    buildSearchText(staged, chunk) {
+        const properties = Object.entries(staged.metadata.properties ?? {})
+            .map(([key, value]) => `${key}: ${String(value)}`)
+            .join('\n');
+        return [staged.metadata.title, staged.metadata.path, chunk.headingPath.join(' / '), properties, chunk.content]
+            .filter(Boolean)
+            .join('\n');
+    }
+    async computeEmbeddings(texts) {
+        if (texts.length === 0) {
+            return { values: [], failed: false };
+        }
+        const configuredMaxBatchSize = this.embeddings.maxBatchSize;
+        const maxBatchSize = typeof configuredMaxBatchSize === 'number' &&
+            Number.isInteger(configuredMaxBatchSize) &&
+            configuredMaxBatchSize > 0
+            ? configuredMaxBatchSize
+            : 100;
+        const values = [];
+        let failed = false;
+        for (let offset = 0; offset < texts.length; offset += maxBatchSize) {
+            const batch = texts.slice(offset, offset + maxBatchSize);
+            try {
+                const batchEmbeddings = await this.embeddings.computeEmbeddingsBulk(batch);
+                if (batchEmbeddings.length !== batch.length) {
+                    throw new Error(`expected ${batch.length} embeddings, received ${batchEmbeddings.length}`);
+                }
+                values.push(...batchEmbeddings);
+            }
+            catch (error) {
+                failed = true;
+                this.logger.warn(`Context evidence embeddings failed: ${error instanceof Error ? error.message : String(error)}`);
+                values.push(...batch.map(() => null));
+            }
+        }
+        return { values, failed };
+    }
+    toRawPath(stagedDir, fullPath) {
+        return relative(stagedDir, fullPath).split('\\').join('/');
+    }
+    headingLevelKey(headingPath) {
+        return `h${Math.min(Math.max(headingPath.length, 1), 6)}`;
+    }
+    slug(value) {
+        const slug = value
+            .toLowerCase()
+            .replace(/[^a-z0-9]+/g, '-')
+            .replace(/^-+|-+$/g, '');
+        return slug || 'section';
+    }
+    normalizeHeadingPath(headingPath) {
+        return headingPath
+            .map((heading) => heading.trim().toLowerCase().replace(/\s+/g, ' '))
+            .filter(Boolean)
+            .join('/');
+    }
+    buildStableCitationKey(sourceKey, externalId, headingSlug, normalizedHeadingPath, occurrence) {
+        const digest = this.sha256([sourceKey, externalId, normalizedHeadingPath, String(occurrence)].join('\0')).slice(0, 16);
+        return `${sourceKey}:${externalId}:${headingSlug}:${digest}`;
+    }
+    estimateTokens(value) {
+        return Math.ceil(value.split(/\s+/).filter(Boolean).length * 1.3);
+    }
+    sha256(value) {
+        return createHash('sha256').update(value).digest('hex');
+    }
+}

package/node_modules/@ktx/context/dist/ingest/context-evidence/context-evidence-index.service.test.d.ts

@@ -0,0 +1 @@
+export {};