@kaelio/ktx 0.5.0 → 0.7.0

package/dist/context/search/backend-conformance.test-utils.test.js

@@ -1,408 +0,0 @@
-import { mkdtemp, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, beforeEach, describe, it } from 'vitest';
-import { SqliteContextEvidenceStore } from '../ingest/context-evidence/sqlite-context-evidence-store.js';
-import { initKtxProject } from '../project/project.js';
-import { searchLocalSlSources, writeLocalSlSource } from '../sl/local-sl.js';
-import { searchLocalKnowledgePages, writeLocalKnowledgePage, } from '../wiki/local-knowledge.js';
-import { assertSearchBackendCapabilities, assertSearchBackendConformanceCase, } from './backend-conformance.test-utils.js';
-const SQLITE_SEARCH_CAPABILITIES = {
-    fts: true,
-    vector: false,
-    fuzzy: false,
-    jsonSearch: true,
-    arraySearch: false,
-};
-const ORDERS_YAML = [
-    'name: orders',
-    'table: public.orders',
-    'grain:',
-    '  - order_id',
-    'columns:',
-    '  - name: order_id',
-    '    type: string',
-    '  - name: revenue',
-    '    type: number',
-    'measures:',
-    '  - name: total_revenue',
-    '    expr: sum(revenue)',
-    '',
-].join('\n');
-const FINANCE_ORDERS_YAML = [
-    'name: orders',
-    'descriptions:',
-    '  user: Finance orders used for invoice reconciliation.',
-    'table: finance.orders',
-    'grain:',
-    '  - order_id',
-    'columns:',
-    '  - name: order_id',
-    '    type: string',
-    '  - name: invoice_status',
-    '    type: string',
-    '',
-].join('\n');
-class FakeEmbeddingPort {
-    maxBatchSize = 16;
-    async computeEmbedding(text) {
-        return text.toLowerCase().includes('semantic revenue') ? [1, 0] : [0, 1];
-    }
-    async computeEmbeddingsBulk(texts) {
-        return Promise.all(texts.map((text) => this.computeEmbedding(text)));
-    }
-}
-function toSlConformanceResult(result) {
-    return {
-        id: `${result.connectionId}/${result.name}`,
-        score: result.score ?? 0,
-        matchReasons: result.matchReasons ?? [],
-        lanes: result.lanes,
-        dictionaryMatches: result.dictionaryMatches,
-    };
-}
-function toWikiConformanceResult(result) {
-    return {
-        id: result.key,
-        score: result.score,
-        matchReasons: result.matchReasons,
-        lanes: result.lanes,
-    };
-}
-function toContextConformanceResult(result) {
-    return {
-        id: `${result.externalId}:${result.stableCitationKey}`,
-        score: result.score,
-        matchReasons: result.matchReasons ?? [],
-        lanes: result.lanes,
-    };
-}
-async function seedSemanticLayerProject(project) {
-    await writeLocalSlSource(project, {
-        connectionId: 'warehouse',
-        sourceName: 'orders',
-        yaml: ORDERS_YAML,
-    });
-    await writeLocalSlSource(project, {
-        connectionId: 'finance',
-        sourceName: 'orders',
-        yaml: FINANCE_ORDERS_YAML,
-    });
-    await project.fileStore.writeFile('raw-sources/warehouse/live-database/sync-1/enrichment/relationship-profile.json', `${JSON.stringify({
-        connectionId: 'warehouse',
-        driver: 'postgres',
-        sqlAvailable: true,
-        queryCount: 2,
-        tables: [],
-        columns: {
-            'orders.status': {
-                table: { catalog: null, db: 'public', name: 'orders' },
-                column: 'status',
-                nativeType: 'text',
-                normalizedType: 'string',
-                rowCount: 10,
-                nullCount: 0,
-                distinctCount: 2,
-                uniquenessRatio: 0.2,
-                nullRate: 0,
-                sampleValues: ['paid', 'refunded'],
-                minTextLength: 4,
-                maxTextLength: 8,
-            },
-        },
-        warnings: [],
-    }, null, 2)}\n`, 'ktx', 'ktx@example.com', 'Seed dictionary profile');
-}
-async function seedWikiProject(project) {
-    await writeLocalKnowledgePage(project, {
-        key: 'metrics-revenue',
-        scope: 'GLOBAL',
-        summary: 'Semantic revenue definition',
-        content: 'Revenue is recognized when an order is paid.',
-        tags: ['finance'],
-        refs: ['semantic-layer/warehouse/orders.yaml'],
-        slRefs: ['orders'],
-    });
-    await writeLocalKnowledgePage(project, {
-        key: 'support-escalations',
-        scope: 'GLOBAL',
-        summary: 'Support escalation process',
-        content: 'Escalations move urgent support tickets to the operations queue.',
-        tags: ['operations'],
-    });
-}
-async function seedContextDocument(subject, input = {}) {
-    const runId = input.runId ?? 'run-1';
-    const syncId = input.syncId ?? 'sync-1';
-    const externalId = input.externalId ?? 'page-1';
-    const title = input.title ?? 'Revenue Policy';
-    const rawPath = input.rawPath ?? `pages/${externalId}/page.md`;
-    const doc = await subject.upsertDocument({
-        runId,
-        connectionId: 'conn-1',
-        sourceKey: 'notion',
-        externalId,
-        externalParentId: null,
-        databaseId: null,
-        dataSourceId: null,
-        title,
-        path: `Company Handbook / ${title}`,
-        url: `https://notion.test/${externalId}`,
-        objectType: 'page',
-        lastEditedAt: new Date('2026-04-30T10:00:00.000Z'),
-        lastEditedBy: 'user-1',
-        rawPath,
-        syncId,
-        contentHash: `hash-${externalId}`,
-        publishState: input.publishState ?? 'published',
-        metadata: input.metadata ?? {},
-    });
-    await subject.replaceChunks(doc.id, [
-        {
-            chunkKey: 'intro',
-            headingPath: ['Policy'],
-            ordinal: 0,
-            content: input.content ?? `${title} requires approval from the accountable owner.`,
-            searchText: input.searchText ?? `${title} approval accountable owner`,
-            embedding: input.embedding ?? [1, 0, 0],
-            tokenCount: 8,
-            citation: {
-                source: 'notion',
-                pageId: externalId,
-                title,
-                syncId,
-                rawPath,
-            },
-            stableCitationKey: `notion:${externalId}:intro`,
-            syncId,
-            contentHash: `chunk-${externalId}`,
-        },
-    ]);
-    const read = await subject.readDocumentByExternalId('conn-1', 'notion', externalId, runId);
-    if (!read) {
-        throw new Error(`seeded document ${externalId} was not readable`);
-    }
-    return { documentId: doc.id, chunkId: read.chunks[0].id };
-}
-describe('SQLite hybrid search backend conformance', () => {
-    let tempDir;
-    let project;
-    let dbPath;
-    beforeEach(async () => {
-        tempDir = await mkdtemp(join(tmpdir(), 'ktx-search-conformance-'));
-        project = await initKtxProject({ projectDir: join(tempDir, 'project') });
-        dbPath = join(tempDir, '.ktx', 'db.sqlite');
-    });
-    afterEach(async () => {
-        await rm(tempDir, { recursive: true, force: true });
-    });
-    it('documents SQLite search backend capabilities', () => {
-        assertSearchBackendCapabilities({
-            backendName: 'sqlite',
-            capabilities: SQLITE_SEARCH_CAPABILITIES,
-            expected: {
-                fts: true,
-                vector: false,
-                fuzzy: false,
-                jsonSearch: true,
-                arraySearch: false,
-            },
-        });
-    });
-    it('keeps semantic-layer global ranking, dictionary evidence, and token fallback stable', async () => {
-        await seedSemanticLayerProject(project);
-        const global = await searchLocalSlSources(project, { query: 'orders', limit: 5 });
-        assertSearchBackendConformanceCase({
-            backendName: 'sqlite',
-            surface: 'semantic-layer',
-            caseName: 'global source ranking',
-            results: global.map(toSlConformanceResult),
-            expectedTopIds: ['finance/orders', 'warehouse/orders'],
-            expectedReasonsById: {
-                'finance/orders': ['lexical'],
-                'warehouse/orders': ['lexical'],
-            },
-            expectedLanes: {
-                lexical: { status: 'available' },
-                semantic: { status: 'skipped', reason: 'embedding_unconfigured' },
-            },
-        });
-        const dictionary = await searchLocalSlSources(project, {
-            connectionId: 'warehouse',
-            query: 'refunded',
-            limit: 5,
-        });
-        assertSearchBackendConformanceCase({
-            backendName: 'sqlite',
-            surface: 'semantic-layer',
-            caseName: 'dictionary source evidence',
-            results: dictionary.map(toSlConformanceResult),
-            expectedTopIds: ['warehouse/orders'],
-            expectedReasonsById: {
-                'warehouse/orders': ['dictionary'],
-            },
-            expectedLanes: {
-                dictionary: { status: 'available' },
-                semantic: { status: 'skipped', reason: 'embedding_unconfigured' },
-            },
-            expectedDictionaryMatchesById: {
-                'warehouse/orders': [{ column: 'status', values: ['refunded'] }],
-            },
-        });
-        const token = await searchLocalSlSources(project, {
-            connectionId: 'warehouse',
-            query: 'orders---',
-            limit: 5,
-        });
-        assertSearchBackendConformanceCase({
-            backendName: 'sqlite',
-            surface: 'semantic-layer',
-            caseName: 'token fallback reason',
-            results: token.map(toSlConformanceResult),
-            expectedTopIds: ['warehouse/orders'],
-            expectedReasonsById: {
-                'warehouse/orders': ['token'],
-            },
-            expectedLanes: {
-                token: { status: 'available' },
-            },
-        });
-    });
-    it('keeps wiki lexical, semantic, and token behavior stable', async () => {
-        await seedWikiProject(project);
-        const lexical = await searchLocalKnowledgePages(project, {
-            query: 'paid order',
-            userId: 'local',
-            limit: 5,
-        });
-        assertSearchBackendConformanceCase({
-            backendName: 'sqlite',
-            surface: 'wiki',
-            caseName: 'lexical page ranking',
-            results: lexical.map(toWikiConformanceResult),
-            expectedTopIds: ['metrics-revenue'],
-            expectedReasonsById: {
-                'metrics-revenue': ['lexical'],
-            },
-            expectedLanes: {
-                lexical: { status: 'available' },
-                semantic: { status: 'skipped', reason: 'embedding_unconfigured' },
-            },
-        });
-        const semantic = await searchLocalKnowledgePages(project, {
-            query: 'semantic revenue',
-            userId: 'local',
-            limit: 5,
-            embeddingService: new FakeEmbeddingPort(),
-        });
-        assertSearchBackendConformanceCase({
-            backendName: 'sqlite',
-            surface: 'wiki',
-            caseName: 'semantic page ranking',
-            results: semantic.map(toWikiConformanceResult),
-            expectedTopIds: ['metrics-revenue'],
-            expectedReasonsById: {
-                'metrics-revenue': ['semantic'],
-            },
-            expectedLanes: {
-                semantic: { status: 'available' },
-            },
-        });
-        const token = await searchLocalKnowledgePages(project, {
-            query: 'paid---',
-            userId: 'local',
-            limit: 5,
-        });
-        assertSearchBackendConformanceCase({
-            backendName: 'sqlite',
-            surface: 'wiki',
-            caseName: 'token page fallback',
-            results: token.map(toWikiConformanceResult),
-            expectedTopIds: ['metrics-revenue'],
-            expectedReasonsById: {
-                'metrics-revenue': ['token'],
-            },
-            expectedLanes: {
-                token: { status: 'available' },
-            },
-        });
-    });
-    it('keeps context-evidence lane fusion and token fallback stable', async () => {
-        const subject = new SqliteContextEvidenceStore({ dbPath });
-        await seedContextDocument(subject, {
-            externalId: 'page-discount',
-            title: 'Enterprise Discount Policy',
-            content: 'Enterprise discounts require finance approval before quote approval.',
-            searchText: 'enterprise discount finance approval quote',
-            embedding: [1, 0, 0],
-        });
-        await seedContextDocument(subject, {
-            externalId: 'page-owner',
-            title: 'Accountable Owner Policy',
-            content: 'Every policy has an accountable owner and review date.',
-            searchText: 'accountable owner review date',
-            embedding: [0.95, 0.05, 0],
-        });
-        await seedContextDocument(subject, {
-            externalId: 'page-expense',
-            title: 'Expense Policy',
-            content: 'Expense reimbursement requires receipt review.',
-            searchText: 'expense reimbursement receipt review',
-            embedding: [0, 1, 0],
-        });
-        const fused = await subject.searchRRF({
-            connectionId: 'conn-1',
-            sourceKey: 'notion',
-            queryEmbedding: [1, 0, 0],
-            queryText: 'enterprise discount approval',
-            limit: 2,
-            includeDeleted: false,
-        });
-        assertSearchBackendConformanceCase({
-            backendName: 'sqlite',
-            surface: 'context-evidence',
-            caseName: 'chunk lane fusion',
-            results: fused.map(toContextConformanceResult),
-            expectedTopIds: ['page-discount:notion:page-discount:intro'],
-            expectedReasonsById: {
-                'page-discount:notion:page-discount:intro': ['lexical', 'semantic', 'token'],
-            },
-            expectedLanes: {
-                lexical: { status: 'available' },
-                semantic: { status: 'available' },
-                token: { status: 'available' },
-            },
-        });
-        const tokenSubject = new SqliteContextEvidenceStore({ dbPath: join(tempDir, 'token.sqlite') });
-        await seedContextDocument(tokenSubject, {
-            externalId: 'page-cpp',
-            title: 'C++ Warehouse Notes',
-            content: 'C++ parser notes for warehouse extraction.',
-            searchText: 'C++ parser warehouse extraction',
-            embedding: null,
-        });
-        const token = await tokenSubject.searchRRF({
-            connectionId: 'conn-1',
-            sourceKey: 'notion',
-            queryEmbedding: null,
-            queryText: '++',
-            limit: 5,
-            includeDeleted: false,
-        });
-        assertSearchBackendConformanceCase({
-            backendName: 'sqlite',
-            surface: 'context-evidence',
-            caseName: 'fts-empty token fallback',
-            results: token.map(toContextConformanceResult),
-            expectedTopIds: ['page-cpp:notion:page-cpp:intro'],
-            expectedReasonsById: {
-                'page-cpp:notion:page-cpp:intro': ['token'],
-            },
-            expectedLanes: {
-                lexical: { status: 'skipped', reason: 'fts_query_empty' },
-                semantic: { status: 'skipped', reason: 'embedding_unconfigured' },
-                token: { status: 'available' },
-            },
-        });
-    });
-});

package/dist/context/search/discover.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/context/search/discover.test.js

@@ -1,197 +0,0 @@
-import { mkdtemp, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-import { initKtxProject } from '../../context/project/project.js';
-import { writeLocalKnowledgePage } from '../wiki/local-knowledge.js';
-import { createKtxDiscoverDataService } from './discover.js';
-describe('createKtxDiscoverDataService', () => {
-    let tempDir;
-    let project;
-    beforeEach(async () => {
-        tempDir = await mkdtemp(join(tmpdir(), 'ktx-discover-data-'));
-        project = await initKtxProject({ projectDir: join(tempDir, 'project') });
-        project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL' };
-        project.config.connections.billing = { driver: 'postgres', url: 'env:BILLING_DATABASE_URL' };
-    });
-    afterEach(async () => {
-        await rm(tempDir, { recursive: true, force: true });
-    });
-    async function seedWiki() {
-        await writeLocalKnowledgePage(project, {
-            key: 'orders-playbook',
-            scope: 'GLOBAL',
-            summary: 'Paid order operations',
-            content: 'Use paid orders and order_count to inspect monthly customer activity for Acme Corp.',
-            tags: ['orders'],
-        });
-    }
-    async function seedSl() {
-        await project.fileStore.writeFile('semantic-layer/warehouse/orders.yaml', [
-            'name: orders',
-            'descriptions:',
-            '  user: Paid order facts',
-            'table: public.orders',
-            'grain: [id]',
-            'columns:',
-            '  - name: status',
-            '    type: string',
-            '    descriptions:',
-            '      user: Payment status for the order',
-            '  - name: ordered_at',
-            '    type: time',
-            'measures:',
-            '  - name: order_count',
-            '    expr: count(*)',
-            '    description: Number of paid orders',
-            '',
-        ].join('\n'), 'ktx', 'ktx@example.com', 'seed sl source');
-    }
-    async function seedScan(input) {
-        const connectionId = input.connectionId ?? 'warehouse';
-        const root = `raw-sources/${connectionId}/live-database/${input.syncId}`;
-        const tableName = input.tableName ?? 'orders';
-        await project.fileStore.writeFile(`${root}/connection.json`, JSON.stringify({
-            connectionId,
-            driver: 'postgres',
-            extractedAt: `2026-05-14T09:00:00.000Z`,
-            scope: { schemas: ['public'] },
-        }, null, 2), 'ktx', 'ktx@example.com', 'seed scan connection');
-        await project.fileStore.writeFile(`${root}/tables/public-${tableName}.json`, JSON.stringify({
-            catalog: null,
-            db: 'public',
-            name: tableName,
-            kind: 'table',
-            comment: input.comment ?? 'Orders table from warehouse',
-            estimatedRows: 123,
-            descriptions: { db: input.comment ?? 'Orders table from warehouse' },
-            columns: [
-                {
-                    name: 'id',
-                    nativeType: 'integer',
-                    normalizedType: 'integer',
-                    dimensionType: 'number',
-                    nullable: false,
-                    primaryKey: true,
-                    comment: 'Order id',
-                },
-                {
-                    name: 'status',
-                    nativeType: 'text',
-                    normalizedType: 'text',
-                    dimensionType: 'string',
-                    nullable: false,
-                    primaryKey: false,
-                    comment: 'Order status',
-                    sampleValues: input.sampleValues ?? ['paid', 'pending'],
-                },
-            ],
-            foreignKeys: [],
-        }, null, 2), 'ktx', 'ktx@example.com', 'seed table');
-        await project.fileStore.writeFile(`${root}/scan-report.json`, JSON.stringify({
-            connectionId,
-            driver: 'postgres',
-            syncId: input.syncId,
-            runId: `scan-${input.syncId}`,
-            trigger: 'mcp',
-            mode: 'enriched',
-            dryRun: false,
-            artifactPaths: {
-                rawSourcesDir: root,
-                reportPath: `${root}/scan-report.json`,
-                manifestShards: [],
-                enrichmentArtifacts: [],
-            },
-            diffSummary: {
-                tablesAdded: 1,
-                tablesModified: 0,
-                tablesDeleted: 0,
-                tablesUnchanged: 0,
-                columnsAdded: 0,
-                columnsModified: 0,
-                columnsDeleted: 0,
-            },
-            manifestShardsWritten: 0,
-            structuralSyncStats: {
-                tablesCreated: 0,
-                tablesUpdated: 0,
-                tablesDeleted: 0,
-                columnsCreated: 0,
-                columnsUpdated: 0,
-                columnsDeleted: 0,
-            },
-            enrichment: {
-                dataDictionary: 'completed',
-                tableDescriptions: 'completed',
-                columnDescriptions: 'completed',
-                embeddings: 'skipped',
-                deterministicRelationships: 'skipped',
-                llmRelationshipValidation: 'skipped',
-                statisticalValidation: 'skipped',
-            },
-            capabilityGaps: [],
-            warnings: [],
-            relationships: { accepted: 0, review: 0, rejected: 0, skipped: 0 },
-            enrichmentState: { resumedStages: [], completedStages: [], failedStages: [] },
-            createdAt: '2026-05-14T09:00:00.000Z',
-        }, null, 2), 'ktx', 'ktx@example.com', 'seed scan report');
-    }
-    it('returns unified ranked refs across wiki, semantic-layer, and raw schema', async () => {
-        await seedWiki();
-        await seedSl();
-        await seedScan({ syncId: 'sync-1', sampleValues: ['paid', 'refunded'] });
-        const service = createKtxDiscoverDataService(project, { userId: 'local-user' });
-        const results = await service.search({ query: 'paid orders', connectionId: 'warehouse', limit: 10 });
-        expect(results.map((result) => result.kind)).toEqual(expect.arrayContaining(['wiki', 'sl_source', 'sl_measure', 'sl_dimension', 'table', 'column']));
-        expect(results.every((result) => result.score >= 0 && result.score <= 1)).toBe(true);
-        expect(results.every((result) => result.snippet === null || result.snippet.length <= 200)).toBe(true);
-        expect(results).toContainEqual(expect.objectContaining({
-            kind: 'table',
-            id: 'public.orders',
-            connectionId: 'warehouse',
-            tableRef: { catalog: null, db: 'public', name: 'orders' },
-            matchedOn: expect.stringMatching(/name|description|comment|display/),
-        }));
-        expect(results).toContainEqual(expect.objectContaining({
-            kind: 'column',
-            id: 'public.orders.status',
-            connectionId: 'warehouse',
-            columnName: 'status',
-            matchedOn: expect.stringMatching(/name|comment|description|sample_value/),
-        }));
-        expect(results).toContainEqual(expect.objectContaining({
-            kind: 'sl_measure',
-            id: 'orders.order_count',
-            connectionId: 'warehouse',
-            summary: 'Number of paid orders',
-            snippet: 'count(*)',
-            matchedOn: expect.stringMatching(/name|description|expr/),
-        }));
-    });
-    it('honors kind filters and connection scope', async () => {
-        await seedWiki();
-        await seedSl();
-        await seedScan({ syncId: 'sync-1', connectionId: 'warehouse', tableName: 'orders' });
-        await seedScan({ syncId: 'sync-2', connectionId: 'billing', tableName: 'invoices', comment: 'Billing invoices' });
-        const service = createKtxDiscoverDataService(project);
-        const results = await service.search({
-            query: 'orders',
-            connectionId: 'warehouse',
-            kinds: ['table', 'column'],
-            limit: 10,
-        });
-        expect(results.every((result) => result.kind === 'table' || result.kind === 'column')).toBe(true);
-        expect(results.every((result) => result.connectionId === 'warehouse')).toBe(true);
-        expect(results.some((result) => result.id.includes('invoices'))).toBe(false);
-        expect(results.some((result) => result.kind === 'wiki')).toBe(false);
-    });
-    it('re-reads the latest scan artifacts on each call', async () => {
-        await seedScan({ syncId: 'sync-1', tableName: 'orders', comment: 'Old orders table' });
-        const service = createKtxDiscoverDataService(project);
-        await expect(service.search({ query: 'orders', connectionId: 'warehouse', kinds: ['table'], limit: 10 })).resolves.toEqual(expect.arrayContaining([expect.objectContaining({ id: 'public.orders' })]));
-        await seedScan({ syncId: 'sync-2', tableName: 'invoices', comment: 'Invoice facts' });
-        const fresh = await service.search({ query: 'invoice', connectionId: 'warehouse', kinds: ['table'], limit: 10 });
-        expect(fresh).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'public.invoices' })]));
-        expect(fresh.some((result) => result.id === 'public.orders')).toBe(false);
-    });
-});

package/dist/context/search/hybrid-search-core.test.d.ts

@@ -1 +0,0 @@
-export {};