@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/node_modules/@ktx/context/dist/sl/sl-dictionary-profile.js

@@ -0,0 +1,80 @@
+import { defaultKtxDataDictionarySettings, isKtxDataDictionaryCandidate } from '../scan/index.js';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function parseProfile(raw) {
+    const parsed = JSON.parse(raw);
+    if (!isRecord(parsed)) {
+        return null;
+    }
+    return parsed;
+}
+function normalizedValues(values) {
+    const seen = new Set();
+    const result = [];
+    for (const value of values ?? []) {
+        const text = String(value).trim();
+        const key = text.toLowerCase();
+        if (text.length === 0 || seen.has(key)) {
+            continue;
+        }
+        seen.add(key);
+        result.push(text);
+    }
+    return result.sort((left, right) => left.localeCompare(right));
+}
+function columnEntries(connectionId, column) {
+    const sourceName = column.table?.name;
+    const columnName = column.column;
+    if (!sourceName || !columnName) {
+        return [];
+    }
+    const columnType = column.normalizedType ?? column.nativeType ?? '';
+    if (!isKtxDataDictionaryCandidate(columnType, columnName)) {
+        return [];
+    }
+    const cardinality = typeof column.distinctCount === 'number' ? column.distinctCount : null;
+    if (cardinality !== null && cardinality > defaultKtxDataDictionarySettings.cardinalityThreshold) {
+        return [];
+    }
+    return normalizedValues(column.sampleValues).map((value) => ({
+        connectionId,
+        sourceName,
+        columnName,
+        value,
+        cardinality,
+    }));
+}
+async function latestProfilePath(project, connectionId) {
+    const root = `raw-sources/${connectionId}/live-database`;
+    let files;
+    try {
+        files = (await project.fileStore.listFiles(root)).files;
+    }
+    catch {
+        return null;
+    }
+    return (files
+        .filter((path) => path.endsWith('/enrichment/relationship-profile.json'))
+        .sort((left, right) => left.localeCompare(right))
+        .at(-1) ?? null);
+}
+export async function loadLatestSlDictionaryEntries(project, connectionIds) {
+    const entries = [];
+    for (const connectionId of [...new Set(connectionIds)].sort()) {
+        const path = await latestProfilePath(project, connectionId);
+        if (!path) {
+            continue;
+        }
+        const raw = await project.fileStore.readFile(path);
+        const profile = parseProfile(raw.content);
+        const profileConnectionId = profile?.connectionId ?? connectionId;
+        for (const column of Object.values(profile?.columns ?? {})) {
+            entries.push(...columnEntries(profileConnectionId, column));
+        }
+    }
+    return entries.sort((left, right) => left.connectionId.localeCompare(right.connectionId) ||
+        left.sourceName.localeCompare(right.sourceName) ||
+        left.columnName.localeCompare(right.columnName) ||
+        left.value.localeCompare(right.value));
+}

package/node_modules/@ktx/context/dist/sl/sl-dictionary-profile.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/sl/sl-dictionary-profile.test.js

@@ -0,0 +1,88 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { initKtxProject } from '../project/index.js';
+import { loadLatestSlDictionaryEntries } from './sl-dictionary-profile.js';
+describe('loadLatestSlDictionaryEntries', () => {
+    let tempDir;
+    let project;
+    beforeEach(async () => {
+        tempDir = await mkdtemp(join(tmpdir(), 'ktx-sl-dictionary-profile-'));
+        project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' });
+    });
+    afterEach(async () => {
+        await rm(tempDir, { recursive: true, force: true });
+    });
+    it('loads latest relationship-profile sample values for dictionary candidate columns', async () => {
+        await project.fileStore.writeFile('raw-sources/warehouse/live-database/sync-1/enrichment/relationship-profile.json', `${JSON.stringify({
+            connectionId: 'warehouse',
+            driver: 'postgres',
+            sqlAvailable: true,
+            queryCount: 4,
+            tables: [],
+            columns: {
+                'orders.status': {
+                    table: { catalog: null, db: 'public', name: 'orders' },
+                    column: 'status',
+                    nativeType: 'text',
+                    normalizedType: 'string',
+                    rowCount: 20,
+                    nullCount: 0,
+                    distinctCount: 3,
+                    uniquenessRatio: 0.15,
+                    nullRate: 0,
+                    sampleValues: ['paid', 'refunded', 'pending'],
+                    minTextLength: 4,
+                    maxTextLength: 8,
+                },
+                'orders.customer_id': {
+                    table: { catalog: null, db: 'public', name: 'orders' },
+                    column: 'customer_id',
+                    nativeType: 'text',
+                    normalizedType: 'string',
+                    rowCount: 20,
+                    nullCount: 0,
+                    distinctCount: 20,
+                    uniquenessRatio: 1,
+                    nullRate: 0,
+                    sampleValues: ['cus_1'],
+                    minTextLength: 5,
+                    maxTextLength: 5,
+                },
+            },
+            warnings: [],
+        }, null, 2)}\n`, 'ktx', 'ktx@example.com', 'Seed profile');
+        await project.fileStore.writeFile('raw-sources/warehouse/live-database/sync-2/enrichment/relationship-profile.json', `${JSON.stringify({
+            connectionId: 'warehouse',
+            driver: 'postgres',
+            sqlAvailable: true,
+            queryCount: 4,
+            tables: [],
+            columns: {
+                'orders.status': {
+                    table: { catalog: null, db: 'public', name: 'orders' },
+                    column: 'status',
+                    nativeType: 'text',
+                    normalizedType: 'string',
+                    rowCount: 20,
+                    nullCount: 0,
+                    distinctCount: 2,
+                    uniquenessRatio: 0.1,
+                    nullRate: 0,
+                    sampleValues: ['settled', 'voided'],
+                    minTextLength: 6,
+                    maxTextLength: 7,
+                },
+            },
+            warnings: [],
+        }, null, 2)}\n`, 'ktx', 'ktx@example.com', 'Seed newer profile');
+        await expect(loadLatestSlDictionaryEntries(project, ['warehouse'])).resolves.toEqual([
+            { connectionId: 'warehouse', sourceName: 'orders', columnName: 'status', value: 'settled', cardinality: 2 },
+            { connectionId: 'warehouse', sourceName: 'orders', columnName: 'status', value: 'voided', cardinality: 2 },
+        ]);
+    });
+    it('returns an empty list when no relationship profile exists', async () => {
+        await expect(loadLatestSlDictionaryEntries(project, ['warehouse'])).resolves.toEqual([]);
+    });
+});

package/node_modules/@ktx/context/dist/sl/sl-search.service.d.ts

@@ -0,0 +1,16 @@
+import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js';
+import type { SlSourcesIndexPort } from './ports.js';
+import type { SemanticLayerSource } from './types.js';
+export declare function buildSemanticLayerSourceSearchText(source: SemanticLayerSource, priority?: string[]): string;
+export declare class SlSearchService {
+    private readonly embeddingService;
+    private readonly slSourcesRepository;
+    private readonly logger;
+    constructor(embeddingService: KtxEmbeddingPort, slSourcesRepository: SlSourcesIndexPort, logger?: KtxLogger);
+    indexSources(connectionId: string, sources: SemanticLayerSource[]): Promise<void>;
+    search(connectionId: string, query: string, limit?: number, minRrfScore?: number): Promise<Array<{
+        sourceName: string;
+        score: number;
+    }>>;
+    buildSearchText(source: SemanticLayerSource, priority?: string[]): string;
+}

package/node_modules/@ktx/context/dist/sl/sl-search.service.js

@@ -0,0 +1,133 @@
+import { noopLogger } from '../core/index.js';
+import { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js';
+export function buildSemanticLayerSourceSearchText(source, priority = DEFAULT_PRIORITY) {
+    const config = { priority };
+    const parts = [source.name.replace(/_/g, ' ')];
+    const sourceDesc = resolveDescription(source.descriptions, config);
+    if (sourceDesc) {
+        parts.push(sourceDesc);
+    }
+    if (source.table) {
+        parts.push(`table: ${source.table}`);
+    }
+    if (source.default_time_dimension?.dbt) {
+        parts.push(`dbt default time: ${source.default_time_dimension.dbt}`);
+    }
+    for (const col of source.columns ?? []) {
+        const colDesc = resolveDescription(col.descriptions, config);
+        let extra = '';
+        if (col.enum_values?.dbt?.length) {
+            extra += ` [dbt values: ${col.enum_values.dbt.join(', ')}]`;
+        }
+        if (col.constraints?.dbt?.not_null) {
+            extra += ' not_null';
+        }
+        if (col.constraints?.dbt?.unique) {
+            extra += ' unique';
+        }
+        parts.push(`${col.name} (${col.type})${colDesc ? ` ${colDesc}` : ''}${extra}`);
+    }
+    for (const m of source.measures ?? []) {
+        parts.push(`measure: ${m.name} ${m.expr}${m.description ? ` ${m.description}` : ''}`);
+    }
+    for (const j of source.joins ?? []) {
+        parts.push(`join: ${j.to} (${j.relationship})`);
+    }
+    if (source.tags?.dbt?.length) {
+        parts.push(`dbt tags: ${source.tags.dbt.join(', ')}`);
+    }
+    if (source.freshness?.dbt) {
+        const fd = source.freshness.dbt;
+        const bits = [];
+        if (fd.loaded_at_field) {
+            bits.push(`loaded_at=${fd.loaded_at_field}`);
+        }
+        if (fd.raw !== undefined) {
+            let rawStr = JSON.stringify(fd.raw);
+            if (rawStr.length > 120) {
+                rawStr = `${rawStr.slice(0, 117)}...`;
+            }
+            bits.push(rawStr);
+        }
+        if (bits.length > 0) {
+            parts.push(`dbt freshness: ${bits.join(' ')}`);
+        }
+    }
+    return parts.join('. ');
+}
+export class SlSearchService {
+    embeddingService;
+    slSourcesRepository;
+    logger;
+    constructor(embeddingService, slSourcesRepository, logger = noopLogger) {
+        this.embeddingService = embeddingService;
+        this.slSourcesRepository = slSourcesRepository;
+        this.logger = logger;
+    }
+    async indexSources(connectionId, sources) {
+        if (sources.length === 0) {
+            await this.slSourcesRepository.deleteByConnection(connectionId);
+            return;
+        }
+        // Detect which sources actually changed by comparing search_text
+        const existing = await this.slSourcesRepository.getExistingSearchTexts(connectionId);
+        const searchTexts = sources.map((s) => this.buildSearchText(s));
+        const changedIndices = [];
+        for (let i = 0; i < sources.length; i++) {
+            const prev = existing.get(sources[i].name);
+            if (!prev || prev.searchText !== searchTexts[i] || !prev.hasEmbedding) {
+                changedIndices.push(i);
+            }
+        }
+        if (changedIndices.length === 0) {
+            // Still clean up stale sources even if nothing changed
+            const keepNames = sources.map((s) => s.name);
+            await this.slSourcesRepository.deleteStale(connectionId, keepNames);
+            this.logger.log(`SL sources for connection ${connectionId}: all ${sources.length} up to date, 0 reindexed`);
+            return;
+        }
+        // Compute embeddings only for changed sources
+        const changedTexts = changedIndices.map((i) => searchTexts[i]);
+        let changedEmbeddings;
+        try {
+            const batchSize = this.embeddingService.maxBatchSize;
+            const allEmbeddings = [];
+            for (let i = 0; i < changedTexts.length; i += batchSize) {
+                const batch = changedTexts.slice(i, i + batchSize);
+                const batchEmbeddings = await this.embeddingService.computeEmbeddingsBulk(batch);
+                allEmbeddings.push(...batchEmbeddings);
+            }
+            changedEmbeddings = allEmbeddings;
+        }
+        catch (error) {
+            this.logger.warn(`Failed to compute SL source embeddings: ${error instanceof Error ? error.message : String(error)}`);
+            changedEmbeddings = changedIndices.map(() => null);
+        }
+        const rows = changedIndices.map((srcIdx, i) => {
+            return {
+                sourceName: sources[srcIdx].name,
+                searchText: searchTexts[srcIdx],
+                embedding: changedEmbeddings[i],
+            };
+        });
+        await this.slSourcesRepository.upsertSources(connectionId, rows);
+        // Remove sources that no longer exist in YAML
+        const keepNames = sources.map((s) => s.name);
+        await this.slSourcesRepository.deleteStale(connectionId, keepNames);
+        this.logger.log(`SL sources for connection ${connectionId}: ${changedIndices.length}/${sources.length} reindexed, ${sources.length - changedIndices.length} unchanged`);
+    }
+    async search(connectionId, query, limit = 15, minRrfScore = 0) {
+        let queryEmbedding = null;
+        try {
+            queryEmbedding = await this.embeddingService.computeEmbedding(query);
+        }
+        catch (error) {
+            this.logger.warn(`Failed to compute query embedding, falling back to FTS + trigram: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        const results = await this.slSourcesRepository.search(connectionId, queryEmbedding, query, limit, minRrfScore);
+        return results.map((r) => ({ sourceName: r.sourceName, score: r.rrfScore }));
+    }
+    buildSearchText(source, priority = DEFAULT_PRIORITY) {
+        return buildSemanticLayerSourceSearchText(source, priority);
+    }
+}

package/node_modules/@ktx/context/dist/sl/sl-search.service.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/sl/sl-search.service.test.js

@@ -0,0 +1,142 @@
+import { describe, expect, it, vi } from 'vitest';
+import { buildSemanticLayerSourceSearchText, SlSearchService } from './sl-search.service.js';
+describe('SlSearchService', () => {
+    it('builds search text from source, columns, measures, and joins', () => {
+        const service = new SlSearchService({ maxBatchSize: 16, computeEmbedding: vi.fn(), computeEmbeddingsBulk: vi.fn() }, {
+            upsertSources: vi.fn(),
+            getExistingSearchTexts: vi.fn(),
+            deleteStale: vi.fn(),
+            deleteByConnection: vi.fn(),
+            deleteByConnectionAndName: vi.fn(),
+            search: vi.fn(),
+        });
+        const source = {
+            name: 'orders',
+            descriptions: { user: 'Customer orders' },
+            table: 'public.orders',
+            grain: ['id'],
+            columns: [
+                { name: 'id', type: 'string' },
+                { name: 'amount', type: 'number', descriptions: { user: 'Order amount' } },
+            ],
+            measures: [{ name: 'revenue', expr: 'sum(amount)', description: 'Gross revenue' }],
+            joins: [{ to: 'customers', on: 'orders.customer_id = customers.id', relationship: 'many_to_one' }],
+        };
+        expect(service.buildSearchText(source)).toContain('orders');
+        expect(service.buildSearchText(source)).toContain('Customer orders');
+        expect(service.buildSearchText(source)).toContain('amount (number) Order amount');
+        expect(service.buildSearchText(source)).toContain('measure: revenue sum(amount) Gross revenue');
+        expect(service.buildSearchText(source)).toContain('join: customers (many_to_one)');
+    });
+    it('exports the same canonical search text builder used by SlSearchService', () => {
+        const service = new SlSearchService({ maxBatchSize: 16, computeEmbedding: vi.fn(), computeEmbeddingsBulk: vi.fn() }, {
+            upsertSources: vi.fn(),
+            getExistingSearchTexts: vi.fn(),
+            deleteStale: vi.fn(),
+            deleteByConnection: vi.fn(),
+            deleteByConnectionAndName: vi.fn(),
+            search: vi.fn(),
+        });
+        const source = {
+            name: 'orders',
+            descriptions: { user: 'Customer orders' },
+            table: 'public.orders',
+            grain: ['id'],
+            columns: [
+                {
+                    name: 'status',
+                    type: 'string',
+                    enum_values: { dbt: ['paid', 'refunded'] },
+                    constraints: { dbt: { not_null: true } },
+                },
+            ],
+            joins: [{ to: 'customers', on: 'orders.customer_id = customers.id', relationship: 'many_to_one' }],
+            measures: [{ name: 'total_revenue', expr: 'sum(revenue)', description: 'Gross revenue' }],
+            tags: { dbt: ['finance'] },
+        };
+        expect(buildSemanticLayerSourceSearchText(source)).toBe(service.buildSearchText(source));
+        expect(buildSemanticLayerSourceSearchText(source)).toContain('dbt values: paid, refunded');
+        expect(buildSemanticLayerSourceSearchText(source)).toContain('measure: total_revenue sum(revenue) Gross revenue');
+        expect(buildSemanticLayerSourceSearchText(source)).toContain('dbt tags: finance');
+    });
+    it('includes dbt enum, not_null, and unique tokens for columns', () => {
+        const service = new SlSearchService({ maxBatchSize: 16, computeEmbedding: vi.fn(), computeEmbeddingsBulk: vi.fn() }, {
+            upsertSources: vi.fn(),
+            getExistingSearchTexts: vi.fn(),
+            deleteStale: vi.fn(),
+            deleteByConnection: vi.fn(),
+            deleteByConnectionAndName: vi.fn(),
+            search: vi.fn(),
+        });
+        const source = {
+            name: 'src_orders',
+            table: 'public.orders',
+            grain: [],
+            columns: [
+                {
+                    name: 'status',
+                    type: 'string',
+                    descriptions: {},
+                    enum_values: { dbt: ['a', 'b'] },
+                    constraints: { dbt: { not_null: true, unique: true } },
+                },
+            ],
+            joins: [],
+            measures: [],
+        };
+        const text = service.buildSearchText(source);
+        expect(text).toContain('dbt values: a, b');
+        expect(text).toContain('not_null');
+        expect(text).toContain('unique');
+    });
+    it('includes dbt default time token for MetricFlow agg_time_dimension', () => {
+        const service = new SlSearchService({ maxBatchSize: 16, computeEmbedding: vi.fn(), computeEmbeddingsBulk: vi.fn() }, {
+            upsertSources: vi.fn(),
+            getExistingSearchTexts: vi.fn(),
+            deleteStale: vi.fn(),
+            deleteByConnection: vi.fn(),
+            deleteByConnectionAndName: vi.fn(),
+            search: vi.fn(),
+        });
+        const source = {
+            name: 'orders',
+            table: 'public.orders',
+            grain: ['id'],
+            columns: [{ name: 'id', type: 'number' }],
+            joins: [],
+            measures: [],
+            default_time_dimension: { dbt: 'order_date' },
+        };
+        expect(service.buildSearchText(source)).toContain('dbt default time: order_date');
+    });
+    it('includes dbt table tags and freshness from manifest-backed source', () => {
+        const service = new SlSearchService({ maxBatchSize: 16, computeEmbedding: vi.fn(), computeEmbeddingsBulk: vi.fn() }, {
+            upsertSources: vi.fn(),
+            getExistingSearchTexts: vi.fn(),
+            deleteStale: vi.fn(),
+            deleteByConnection: vi.fn(),
+            deleteByConnectionAndName: vi.fn(),
+            search: vi.fn(),
+        });
+        const source = {
+            name: 'customers',
+            table: 'jaffle.customers',
+            grain: ['id'],
+            columns: [{ name: 'id', type: 'number' }],
+            joins: [],
+            measures: [],
+            tags: { dbt: ['raw', 'core'] },
+            freshness: {
+                dbt: {
+                    loaded_at_field: 'updated_at',
+                    raw: { warn_after: { count: 12, period: 'hour' } },
+                },
+            },
+        };
+        const text = service.buildSearchText(source);
+        expect(text).toContain('dbt tags: raw, core');
+        expect(text).toContain('dbt freshness:');
+        expect(text).toContain('loaded_at=updated_at');
+        expect(text).toContain('warn_after');
+    });
+});

package/node_modules/@ktx/context/dist/sl/sl-validator.port.d.ts

@@ -0,0 +1,7 @@
+export interface SlValidationResult {
+    errors: string[];
+    warnings: string[];
+}
+export interface SlValidatorPort<TDeps = unknown> {
+    validateSingleSource(deps: TDeps, connectionId: string, sourceName: string): Promise<SlValidationResult>;
+}

package/node_modules/@ktx/context/dist/sl/sl-validator.port.js

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/sl/sqlite-sl-sources-index.d.ts

@@ -0,0 +1,54 @@
+import type { SlSourcesIndexPort } from './ports.js';
+import type { SlDictionaryEntry } from './sl-dictionary-profile.js';
+import type { SlDictionaryMatch } from './types.js';
+export interface SqliteSlSourcesIndexOptions {
+    dbPath: string;
+}
+export interface SlSqliteLaneCandidate {
+    id: string;
+    connectionId: string;
+    sourceName: string;
+    rank: number;
+    rawScore: number;
+}
+export interface SlSqliteDictionaryCandidate extends SlSqliteLaneCandidate {
+    matches: SlDictionaryMatch[];
+}
+export declare class SqliteSlSourcesIndex implements SlSourcesIndexPort {
+    private readonly db;
+    constructor(options: SqliteSlSourcesIndexOptions);
+    upsertSources(connectionId: string, sources: Array<{
+        sourceName: string;
+        searchText: string;
+        embedding: number[] | null;
+        contentHash?: string | null;
+    }>): Promise<void>;
+    getExistingSearchTexts(connectionId: string): Promise<Map<string, {
+        searchText: string;
+        hasEmbedding: boolean;
+    }>>;
+    deleteStale(connectionId: string, keepNames: string[]): Promise<void>;
+    deleteByConnection(connectionId: string): Promise<void>;
+    deleteByConnectionAndName(connectionId: string, sourceName: string): Promise<void>;
+    replaceDictionaryEntries(connectionId: string, entries: SlDictionaryEntry[]): Promise<void>;
+    searchLexicalCandidates(input: {
+        connectionIds?: readonly string[];
+        queryText: string;
+        limit: number;
+    }): Promise<SlSqliteLaneCandidate[]>;
+    searchSemanticCandidates(input: {
+        connectionIds?: readonly string[];
+        queryEmbedding: number[];
+        limit: number;
+    }): Promise<SlSqliteLaneCandidate[]>;
+    searchDictionaryCandidates(input: {
+        connectionIds?: readonly string[];
+        queryText: string;
+        limit: number;
+    }): Promise<SlSqliteDictionaryCandidate[]>;
+    search(connectionId: string, _queryEmbedding: number[] | null, queryText: string, limit: number, minRrfScore?: number): Promise<Array<{
+        sourceName: string;
+        rrfScore: number;
+    }>>;
+    private deleteByConnectionAndNameSync;
+}