@kaelio/ktx 0.5.0 → 0.6.0

package/dist/context/scan/relationship-profiling.test.js

@@ -1,392 +0,0 @@
-import { readFile } from 'node:fs/promises';
-import { join } from 'node:path';
-import Database from 'better-sqlite3';
-import { afterEach, describe, expect, it, vi } from 'vitest';
-import { snapshotToKtxEnrichedSchema } from './local-enrichment.js';
-import { loadKtxRelationshipBenchmarkFixture, maskKtxRelationshipBenchmarkSnapshot } from './relationship-benchmarks.js';
-import { createKtxRelationshipProfileCache, formatKtxRelationshipTableRef, profileKtxRelationshipSchema, quoteKtxRelationshipIdentifier, } from './relationship-profiling.js';
-class InMemorySqliteExecutor {
-    db = new Database(':memory:');
-    queryCount = 0;
-    executeReadOnly(input, _ctx) {
-        this.queryCount += 1;
-        const rows = this.db.prepare(input.sql).all();
-        const headers = Object.keys(rows[0] ?? {});
-        return Promise.resolve({
-            headers,
-            rows: rows.map((row) => headers.map((header) => row[header])),
-            totalRows: rows.length,
-            rowCount: rows.length,
-        });
-    }
-    close() {
-        this.db.close();
-    }
-}
-class FileSqliteExecutor {
-    db;
-    queryCount = 0;
-    constructor(dataPath) {
-        this.db = new Database(dataPath, { readonly: true, fileMustExist: true });
-    }
-    executeReadOnly(input, _ctx) {
-        this.queryCount += 1;
-        const rows = this.db.prepare(input.sql).all();
-        const headers = Object.keys(rows[0] ?? {});
-        return Promise.resolve({
-            headers,
-            rows: rows.map((row) => headers.map((header) => row[header])),
-            totalRows: rows.length,
-            rowCount: rows.length,
-        });
-    }
-    close() {
-        this.db.close();
-    }
-}
-function column(tableId, name, overrides = {}) {
-    const tableRef = overrides.tableRef ?? { catalog: null, db: null, name: tableId };
-    return {
-        id: `${tableId}.${name}`,
-        tableId,
-        tableRef,
-        name,
-        nativeType: overrides.nativeType ?? 'INTEGER',
-        normalizedType: overrides.normalizedType ?? 'integer',
-        dimensionType: overrides.dimensionType ?? 'number',
-        nullable: overrides.nullable ?? true,
-        primaryKey: overrides.primaryKey ?? false,
-        parentColumnId: null,
-        descriptions: {},
-        embedding: null,
-        sampleValues: null,
-        cardinality: null,
-        ...overrides,
-    };
-}
-function table(name, columns) {
-    const ref = { catalog: null, db: null, name };
-    return {
-        id: name,
-        ref,
-        enabled: true,
-        descriptions: {},
-        columns: columns.map((item) => ({ ...item, tableId: name, tableRef: ref })),
-    };
-}
-function schema(tables) {
-    return { connectionId: 'warehouse', tables, relationships: [] };
-}
-describe('relationship profiling', () => {
-    let executor = null;
-    afterEach(() => {
-        executor?.close();
-        executor = null;
-    });
-    it('keeps profiling on the batched table path', async () => {
-        const source = await readFile(new URL('relationship-profiling.ts', import.meta.url), 'utf-8');
-        expect(source).not.toMatch(new RegExp('queryColumn' + 'Profile'));
-        expect(source).not.toMatch(/for \(const column of table\.columns\)[\s\S]*executeReadOnly/);
-        expect(source).toMatch(/queryTableProfile/);
-        expect(source).toMatch(/UNION ALL/);
-    });
-    it('quotes identifiers and formats table refs for supported local SQL drivers', () => {
-        expect(quoteKtxRelationshipIdentifier('sqlite', 'odd"name')).toBe('"odd""name"');
-        expect(quoteKtxRelationshipIdentifier('mysql', 'odd`name')).toBe('`odd``name`');
-        expect(quoteKtxRelationshipIdentifier('sqlserver', 'odd]name')).toBe('[odd]]name]');
-        expect(formatKtxRelationshipTableRef('sqlite', { catalog: null, db: null, name: 'accounts' })).toBe('"accounts"');
-        expect(formatKtxRelationshipTableRef('postgres', { catalog: null, db: 'analytics', name: 'accounts' })).toBe('"analytics"."accounts"');
-    });
-    it('profiles row count, null rate, uniqueness, sample values, and text lengths', async () => {
-        executor = new InMemorySqliteExecutor();
-        executor.db.exec(`
-      CREATE TABLE accounts (id INTEGER, code TEXT, parent_id INTEGER);
-      INSERT INTO accounts (id, code, parent_id) VALUES
-        (1, 'A-1', NULL),
-        (2, 'B-2', 1),
-        (3, 'C-3', 1),
-        (4, 'C-3', 2);
-    `);
-        const result = await profileKtxRelationshipSchema({
-            connectionId: 'warehouse',
-            driver: 'sqlite',
-            schema: schema([
-                table('accounts', [
-                    column('accounts', 'id', { primaryKey: false, nullable: false }),
-                    column('accounts', 'code', { nativeType: 'TEXT', normalizedType: 'text', dimensionType: 'string' }),
-                    column('accounts', 'parent_id'),
-                ]),
-            ]),
-            executor,
-            ctx: { runId: 'profile-test' },
-            sampleValuesPerColumn: 3,
-        });
-        expect(result.sqlAvailable).toBe(true);
-        expect(result.queryCount).toBe(1);
-        expect(executor.queryCount).toBe(1);
-        expect(result.tables).toHaveLength(1);
-        expect(result.tables[0]).toMatchObject({ table: { name: 'accounts' }, rowCount: 4 });
-        expect(result.columns['accounts.id']).toMatchObject({
-            table: { name: 'accounts' },
-            column: 'id',
-            rowCount: 4,
-            nullCount: 0,
-            distinctCount: 4,
-            uniquenessRatio: 1,
-            nullRate: 0,
-            minTextLength: 1,
-            maxTextLength: 1,
-        });
-        expect(result.columns['accounts.code']).toMatchObject({
-            distinctCount: 3,
-            uniquenessRatio: 0.75,
-            sampleValues: ['C-3', 'A-1', 'B-2'],
-            minTextLength: 3,
-            maxTextLength: 3,
-        });
-        expect(result.columns['accounts.parent_id']).toMatchObject({
-            nullCount: 1,
-            distinctCount: 2,
-            uniquenessRatio: 0.5,
-            nullRate: 0.25,
-        });
-    });
-    it('profiles each enabled table with one read-only SQL query', async () => {
-        executor = new InMemorySqliteExecutor();
-        executor.db.exec(`
-      CREATE TABLE accounts (id INTEGER, code TEXT, parent_id INTEGER);
-      CREATE TABLE users (id INTEGER, account_id INTEGER);
-      INSERT INTO accounts (id, code, parent_id) VALUES
-        (1, 'A-1', NULL),
-        (2, 'B-2', 1),
-        (3, 'C-3', 1),
-        (4, 'C-3', 2);
-      INSERT INTO users (id, account_id) VALUES
-        (10, 1),
-        (11, 1),
-        (12, 2);
-    `);
-        const result = await profileKtxRelationshipSchema({
-            connectionId: 'warehouse',
-            driver: 'sqlite',
-            schema: schema([
-                table('accounts', [
-                    column('accounts', 'id', { nullable: false }),
-                    column('accounts', 'code', { nativeType: 'TEXT', normalizedType: 'text', dimensionType: 'string' }),
-                    column('accounts', 'parent_id'),
-                ]),
-                table('users', [column('users', 'id', { nullable: false }), column('users', 'account_id')]),
-            ]),
-            executor,
-            ctx: { runId: 'profile-batched-query-count' },
-            sampleValuesPerColumn: 3,
-        });
-        expect(result.sqlAvailable).toBe(true);
-        expect(result.queryCount).toBe(2);
-        expect(executor.queryCount).toBe(2);
-        expect(result.tables).toEqual([
-            { table: { catalog: null, db: null, name: 'accounts' }, rowCount: 4 },
-            { table: { catalog: null, db: null, name: 'users' }, rowCount: 3 },
-        ]);
-        expect(result.columns['accounts.code']).toMatchObject({
-            distinctCount: 3,
-            uniquenessRatio: 0.75,
-            sampleValues: ['C-3', 'A-1', 'B-2'],
-        });
-        expect(result.columns['users.account_id']).toMatchObject({
-            rowCount: 3,
-            nullCount: 0,
-            distinctCount: 2,
-            uniquenessRatio: 2 / 3,
-        });
-    });
-    it('bounds column profile statistics with profileSampleRows', async () => {
-        const executor = new InMemorySqliteExecutor();
-        executor.db.exec(`
-      CREATE TABLE accounts (id INTEGER NOT NULL, account_code TEXT NOT NULL);
-      INSERT INTO accounts VALUES (1, 'a1'), (2, 'a2'), (3, 'a3'), (4, 'a4');
-    `);
-        const profiles = await profileKtxRelationshipSchema({
-            connectionId: 'warehouse',
-            driver: 'sqlite',
-            schema: schema([
-                table('accounts', [
-                    column('accounts', 'id', { nullable: false }),
-                    column('accounts', 'account_code', {
-                        nativeType: 'TEXT',
-                        normalizedType: 'text',
-                        dimensionType: 'string',
-                        nullable: false,
-                    }),
-                ]),
-            ]),
-            executor,
-            ctx: { runId: 'profile-sample-rows' },
-            profileSampleRows: 2,
-        });
-        expect(profiles.queryCount).toBe(1);
-        expect(executor.queryCount).toBe(1);
-        expect(profiles.tables).toEqual([{ table: { catalog: null, db: null, name: 'accounts' }, rowCount: 4 }]);
-        expect(profiles.columns['accounts.id']).toMatchObject({
-            rowCount: 2,
-            distinctCount: 2,
-            uniquenessRatio: 1,
-        });
-        expect(profiles.columns['accounts.account_code']?.sampleValues).toEqual(['a1', 'a2']);
-        executor.close();
-    });
-    it('reuses a profile cache inside one scan run but re-queries with a fresh cache', async () => {
-        executor = new InMemorySqliteExecutor();
-        executor.db.exec(`
-      CREATE TABLE accounts (id INTEGER NOT NULL, account_code TEXT NOT NULL);
-      INSERT INTO accounts VALUES (1, 'a1'), (2, 'a2'), (3, 'a2');
-    `);
-        const relationshipSchema = schema([
-            table('accounts', [
-                column('accounts', 'id', { nullable: false }),
-                column('accounts', 'account_code', {
-                    nativeType: 'TEXT',
-                    normalizedType: 'text',
-                    dimensionType: 'string',
-                    nullable: false,
-                }),
-            ]),
-        ]);
-        const cache = createKtxRelationshipProfileCache();
-        const first = await profileKtxRelationshipSchema({
-            connectionId: 'warehouse',
-            driver: 'sqlite',
-            schema: relationshipSchema,
-            executor,
-            ctx: { runId: 'profile-cache-run' },
-            cache,
-        });
-        const second = await profileKtxRelationshipSchema({
-            connectionId: 'warehouse',
-            driver: 'sqlite',
-            schema: relationshipSchema,
-            executor,
-            ctx: { runId: 'profile-cache-run' },
-            cache,
-        });
-        const third = await profileKtxRelationshipSchema({
-            connectionId: 'warehouse',
-            driver: 'sqlite',
-            schema: relationshipSchema,
-            executor,
-            ctx: { runId: 'profile-cache-fresh-run' },
-            cache: createKtxRelationshipProfileCache(),
-        });
-        expect(first.queryCount).toBe(1);
-        expect(second.queryCount).toBe(0);
-        expect(third.queryCount).toBe(1);
-        expect(executor.queryCount).toBe(2);
-        expect(second.tables).toEqual(first.tables);
-        expect(second.columns).toEqual(first.columns);
-    });
-    it('profiles the checked-in scale stress fixture with one query per table', async () => {
-        const fixtureRoot = new URL('../../test/fixtures/relationship-benchmarks/', import.meta.url);
-        const fixture = await loadKtxRelationshipBenchmarkFixture(join(fixtureRoot.pathname, 'scale_stress_no_declared_constraints'));
-        if (!fixture.dataPath) {
-            throw new Error('scale_stress_no_declared_constraints is missing data.sqlite');
-        }
-        const maskedSnapshot = maskKtxRelationshipBenchmarkSnapshot(fixture.snapshot, 'declared_pks_and_declared_fks_removed');
-        const scaleExecutor = new FileSqliteExecutor(fixture.dataPath);
-        try {
-            const result = await profileKtxRelationshipSchema({
-                connectionId: fixture.snapshot.connectionId,
-                driver: fixture.snapshot.driver,
-                schema: snapshotToKtxEnrichedSchema(maskedSnapshot, new Map()),
-                executor: scaleExecutor,
-                ctx: { runId: 'scale-stress-profile-query-count' },
-                profileSampleRows: 3,
-            });
-            expect(fixture.snapshot.tables).toHaveLength(400);
-            expect(result.queryCount).toBe(400);
-            expect(result.queryCount).toBeLessThanOrEqual(2 * fixture.snapshot.tables.length);
-            expect(scaleExecutor.queryCount).toBe(400);
-        }
-        finally {
-            scaleExecutor.close();
-        }
-    });
-    it('profiles tables concurrently up to profileConcurrency', async () => {
-        let inFlight = 0;
-        let maxInFlight = 0;
-        const executor = {
-            executeReadOnly: vi.fn(async (input) => {
-                inFlight += 1;
-                maxInFlight = Math.max(maxInFlight, inFlight);
-                await new Promise((resolve) => setTimeout(resolve, 10));
-                inFlight -= 1;
-                return {
-                    headers: [
-                        'column_name',
-                        'table_row_count',
-                        'row_count',
-                        'null_count',
-                        'distinct_count',
-                        'min_text_length',
-                        'max_text_length',
-                        'sample_values',
-                    ],
-                    rows: [[input.sql.includes('accounts') ? 'id' : 'account_id', 2, 2, 0, 2, 1, 2, '1\u001f2']],
-                    totalRows: 1,
-                    rowCount: 1,
-                };
-            }),
-        };
-        await profileKtxRelationshipSchema({
-            connectionId: 'warehouse',
-            driver: 'sqlite',
-            schema: schemaWithTables(['accounts', 'orders', 'payments', 'refunds']),
-            executor,
-            ctx: { runId: 'profile-concurrency' },
-            profileConcurrency: 4,
-        });
-        expect(maxInFlight).toBe(4);
-    });
-    it('keeps profiling other tables when one table profile fails', async () => {
-        const executor = {
-            executeReadOnly: vi.fn(async (input) => {
-                if (input.sql.includes('"orders"')) {
-                    throw new Error('orders unavailable');
-                }
-                return {
-                    headers: [
-                        'column_name',
-                        'table_row_count',
-                        'row_count',
-                        'null_count',
-                        'distinct_count',
-                        'min_text_length',
-                        'max_text_length',
-                        'sample_values',
-                    ],
-                    rows: [['id', 2, 2, 0, 2, 1, 2, '1\u001f2']],
-                    totalRows: 1,
-                    rowCount: 1,
-                };
-            }),
-        };
-        const result = await profileKtxRelationshipSchema({
-            connectionId: 'warehouse',
-            driver: 'sqlite',
-            schema: schemaWithTables(['accounts', 'orders']),
-            executor,
-            ctx: { runId: 'profile-error-isolated' },
-            profileConcurrency: 2,
-        });
-        expect(result.warnings).toContain('profile_failed:orders:orders unavailable');
-        expect(result.tables).toHaveLength(2);
-        expect(Object.keys(result.columns)).toContain('accounts.id');
-    });
-});
-function schemaWithTables(names) {
-    return schema(names.map((name) => table(name, [
-        column(name, name === 'orders' ? 'account_id' : 'id', {
-            nullable: false,
-            primaryKey: name !== 'orders',
-        }),
-    ])));
-}

package/dist/context/scan/relationship-scoring.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/context/scan/relationship-scoring.test.js

@@ -1,86 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { calibrateWeightsFromSyntheticFixtures, defaultKtxRelationshipScoreWeights, normalizeKtxRelationshipScoreWeights, scoreKtxRelationshipCandidate, } from './relationship-scoring.js';
-function signals(overrides = {}) {
-    return {
-        nameSimilarity: 0.5,
-        typeCompatibility: 1,
-        valueOverlap: 0,
-        embeddingSimilarity: 0,
-        profileUniqueness: 0.5,
-        profileNullRate: 0.5,
-        structuralPrior: 0.5,
-        ...overrides,
-    };
-}
-describe('relationship scoring', () => {
-    it('scores stronger evidence higher without hard-gating on names', () => {
-        const weakNameStrongProfile = scoreKtxRelationshipCandidate(signals({
-            nameSimilarity: 0.05,
-            typeCompatibility: 1,
-            valueOverlap: 0.7,
-            profileUniqueness: 1,
-            profileNullRate: 1,
-            structuralPrior: 0.7,
-        }));
-        const strongNameWeakProfile = scoreKtxRelationshipCandidate(signals({
-            nameSimilarity: 0.95,
-            typeCompatibility: 1,
-            valueOverlap: 0,
-            profileUniqueness: 0.3,
-            profileNullRate: 0.4,
-            structuralPrior: 0.5,
-        }));
-        expect(weakNameStrongProfile.score).toBeGreaterThan(strongNameWeakProfile.score);
-        expect(weakNameStrongProfile.contributions.profileUniqueness).toBeGreaterThan(0);
-        expect(weakNameStrongProfile.contributions.nameSimilarity).toBeLessThan(0.02);
-    });
-    it('normalizes partial and invalid weights into a usable vector', () => {
-        const weights = normalizeKtxRelationshipScoreWeights({
-            nameSimilarity: 3,
-            typeCompatibility: -1,
-            valueOverlap: Number.POSITIVE_INFINITY,
-            profileUniqueness: 1,
-        });
-        const total = Object.values(weights).reduce((sum, value) => sum + value, 0);
-        expect(total).toBeCloseTo(1, 6);
-        expect(weights.nameSimilarity).toBeGreaterThan(weights.profileUniqueness);
-        expect(weights.typeCompatibility).toBe(0);
-        expect(weights.valueOverlap).toBe(0);
-    });
-    it('returns deterministic defaults as a defensive copy', () => {
-        const first = defaultKtxRelationshipScoreWeights();
-        const second = defaultKtxRelationshipScoreWeights();
-        expect(first).toEqual(second);
-        expect(first).not.toBe(second);
-        expect(Object.values(first).reduce((sum, value) => sum + value, 0)).toBeCloseTo(1, 6);
-    });
-    it('calibrates only from synthetic observations', () => {
-        expect(() => calibrateWeightsFromSyntheticFixtures([
-            {
-                fixtureId: 'chinook_with_declared_metadata',
-                origin: 'public',
-                expectedRelationship: true,
-                signals: signals({ nameSimilarity: 1 }),
-            },
-        ])).toThrow(/synthetic/i);
-    });
-    it('calibrates deterministic weights from positive and negative synthetic observations', () => {
-        const weights = calibrateWeightsFromSyntheticFixtures([
-            {
-                fixtureId: 'synthetic_positive',
-                origin: 'synthetic',
-                expectedRelationship: true,
-                signals: signals({ nameSimilarity: 0.8, valueOverlap: 0.9, profileUniqueness: 1, profileNullRate: 1 }),
-            },
-            {
-                fixtureId: 'synthetic_negative',
-                origin: 'synthetic',
-                expectedRelationship: false,
-                signals: signals({ nameSimilarity: 0.2, valueOverlap: 0.1, profileUniqueness: 0.4, profileNullRate: 0.5 }),
-            },
-        ]);
-        expect(Object.values(weights).reduce((sum, value) => sum + value, 0)).toBeCloseTo(1, 6);
-        expect(weights.valueOverlap).toBeGreaterThan(weights.structuralPrior);
-        expect(weights.profileUniqueness).toBeGreaterThan(weights.embeddingSimilarity);
-    });
-});

package/dist/context/scan/relationship-validation.test.d.ts

@@ -1 +0,0 @@
-export {};