@kaelio/ktx 0.5.0 → 0.7.0

package/dist/context/ingest/adapters/live-database/daemon-introspection.test.js

@@ -1,241 +0,0 @@
-import { once } from 'node:events';
-import { createServer } from 'node:http';
-import { describe, expect, it, vi } from 'vitest';
-import { tableRefSet } from '../../../scan/table-ref.js';
-import { createDaemonLiveDatabaseIntrospection } from './daemon-introspection.js';
-const daemonResponse = {
-    connection_id: 'warehouse',
-    extracted_at: '2026-04-28T10:00:00+00:00',
-    metadata: { driver: 'postgres', schemas: ['public'] },
-    tables: [
-        {
-            catalog: 'warehouse',
-            db: 'public',
-            name: 'customers',
-            comment: null,
-            columns: [{ name: 'id', type: 'integer', nullable: false, primary_key: true, comment: null }],
-            foreign_keys: [],
-        },
-        {
-            catalog: 'warehouse',
-            db: 'public',
-            name: 'orders',
-            comment: 'Order facts',
-            columns: [
-                { name: 'id', type: 'integer', nullable: false, primary_key: true, comment: 'Order id' },
-                { name: 'customer_id', type: 'integer', nullable: false, primary_key: false, comment: null },
-            ],
-            foreign_keys: [
-                {
-                    from_column: 'customer_id',
-                    to_table: 'customers',
-                    to_column: 'id',
-                    constraint_name: 'orders_customer_id_fkey',
-                },
-            ],
-        },
-    ],
-};
-describe('createDaemonLiveDatabaseIntrospection', () => {
-    it('calls the database-introspect daemon command and maps the snapshot response', async () => {
-        const runJson = vi.fn(async () => daemonResponse);
-        const introspection = createDaemonLiveDatabaseIntrospection({
-            connections: {
-                warehouse: {
-                    driver: 'postgres',
-                    url: 'postgres://localhost:5432/warehouse',
-                },
-            },
-            schemas: ['public'],
-            runJson,
-        });
-        await expect(introspection.extractSchema('warehouse')).resolves.toEqual({
-            connectionId: 'warehouse',
-            driver: 'postgres',
-            extractedAt: '2026-04-28T10:00:00+00:00',
-            scope: { schemas: ['public'] },
-            metadata: { driver: 'postgres', schemas: ['public'] },
-            tables: [
-                {
-                    catalog: 'warehouse',
-                    db: 'public',
-                    name: 'customers',
-                    kind: 'table',
-                    comment: null,
-                    estimatedRows: null,
-                    columns: [
-                        {
-                            name: 'id',
-                            nativeType: 'integer',
-                            normalizedType: 'integer',
-                            dimensionType: 'number',
-                            nullable: false,
-                            primaryKey: true,
-                            comment: null,
-                        },
-                    ],
-                    foreignKeys: [],
-                },
-                {
-                    catalog: 'warehouse',
-                    db: 'public',
-                    name: 'orders',
-                    kind: 'table',
-                    comment: 'Order facts',
-                    estimatedRows: null,
-                    columns: [
-                        {
-                            name: 'id',
-                            nativeType: 'integer',
-                            normalizedType: 'integer',
-                            dimensionType: 'number',
-                            nullable: false,
-                            primaryKey: true,
-                            comment: 'Order id',
-                        },
-                        {
-                            name: 'customer_id',
-                            nativeType: 'integer',
-                            normalizedType: 'integer',
-                            dimensionType: 'number',
-                            nullable: false,
-                            primaryKey: false,
-                            comment: null,
-                        },
-                    ],
-                    foreignKeys: [
-                        {
-                            fromColumn: 'customer_id',
-                            toCatalog: null,
-                            toDb: null,
-                            toTable: 'customers',
-                            toColumn: 'id',
-                            constraintName: 'orders_customer_id_fkey',
-                        },
-                    ],
-                },
-            ],
-        });
-        expect(runJson).toHaveBeenCalledWith('database-introspect', {
-            connection_id: 'warehouse',
-            driver: 'postgres',
-            url: 'postgres://localhost:5432/warehouse',
-            schemas: ['public'],
-            statement_timeout_ms: 30_000,
-            connection_timeout_seconds: 5,
-        });
-    });
-    it('calls a running daemon HTTP endpoint when baseUrl is configured', async () => {
-        const requests = [];
-        const server = createServer((request, response) => {
-            const chunks = [];
-            request.on('data', (chunk) => chunks.push(chunk));
-            request.on('end', () => {
-                requests.push({
-                    url: request.url,
-                    body: JSON.parse(Buffer.concat(chunks).toString('utf8')),
-                });
-                response.writeHead(200, { 'content-type': 'application/json' });
-                response.end(JSON.stringify(daemonResponse));
-            });
-        });
-        server.listen(0, '127.0.0.1');
-        await once(server, 'listening');
-        try {
-            const address = server.address();
-            if (!address || typeof address === 'string') {
-                throw new Error('expected TCP server address');
-            }
-            const introspection = createDaemonLiveDatabaseIntrospection({
-                connections: {
-                    warehouse: {
-                        driver: 'postgresql',
-                        url: 'postgres://localhost:5432/warehouse',
-                    },
-                },
-                baseUrl: `http://127.0.0.1:${address.port}`,
-            });
-            await expect(introspection.extractSchema('warehouse', {
-                tableScope: tableRefSet([{ catalog: 'warehouse', db: 'public', name: 'orders' }]),
-            })).resolves.toMatchObject({
-                connectionId: 'warehouse',
-                tables: [{ name: 'customers' }, { name: 'orders' }],
-            });
-            expect(requests).toEqual([
-                {
-                    url: '/database/introspect',
-                    body: {
-                        connection_id: 'warehouse',
-                        driver: 'postgres',
-                        url: 'postgres://localhost:5432/warehouse',
-                        schemas: ['public'],
-                        statement_timeout_ms: 30_000,
-                        connection_timeout_seconds: 5,
-                        table_scope: [{ catalog: 'warehouse', db: 'public', name: 'orders' }],
-                    },
-                },
-            ]);
-        }
-        finally {
-            server.close();
-        }
-    });
-    it('requires a configured postgres connection with a url', async () => {
-        const introspection = createDaemonLiveDatabaseIntrospection({
-            connections: {
-                warehouse: {
-                    driver: 'postgres',
-                },
-            },
-            runJson: vi.fn(async () => daemonResponse),
-        });
-        await expect(introspection.extractSchema('warehouse')).rejects.toThrow('Local live-database ingest requires connections.warehouse.url.');
-    });
-    it('rejects unsupported local connection drivers before calling the daemon', async () => {
-        const runJson = vi.fn(async () => daemonResponse);
-        const introspection = createDaemonLiveDatabaseIntrospection({
-            connections: {
-                warehouse: {
-                    driver: 'snowflake',
-                    url: 'snowflake://example',
-                },
-            },
-            runJson,
-        });
-        await expect(introspection.extractSchema('warehouse')).rejects.toThrow('Local live-database ingest cannot run driver "snowflake".');
-        expect(runJson).not.toHaveBeenCalled();
-    });
-    it('does not use connection enabled_tables as a response filter', async () => {
-        const runJson = vi.fn(async () => daemonResponse);
-        const introspection = createDaemonLiveDatabaseIntrospection({
-            connections: {
-                warehouse: {
-                    driver: 'postgres',
-                    url: 'postgres://localhost:5432/warehouse',
-                    enabled_tables: ['public.orders'],
-                },
-            },
-            schemas: ['public'],
-            runJson,
-        });
-        const snapshot = await introspection.extractSchema('warehouse');
-        expect(snapshot.tables.map((table) => `${table.db}.${table.name}`)).toEqual(['public.customers', 'public.orders']);
-        expect(runJson).toHaveBeenCalledWith('database-introspect', expect.not.objectContaining({ table_scope: expect.anything() }));
-    });
-    it('passes through every table when enabled_tables is omitted or empty', async () => {
-        const runJson = vi.fn(async () => daemonResponse);
-        const introspection = createDaemonLiveDatabaseIntrospection({
-            connections: {
-                warehouse: {
-                    driver: 'postgres',
-                    url: 'postgres://localhost:5432/warehouse',
-                    enabled_tables: [],
-                },
-            },
-            schemas: ['public'],
-            runJson,
-        });
-        const snapshot = await introspection.extractSchema('warehouse');
-        expect(snapshot.tables.map((table) => table.name)).toEqual(['customers', 'orders']);
-    });
-});

package/dist/context/ingest/adapters/live-database/live-database.adapter.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/context/ingest/adapters/live-database/live-database.adapter.test.js

@@ -1,105 +0,0 @@
-import { mkdtemp, readdir, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { describe, expect, it, vi } from 'vitest';
-import { tableRefSet } from '../../../scan/table-ref.js';
-import { LiveDatabaseSourceAdapter } from './live-database.adapter.js';
-describe('LiveDatabaseSourceAdapter', () => {
-    it('fetches a schema snapshot through the introspection port', async () => {
-        const extractSchema = vi.fn().mockResolvedValue({
-            connectionId: 'conn-1',
-            driver: 'postgres',
-            extractedAt: '2026-04-27T00:00:00.000Z',
-            scope: { schemas: ['public'] },
-            metadata: {},
-            tables: [
-                {
-                    name: 'orders',
-                    catalog: null,
-                    db: 'public',
-                    kind: 'table',
-                    comment: null,
-                    estimatedRows: null,
-                    columns: [
-                        {
-                            name: 'id',
-                            nativeType: 'integer',
-                            normalizedType: 'integer',
-                            dimensionType: 'number',
-                            nullable: false,
-                            primaryKey: true,
-                            comment: null,
-                        },
-                    ],
-                    foreignKeys: [],
-                },
-            ],
-        });
-        const adapter = new LiveDatabaseSourceAdapter({
-            introspection: { extractSchema },
-            now: () => new Date('2026-04-27T00:00:00.000Z'),
-        });
-        const dir = await mkdtemp(join(tmpdir(), 'ktx-live-db-adapter-'));
-        await adapter.fetch(undefined, dir, { connectionId: 'conn-1', sourceKey: 'live-database' });
-        expect(extractSchema).toHaveBeenCalledWith('conn-1', { tableScope: undefined });
-        await expect(adapter.detect(dir)).resolves.toBe(true);
-        const chunked = await adapter.chunk(dir);
-        expect(chunked.workUnits.map((wu) => wu.unitKey)).toEqual(['live-database-public-orders']);
-    });
-    it('declares the live database source and skill', () => {
-        const adapter = new LiveDatabaseSourceAdapter({
-            introspection: { extractSchema: vi.fn() },
-        });
-        expect(adapter.source).toBe('live-database');
-        expect(adapter.skillNames).toEqual(['live_database_ingest']);
-    });
-    it('threads tableScope from fetch context into the introspection port without post-filtering', async () => {
-        const extractSchema = vi.fn(async (_connectionId, _options) => ({
-            connectionId: 'warehouse',
-            driver: 'snowflake',
-            extractedAt: '2026-05-22T00:00:00.000Z',
-            scope: {},
-            metadata: {},
-            tables: [
-                {
-                    catalog: 'A',
-                    db: 'MARTS',
-                    name: 'IN_SCOPE',
-                    kind: 'table',
-                    comment: null,
-                    estimatedRows: 0,
-                    columns: [],
-                    foreignKeys: [],
-                },
-                {
-                    catalog: 'A',
-                    db: 'MARTS',
-                    name: 'OUT_OF_SCOPE',
-                    kind: 'table',
-                    comment: null,
-                    estimatedRows: 0,
-                    columns: [],
-                    foreignKeys: [],
-                },
-            ],
-        }));
-        const scope = tableRefSet([{ catalog: 'A', db: 'MARTS', name: 'IN_SCOPE' }]);
-        const adapter = new LiveDatabaseSourceAdapter({
-            introspection: { extractSchema },
-        });
-        const stagedDir = await mkdtemp(join(tmpdir(), 'ktx-livedb-scope-'));
-        try {
-            await adapter.fetch(undefined, stagedDir, {
-                connectionId: 'warehouse',
-                sourceKey: 'live-database',
-                tableScope: scope,
-            });
-            expect(extractSchema).toHaveBeenCalledWith('warehouse', { tableScope: scope });
-            const tables = await readdir(join(stagedDir, 'tables'));
-            expect(tables).toHaveLength(2);
-        }
-        finally {
-            await rm(stagedDir, { recursive: true, force: true });
-        }
-    });
-});

package/dist/context/ingest/adapters/live-database/manifest.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/context/ingest/adapters/live-database/manifest.test.js

@@ -1,291 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { buildLiveDatabaseManifestShards, } from './manifest.js';
-function shardObject(shards) {
-    return Object.fromEntries([...shards.entries()].sort(([a], [b]) => a.localeCompare(b)));
-}
-describe('buildLiveDatabaseManifestShards', () => {
-    it('builds shard objects with generated joins and preserved external descriptions', () => {
-        const existingDescriptions = new Map([
-            [
-                'orders',
-                {
-                    table: { user: 'Pinned analyst description', db: 'Old db description' },
-                    columns: new Map([['id', { user: 'Pinned id description', db: 'Old id description' }]]),
-                },
-            ],
-        ]);
-        const preservedJoins = new Map([
-            [
-                'orders',
-                [
-                    {
-                        to: 'customers',
-                        on: 'orders.account_id = customers.id',
-                        relationship: 'many_to_one',
-                        source: 'manual',
-                    },
-                    {
-                        to: 'missing_accounts',
-                        on: 'orders.account_id = missing_accounts.id',
-                        relationship: 'many_to_one',
-                        source: 'manual',
-                    },
-                ],
-            ],
-        ]);
-        const result = buildLiveDatabaseManifestShards({
-            connectionType: 'POSTGRESQL',
-            mapColumnType: (nativeType) => nativeType.toLowerCase(),
-            existingDescriptions,
-            existingPreservedJoins: preservedJoins,
-            tables: [
-                {
-                    name: 'orders',
-                    catalog: null,
-                    db: 'public',
-                    descriptions: { db: 'Fresh db description', ai: 'Generated AI description' },
-                    columns: [
-                        {
-                            name: 'id',
-                            type: 'INTEGER',
-                            pk: true,
-                            nullable: false,
-                            descriptions: { db: 'Fresh id description' },
-                        },
-                        {
-                            name: 'customer_id',
-                            type: 'INTEGER',
-                        },
-                    ],
-                },
-                {
-                    name: 'customers',
-                    catalog: null,
-                    db: 'public',
-                    columns: [
-                        {
-                            name: 'id',
-                            type: 'INTEGER',
-                            pk: true,
-                            nullable: false,
-                        },
-                    ],
-                },
-            ],
-            joins: [
-                {
-                    fromTable: 'orders',
-                    fromColumns: ['customer_id'],
-                    toTable: 'customers',
-                    toColumns: ['id'],
-                    relationship: 'MANY_TO_ONE',
-                    source: 'formal',
-                },
-            ],
-        });
-        expect(result.tablesProcessed).toBe(2);
-        expect(shardObject(result.shards)).toEqual({
-            public: {
-                tables: {
-                    orders: {
-                        table: 'public.orders',
-                        descriptions: {
-                            user: 'Pinned analyst description',
-                            db: 'Fresh db description',
-                            ai: 'Generated AI description',
-                        },
-                        columns: [
-                            {
-                                name: 'id',
-                                type: 'integer',
-                                pk: true,
-                                nullable: false,
-                                descriptions: {
-                                    user: 'Pinned id description',
-                                    db: 'Fresh id description',
-                                },
-                            },
-                            {
-                                name: 'customer_id',
-                                type: 'integer',
-                            },
-                        ],
-                        joins: [
-                            {
-                                to: 'customers',
-                                on: 'orders.customer_id = customers.id',
-                                relationship: 'many_to_one',
-                                source: 'formal',
-                            },
-                            {
-                                to: 'customers',
-                                on: 'orders.account_id = customers.id',
-                                relationship: 'many_to_one',
-                                source: 'manual',
-                            },
-                        ],
-                    },
-                    customers: {
-                        table: 'public.customers',
-                        columns: [
-                            {
-                                name: 'id',
-                                type: 'integer',
-                                pk: true,
-                                nullable: false,
-                            },
-                        ],
-                        joins: [
-                            {
-                                to: 'orders',
-                                on: 'customers.id = orders.customer_id',
-                                relationship: 'one_to_many',
-                                source: 'formal',
-                            },
-                        ],
-                    },
-                },
-            },
-        });
-    });
-    it('uses warehouse and schema shard keys for snowflake-style connections', () => {
-        const result = buildLiveDatabaseManifestShards({
-            connectionType: 'SNOWFLAKE',
-            mapColumnType: (nativeType) => nativeType.toLowerCase(),
-            tables: [
-                {
-                    name: 'accounts',
-                    catalog: 'ANALYTICS',
-                    db: 'CORE',
-                    columns: [{ name: 'id', type: 'NUMBER' }],
-                },
-            ],
-            joins: [],
-        });
-        expect(shardObject(result.shards)).toEqual({
-            'ANALYTICS.CORE': {
-                tables: {
-                    accounts: {
-                        table: 'ANALYTICS.CORE.accounts',
-                        columns: [{ name: 'id', type: 'number' }],
-                    },
-                },
-            },
-        });
-    });
-    it('preserves external usage keys while replacing historic SQL managed keys', () => {
-        const existingUsage = new Map([
-            [
-                'orders',
-                {
-                    narrative: 'Old generated usage narrative.',
-                    frequencyTier: 'low',
-                    commonFilters: ['old_status'],
-                    commonJoins: [],
-                    ownerNote: 'Pinned analyst note',
-                },
-            ],
-        ]);
-        const result = buildLiveDatabaseManifestShards({
-            connectionType: 'POSTGRESQL',
-            mapColumnType: (nativeType) => nativeType.toLowerCase(),
-            existingUsage,
-            tables: [
-                {
-                    name: 'orders',
-                    catalog: null,
-                    db: 'public',
-                    usage: {
-                        narrative: 'Fresh generated usage narrative.',
-                        frequencyTier: 'high',
-                        commonFilters: ['status'],
-                        commonGroupBys: ['created_at'],
-                        commonJoins: [{ table: 'public.customers', on: ['customer_id'] }],
-                    },
-                    columns: [{ name: 'id', type: 'INTEGER' }],
-                },
-            ],
-            joins: [],
-        });
-        expect(shardObject(result.shards)).toEqual({
-            public: {
-                tables: {
-                    orders: {
-                        table: 'public.orders',
-                        usage: {
-                            ownerNote: 'Pinned analyst note',
-                            narrative: 'Fresh generated usage narrative.',
-                            frequencyTier: 'high',
-                            commonFilters: ['status'],
-                            commonGroupBys: ['created_at'],
-                            commonJoins: [{ table: 'public.customers', on: ['customer_id'] }],
-                        },
-                        columns: [{ name: 'id', type: 'integer' }],
-                    },
-                },
-            },
-        });
-    });
-    it('renders ordered multi-column joins in both directions', () => {
-        const result = buildLiveDatabaseManifestShards({
-            connectionType: 'POSTGRESQL',
-            mapColumnType: (nativeType) => nativeType,
-            tables: [
-                {
-                    name: 'order_lines',
-                    catalog: null,
-                    db: 'public',
-                    columns: [
-                        { name: 'order_id', type: 'integer' },
-                        { name: 'line_number', type: 'integer' },
-                    ],
-                },
-                {
-                    name: 'order_line_allocations',
-                    catalog: null,
-                    db: 'public',
-                    columns: [
-                        { name: 'order_id', type: 'integer' },
-                        { name: 'line_number', type: 'integer' },
-                    ],
-                },
-            ],
-            joins: [
-                {
-                    fromTable: 'order_line_allocations',
-                    fromColumns: ['order_id', 'line_number'],
-                    toTable: 'order_lines',
-                    toColumns: ['order_id', 'line_number'],
-                    relationship: 'many_to_one',
-                    source: 'inferred',
-                },
-            ],
-        });
-        expect(shardObject(result.shards)).toMatchObject({
-            public: {
-                tables: {
-                    order_line_allocations: {
-                        joins: [
-                            {
-                                to: 'order_lines',
-                                on: 'order_line_allocations.order_id = order_lines.order_id AND order_line_allocations.line_number = order_lines.line_number',
-                                relationship: 'many_to_one',
-                                source: 'inferred',
-                            },
-                        ],
-                    },
-                    order_lines: {
-                        joins: [
-                            {
-                                to: 'order_line_allocations',
-                                on: 'order_lines.order_id = order_line_allocations.order_id AND order_lines.line_number = order_line_allocations.line_number',
-                                relationship: 'one_to_many',
-                                source: 'inferred',
-                            },
-                        ],
-                    },
-                },
-            },
-        });
-    });
-});

package/dist/context/ingest/adapters/live-database/stage.test.d.ts

@@ -1 +0,0 @@
-export {};