@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/node_modules/@ktx/context/dist/ingest/adapters/live-database/structural-sync.test.js

@@ -0,0 +1,416 @@
+import { describe, expect, it } from 'vitest';
+import { planLiveDatabaseStructuralSync } from './structural-sync.js';
+function idFactory() {
+    let next = 1;
+    return () => `id-${next++}`;
+}
+describe('planLiveDatabaseStructuralSync', () => {
+    it('plans table and column creates, updates, deletes, and metadata invalidation', () => {
+        const current = {
+            connectionId: 'conn-1',
+            tables: [
+                {
+                    id: 'tbl-orders',
+                    name: 'orders',
+                    catalog: null,
+                    db: 'public',
+                    enabled: true,
+                    descriptions: { ai: 'Old AI order text', db: 'Old DB order text' },
+                    columns: [
+                        {
+                            id: 'col-order-id',
+                            name: 'id',
+                            type: 'number',
+                            nullable: false,
+                            primaryKey: true,
+                            parentColumnId: null,
+                            descriptions: { db: 'Order id' },
+                            embedding: [1, 2, 3],
+                            sampleValues: null,
+                            cardinality: null,
+                        },
+                        {
+                            id: 'col-order-total',
+                            name: 'total',
+                            type: 'number',
+                            nullable: true,
+                            primaryKey: false,
+                            parentColumnId: null,
+                            descriptions: { ai: 'Old AI total text', db: 'Old total text' },
+                            embedding: [4, 5, 6],
+                            sampleValues: ['10'],
+                            cardinality: 12,
+                        },
+                        {
+                            id: 'col-order-removed',
+                            name: 'removed',
+                            type: 'string',
+                            nullable: true,
+                            primaryKey: false,
+                            parentColumnId: null,
+                            descriptions: {},
+                            embedding: null,
+                            sampleValues: null,
+                            cardinality: null,
+                        },
+                    ],
+                },
+                {
+                    id: 'tbl-removed',
+                    name: 'removed_table',
+                    catalog: null,
+                    db: 'public',
+                    enabled: true,
+                    descriptions: {},
+                    columns: [
+                        {
+                            id: 'col-removed-id',
+                            name: 'id',
+                            type: 'number',
+                            nullable: false,
+                            primaryKey: true,
+                            parentColumnId: null,
+                            descriptions: {},
+                            embedding: null,
+                            sampleValues: null,
+                            cardinality: null,
+                        },
+                    ],
+                },
+            ],
+            links: [
+                {
+                    id: 'inferred-total-link',
+                    fromTableId: 'tbl-orders',
+                    fromColumnId: 'col-order-total',
+                    toTableId: 'tbl-orders',
+                    toColumnId: 'col-order-id',
+                    source: 'inferred',
+                    confidence: 0.7,
+                    relationshipType: 'MANY_TO_ONE',
+                    isPrimaryKeyReference: true,
+                },
+            ],
+        };
+        const plan = planLiveDatabaseStructuralSync({
+            connectionId: 'conn-1',
+            current,
+            extracted: {
+                connectionId: 'conn-1',
+                tables: [
+                    {
+                        name: 'orders',
+                        catalog: null,
+                        db: 'public',
+                        dbComment: 'Fresh DB order text',
+                        columns: [
+                            {
+                                name: 'id',
+                                type: 'number',
+                                nullable: false,
+                                primaryKey: true,
+                                dbComment: 'Order id',
+                            },
+                            {
+                                name: 'total',
+                                type: 'string',
+                                nullable: false,
+                                primaryKey: false,
+                                dbComment: 'Fresh total text',
+                            },
+                            {
+                                name: 'created_at',
+                                type: 'time',
+                                nullable: false,
+                                primaryKey: false,
+                                dbComment: 'Creation timestamp',
+                            },
+                        ],
+                        foreignKeys: [],
+                    },
+                    {
+                        name: 'customers',
+                        catalog: null,
+                        db: 'public',
+                        dbComment: 'Customer table',
+                        columns: [
+                            {
+                                name: 'id',
+                                type: 'number',
+                                nullable: false,
+                                primaryKey: true,
+                                dbComment: null,
+                            },
+                        ],
+                        foreignKeys: [],
+                    },
+                ],
+            },
+            idFactory: idFactory(),
+        });
+        expect(plan.stats).toEqual({
+            tablesCreated: 1,
+            tablesDeleted: 1,
+            columnsCreated: 2,
+            columnsDeleted: 2,
+            columnsModified: 1,
+            formalLinksCreated: 0,
+            formalLinksDeleted: 0,
+        });
+        expect(plan.operations.deleteTableIds).toEqual(['tbl-removed']);
+        expect(plan.operations.deleteColumnIds).toEqual(['col-order-removed']);
+        expect(plan.operations.insertTables).toEqual([
+            {
+                id: 'id-2',
+                connectionId: 'conn-1',
+                name: 'customers',
+                catalog: null,
+                db: 'public',
+                enabled: true,
+            },
+        ]);
+        expect(plan.operations.insertColumns).toEqual([
+            {
+                id: 'id-1',
+                tableId: 'tbl-orders',
+                name: 'created_at',
+                parentColumnId: null,
+            },
+            {
+                id: 'id-3',
+                tableId: 'id-2',
+                name: 'id',
+                parentColumnId: null,
+            },
+        ]);
+        expect(plan.operations.touchColumnIds).toEqual(['col-order-total']);
+        expect(plan.operations.invalidateColumnEmbeddingIds).toEqual(['col-order-total']);
+        expect(plan.inferredLinksToValidate).toEqual(['inferred-total-link']);
+        expect(plan.changes).toEqual({
+            newTableIds: ['id-2'],
+            newColumnIds: ['id-1', 'id-3'],
+            tablesWithStructuralChanges: ['tbl-orders', 'id-2'],
+            columnsWithTypeChange: ['col-order-total'],
+            columnsWithDescriptionChange: ['col-order-total'],
+            tablesWithDescriptionChange: ['tbl-orders'],
+        });
+        const orders = plan.schema.tables.find((table) => table.name === 'orders');
+        expect(orders?.descriptions).toEqual({ db: 'Fresh DB order text' });
+        expect(orders?.columns.map((column) => column.name)).toEqual(['id', 'total', 'created_at']);
+        expect(orders?.columns.find((column) => column.name === 'total')).toMatchObject({
+            id: 'col-order-total',
+            type: 'string',
+            nullable: false,
+            primaryKey: false,
+            descriptions: { db: 'Fresh total text' },
+            embedding: null,
+            sampleValues: ['10'],
+            cardinality: 12,
+        });
+    });
+    it('builds formal links from extracted foreign keys and preserves valid inferred links', () => {
+        const current = {
+            connectionId: 'conn-1',
+            tables: [
+                {
+                    id: 'tbl-orders',
+                    name: 'orders',
+                    catalog: null,
+                    db: 'public',
+                    enabled: true,
+                    descriptions: {},
+                    columns: [
+                        {
+                            id: 'col-orders-id',
+                            name: 'id',
+                            type: 'number',
+                            nullable: false,
+                            primaryKey: true,
+                            parentColumnId: null,
+                            descriptions: {},
+                            embedding: null,
+                            sampleValues: null,
+                            cardinality: null,
+                        },
+                        {
+                            id: 'col-orders-customer',
+                            name: 'customer_id',
+                            type: 'number',
+                            nullable: false,
+                            primaryKey: false,
+                            parentColumnId: null,
+                            descriptions: {},
+                            embedding: null,
+                            sampleValues: null,
+                            cardinality: null,
+                        },
+                    ],
+                },
+                {
+                    id: 'tbl-customers',
+                    name: 'customers',
+                    catalog: null,
+                    db: 'public',
+                    enabled: true,
+                    descriptions: {},
+                    columns: [
+                        {
+                            id: 'col-customers-id',
+                            name: 'id',
+                            type: 'number',
+                            nullable: false,
+                            primaryKey: true,
+                            parentColumnId: null,
+                            descriptions: {},
+                            embedding: null,
+                            sampleValues: null,
+                            cardinality: null,
+                        },
+                    ],
+                },
+            ],
+            links: [
+                {
+                    id: 'formal-existing',
+                    fromTableId: 'tbl-orders',
+                    fromColumnId: 'col-orders-customer',
+                    toTableId: 'tbl-customers',
+                    toColumnId: 'col-customers-id',
+                    source: 'formal',
+                    confidence: 1,
+                    relationshipType: 'MANY_TO_ONE',
+                    isPrimaryKeyReference: true,
+                },
+                {
+                    id: 'inferred-existing',
+                    fromTableId: 'tbl-orders',
+                    fromColumnId: 'col-orders-id',
+                    toTableId: 'tbl-customers',
+                    toColumnId: 'col-customers-id',
+                    source: 'inferred',
+                    confidence: 0.6,
+                    relationshipType: 'MANY_TO_ONE',
+                    isPrimaryKeyReference: true,
+                },
+            ],
+        };
+        const plan = planLiveDatabaseStructuralSync({
+            connectionId: 'conn-1',
+            current,
+            extracted: {
+                connectionId: 'conn-1',
+                tables: [
+                    {
+                        name: 'orders',
+                        catalog: null,
+                        db: 'public',
+                        dbComment: null,
+                        columns: [
+                            { name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null },
+                            { name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null },
+                        ],
+                        foreignKeys: [
+                            {
+                                fromTable: 'orders',
+                                fromColumn: 'customer_id',
+                                toTable: 'customers',
+                                toColumn: 'id',
+                            },
+                        ],
+                    },
+                    {
+                        name: 'customers',
+                        catalog: null,
+                        db: 'public',
+                        dbComment: null,
+                        columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }],
+                        foreignKeys: [],
+                    },
+                ],
+            },
+            idFactory: idFactory(),
+        });
+        expect(plan.stats.formalLinksCreated).toBe(0);
+        expect(plan.stats.formalLinksDeleted).toBe(0);
+        expect(plan.schema.links.map((link) => link.id)).toEqual(['formal-existing', 'inferred-existing']);
+        const planAfterForeignKeyRemoval = planLiveDatabaseStructuralSync({
+            connectionId: 'conn-1',
+            current,
+            extracted: {
+                connectionId: 'conn-1',
+                tables: [
+                    {
+                        name: 'orders',
+                        catalog: null,
+                        db: 'public',
+                        dbComment: null,
+                        columns: [
+                            { name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null },
+                            { name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null },
+                        ],
+                        foreignKeys: [],
+                    },
+                    {
+                        name: 'customers',
+                        catalog: null,
+                        db: 'public',
+                        dbComment: null,
+                        columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }],
+                        foreignKeys: [],
+                    },
+                ],
+            },
+            idFactory: idFactory(),
+        });
+        expect(planAfterForeignKeyRemoval.stats.formalLinksDeleted).toBe(1);
+        expect(planAfterForeignKeyRemoval.schema.links.map((link) => link.id)).toEqual(['inferred-existing']);
+        const planAfterForeignKeyCreation = planLiveDatabaseStructuralSync({
+            connectionId: 'conn-1',
+            current: { ...current, links: [current.links[1]] },
+            extracted: {
+                connectionId: 'conn-1',
+                tables: [
+                    {
+                        name: 'orders',
+                        catalog: null,
+                        db: 'public',
+                        dbComment: null,
+                        columns: [
+                            { name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null },
+                            { name: 'customer_id', type: 'number', nullable: false, primaryKey: false, dbComment: null },
+                        ],
+                        foreignKeys: [
+                            {
+                                fromTable: 'orders',
+                                fromColumn: 'customer_id',
+                                toTable: 'customers',
+                                toColumn: 'id',
+                            },
+                        ],
+                    },
+                    {
+                        name: 'customers',
+                        catalog: null,
+                        db: 'public',
+                        dbComment: null,
+                        columns: [{ name: 'id', type: 'number', nullable: false, primaryKey: true, dbComment: null }],
+                        foreignKeys: [],
+                    },
+                ],
+            },
+            idFactory: idFactory(),
+        });
+        expect(planAfterForeignKeyCreation.stats.formalLinksCreated).toBe(1);
+        expect(planAfterForeignKeyCreation.schema.links[0]).toMatchObject({
+            id: 'id-1',
+            fromTableId: 'tbl-orders',
+            fromColumnId: 'col-orders-customer',
+            toTableId: 'tbl-customers',
+            toColumnId: 'col-customers-id',
+            source: 'formal',
+            confidence: 1,
+            relationshipType: 'MANY_TO_ONE',
+            isPrimaryKeyReference: true,
+        });
+    });
+});

package/node_modules/@ktx/context/dist/ingest/adapters/live-database/types.d.ts

@@ -0,0 +1,8 @@
+import type { KtxSchemaSnapshot } from '../../../scan/types.js';
+export interface LiveDatabaseIntrospectionPort {
+    extractSchema(connectionId: string): Promise<KtxSchemaSnapshot>;
+}
+export interface LiveDatabaseSourceAdapterDeps {
+    introspection: LiveDatabaseIntrospectionPort;
+    now?: () => Date;
+}

package/node_modules/@ktx/context/dist/ingest/adapters/live-database/types.js

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/adapters/looker/chunk.d.ts

@@ -0,0 +1,2 @@
+import type { ChunkResult, DiffSet } from '../../types.js';
+export declare function chunkLookerStagedDir(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;

package/node_modules/@ktx/context/dist/ingest/adapters/looker/chunk.js

@@ -0,0 +1,146 @@
+import { readdir, readFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import { buildLookerReconcileNotes } from './reconcile.js';
+import { STAGED_FILES, stagedDashboardFileSchema, stagedExploreFileSchema, stagedLookFileSchema, } from './types.js';
+async function walk(root) {
+    const entries = await readdir(root, { withFileTypes: true, recursive: true });
+    return entries
+        .filter((entry) => entry.isFile())
+        .map((entry) => relative(root, join(entry.parentPath, entry.name)).replace(/\\/g, '/'))
+        .sort();
+}
+async function loadProject(stagedDir) {
+    const allPaths = await walk(stagedDir);
+    const dashboardsByPath = new Map();
+    const looksByPath = new Map();
+    const explorePaths = [];
+    for (const path of allPaths) {
+        if (/^dashboards\/[^/]+\.json$/.test(path)) {
+            dashboardsByPath.set(path, stagedDashboardFileSchema.parse(JSON.parse(await readFile(join(stagedDir, path), 'utf-8'))));
+            continue;
+        }
+        if (/^looks\/[^/]+\.json$/.test(path)) {
+            looksByPath.set(path, stagedLookFileSchema.parse(JSON.parse(await readFile(join(stagedDir, path), 'utf-8'))));
+            continue;
+        }
+        if (/^explores\/[^/]+\/[^/]+\.json$/.test(path)) {
+            const explore = stagedExploreFileSchema.parse(JSON.parse(await readFile(join(stagedDir, path), 'utf-8')));
+            explorePaths.push(explorePath(explore.modelName, explore.exploreName));
+        }
+    }
+    return { allPaths, dashboardsByPath, looksByPath, explorePaths: [...new Set(explorePaths)].sort() };
+}
+export async function chunkLookerStagedDir(stagedDir, diffSet) {
+    const project = await loadProject(stagedDir);
+    const firstRunUnits = emitFirstRunWorkUnits(project);
+    const result = diffSet ? applyDiffSet(firstRunUnits, diffSet) : { workUnits: firstRunUnits };
+    const eviction = diffSet && diffSet.deleted.length > 0 ? { deletedRawPaths: [...diffSet.deleted].sort() } : result.eviction;
+    return {
+        ...result,
+        eviction,
+        reconcileNotes: result.workUnits.length > 0 || eviction ? buildLookerReconcileNotes() : [],
+    };
+}
+function emitFirstRunWorkUnits(project) {
+    const units = [];
+    for (const path of project.explorePaths) {
+        const parts = /^explores\/([^/]+)\/([^/]+)\.json$/.exec(path);
+        if (!parts) {
+            continue;
+        }
+        const deps = project.allPaths.includes(STAGED_FILES.lookmlModels) ? [STAGED_FILES.lookmlModels] : [];
+        units.push(buildUnit(project, {
+            unitKey: `looker-explore-${parts[1]}-${parts[2]}`,
+            displayLabel: `Looker explore ${parts[1]}.${parts[2]}`,
+            rawFiles: [path, ...evidencePathsForExplore(project, parts[1], parts[2])],
+            dependencyPaths: deps,
+            notes: `Write API-derived SL source looker__${parts[1]}__${parts[2]} and durable domain knowledge for this Looker explore.`,
+        }));
+    }
+    for (const [path, dashboard] of [...project.dashboardsByPath.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+        const deps = new Set();
+        addIfPresent(project, deps, STAGED_FILES.foldersTree);
+        addIfPresent(project, deps, STAGED_FILES.signals.dashboardUsage);
+        addIfPresent(project, deps, STAGED_FILES.signals.scheduledPlans);
+        addIfPresent(project, deps, STAGED_FILES.signals.favorites);
+        if (dashboard.ownerId) {
+            addIfPresent(project, deps, `users/${dashboard.ownerId}.json`);
+        }
+        for (const tile of dashboard.tiles) {
+            addExploreDependency(project, deps, tile.query);
+        }
+        units.push(buildUnit(project, {
+            unitKey: `looker-dashboard-${dashboard.lookerId}`,
+            displayLabel: `Looker dashboard "${dashboard.title}"`,
+            rawFiles: [path, ...evidencePathsForDashboard(project, dashboard.lookerId)],
+            dependencyPaths: [...deps].sort(),
+            notes: 'Extract generalizable metric, segment, and domain knowledge from this dashboard. Treat usage, owner, and folder data as prioritization/provenance context only. Use context_evidence_search/context_evidence_read and context_candidate_write for wiki-bound knowledge; do not write wiki pages directly from this WorkUnit.',
+        }));
+    }
+    for (const [path, look] of [...project.looksByPath.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+        const deps = new Set();
+        addIfPresent(project, deps, STAGED_FILES.foldersTree);
+        addIfPresent(project, deps, STAGED_FILES.signals.lookUsage);
+        addIfPresent(project, deps, STAGED_FILES.signals.scheduledPlans);
+        addIfPresent(project, deps, STAGED_FILES.signals.favorites);
+        if (look.ownerId) {
+            addIfPresent(project, deps, `users/${look.ownerId}.json`);
+        }
+        addExploreDependency(project, deps, look.query);
+        units.push(buildUnit(project, {
+            unitKey: `looker-look-${look.lookerId}`,
+            displayLabel: `Looker Look "${look.title}"`,
+            rawFiles: [path, ...evidencePathsForLook(project, look.lookerId)],
+            dependencyPaths: [...deps].sort(),
+            notes: 'Extract generalizable metric, segment, and domain knowledge from this Look. Treat usage, owner, and folder data as prioritization/provenance context only. Use context_evidence_search/context_evidence_read and context_candidate_write for wiki-bound knowledge; do not write wiki pages directly from this WorkUnit.',
+        }));
+    }
+    return units.sort((a, b) => a.unitKey.localeCompare(b.unitKey));
+}
+function buildUnit(project, input) {
+    const excluded = new Set([...input.rawFiles, ...input.dependencyPaths]);
+    return {
+        ...input,
+        peerFileIndex: project.allPaths.filter((path) => !excluded.has(path)).sort(),
+    };
+}
+function applyDiffSet(firstRunUnits, diffSet) {
+    const touched = new Set([...diffSet.added, ...diffSet.modified]);
+    const workUnits = firstRunUnits.filter((wu) => {
+        const readablePaths = [...wu.rawFiles, ...wu.dependencyPaths];
+        return readablePaths.some((path) => touched.has(path));
+    });
+    return { workUnits };
+}
+function addIfPresent(project, deps, path) {
+    if (project.allPaths.includes(path)) {
+        deps.add(path);
+    }
+}
+function addExploreDependency(project, deps, query) {
+    if (!query) {
+        return;
+    }
+    addIfPresent(project, deps, explorePath(query.model, query.view));
+}
+function evidencePathsForExplore(project, modelName, exploreName) {
+    return existingPaths(project, [
+        `evidence/explores/${modelName}/${exploreName}/metadata.json`,
+        `evidence/explores/${modelName}/${exploreName}/page.md`,
+    ]);
+}
+function evidencePathsForDashboard(project, dashboardId) {
+    return existingPaths(project, [
+        `evidence/dashboards/${dashboardId}/metadata.json`,
+        `evidence/dashboards/${dashboardId}/page.md`,
+    ]);
+}
+function evidencePathsForLook(project, lookId) {
+    return existingPaths(project, [`evidence/looks/${lookId}/metadata.json`, `evidence/looks/${lookId}/page.md`]);
+}
+function existingPaths(project, paths) {
+    return paths.filter((path) => project.allPaths.includes(path));
+}
+function explorePath(modelName, exploreName) {
+    return `explores/${modelName}/${exploreName}.json`;
+}

package/node_modules/@ktx/context/dist/ingest/adapters/looker/chunk.test.d.ts

@@ -0,0 +1 @@
+export {};