@kaelio/ktx 0.5.0 → 0.7.0

package/dist/context/ingest/adapters/metricflow/import-semantic-models.test.js

@@ -1,318 +0,0 @@
-import { describe, expect, it, vi } from 'vitest';
-import { importMetricflowSemanticModels } from './import-semantic-models.js';
-const DBT_SYSTEM_EMAIL = ['system@kae', 'lio.dev'].join('');
-function parseResult() {
-    return {
-        semanticModels: [
-            {
-                name: 'orders',
-                description: 'Orders',
-                modelRef: 'orders',
-                dimensions: [{ name: 'status', column: 'status', type: 'string', label: 'Status' }],
-                measures: [{ type: 'simple', name: 'order_count', column: 'id', aggregation: 'count' }],
-                entities: [{ name: 'customer', type: 'foreign', expr: 'customer_id' }],
-                defaultTimeDimension: null,
-            },
-        ],
-        crossModelMetrics: [
-            {
-                name: 'global_revenue',
-                label: null,
-                description: 'Revenue everywhere',
-                type: 'derived',
-                expr: 'sum(revenue)',
-                dependsOn: [{ metricName: 'orders' }],
-                filter: null,
-            },
-        ],
-        relationships: [{ fromTable: 'orders', fromColumn: 'customer_id', toTable: 'customers', toColumn: 'id' }],
-        warnings: ['parser warning'],
-    };
-}
-describe('importMetricflowSemanticModels', () => {
-    it('writes semantic models through a worktree-scoped semantic layer service and returns touched sources', async () => {
-        const scoped = {
-            getManifestEntry: vi.fn().mockResolvedValue(null),
-            isManifestBacked: vi.fn().mockResolvedValue(false),
-            loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
-            loadSource: vi.fn().mockResolvedValue(null),
-            writeSource: vi.fn().mockResolvedValue({ warnings: [] }),
-        };
-        const semanticLayerService = {
-            forWorktree: vi.fn().mockReturnValue(scoped),
-            getManifestEntry: vi.fn(),
-            isManifestBacked: vi.fn(),
-            loadAllSources: vi.fn(),
-            loadSource: vi.fn(),
-            writeSource: vi.fn(),
-        };
-        const result = await importMetricflowSemanticModels({ semanticLayerService }, {
-            connectionId: 'warehouse-1',
-            parseResult: parseResult(),
-            targetSchema: null,
-            hostTables: [
-                { id: 'orders-table', name: 'orders', catalog: null, db: null, columns: [{ id: 'c1', name: 'customer_id' }] },
-                { id: 'customers-table', name: 'customers', catalog: null, db: null, columns: [{ id: 'c2', name: 'id' }] },
-            ],
-            workdir: '/tmp/session-worktree',
-        });
-        expect(semanticLayerService.forWorktree).toHaveBeenCalledWith('/tmp/session-worktree');
-        expect(scoped.writeSource).toHaveBeenCalledTimes(2);
-        expect(scoped.writeSource).toHaveBeenNthCalledWith(1, 'warehouse-1', expect.objectContaining({ name: 'orders' }), 'dbt MetricFlow', DBT_SYSTEM_EMAIL, 'dbt MetricFlow sync: create source orders', { skipValidation: true });
-        expect(scoped.writeSource).toHaveBeenNthCalledWith(2, 'warehouse-1', expect.objectContaining({ name: 'global-revenue' }), 'dbt MetricFlow', DBT_SYSTEM_EMAIL, 'dbt MetricFlow sync: create cross-model source global-revenue', { skipValidation: true });
-        expect(result).toEqual({
-            sourcesCreated: 1,
-            sourcesUpdated: 0,
-            sourcesSkipped: 0,
-            crossModelSourcesCreated: 1,
-            relationshipsImported: 0,
-            warnings: ['parser warning'],
-            errors: [],
-            touchedSources: [
-                { connectionId: 'warehouse-1', sourceName: 'global-revenue' },
-                { connectionId: 'warehouse-1', sourceName: 'orders' },
-            ],
-        });
-    });
-    it('updates count when an existing semantic model source exists', async () => {
-        const scoped = {
-            getManifestEntry: vi.fn().mockResolvedValue(null),
-            isManifestBacked: vi.fn().mockResolvedValue(false),
-            loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
-            loadSource: vi.fn().mockImplementation((connectionId, sourceName) => Promise.resolve(sourceName === 'orders' ? { name: 'orders' } : null)),
-            writeSource: vi.fn().mockResolvedValue({ warnings: [] }),
-        };
-        const semanticLayerService = {
-            forWorktree: vi.fn().mockReturnValue(scoped),
-            getManifestEntry: vi.fn(),
-            isManifestBacked: vi.fn(),
-            loadAllSources: vi.fn(),
-            loadSource: vi.fn(),
-            writeSource: vi.fn(),
-        };
-        const result = await importMetricflowSemanticModels({ semanticLayerService }, {
-            connectionId: 'warehouse-1',
-            parseResult: { ...parseResult(), crossModelMetrics: [], relationships: [] },
-            targetSchema: null,
-            hostTables: [],
-            workdir: '/tmp/session-worktree',
-        });
-        expect(result.sourcesCreated).toBe(0);
-        expect(result.sourcesUpdated).toBe(1);
-        expect(result.crossModelSourcesCreated).toBe(0);
-    });
-    it('keeps domain write failures structured and continues processing', async () => {
-        const scoped = {
-            getManifestEntry: vi.fn().mockResolvedValue(null),
-            isManifestBacked: vi.fn().mockResolvedValue(false),
-            loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
-            loadSource: vi.fn().mockResolvedValue(null),
-            writeSource: vi.fn().mockRejectedValueOnce(new Error('cannot write orders')).mockResolvedValue({ warnings: [] }),
-        };
-        const semanticLayerService = {
-            forWorktree: vi.fn().mockReturnValue(scoped),
-            getManifestEntry: vi.fn(),
-            isManifestBacked: vi.fn(),
-            loadAllSources: vi.fn(),
-            loadSource: vi.fn(),
-            writeSource: vi.fn(),
-        };
-        const result = await importMetricflowSemanticModels({ semanticLayerService }, {
-            connectionId: 'warehouse-1',
-            parseResult: parseResult(),
-            targetSchema: null,
-            hostTables: [],
-            workdir: '/tmp/session-worktree',
-        });
-        expect(result.sourcesSkipped).toBe(1);
-        expect(result.crossModelSourcesCreated).toBe(1);
-        expect(result.errors).toEqual(["Failed to import semantic model 'orders': cannot write orders"]);
-        expect(result.touchedSources).toEqual([{ connectionId: 'warehouse-1', sourceName: 'global-revenue' }]);
-    });
-    it('writes manifest-backed semantic models as overlays', async () => {
-        const manifestOrders = {
-            name: 'orders',
-            table: 'analytics.orders',
-            grain: ['id'],
-            columns: [
-                { name: 'id', type: 'string' },
-                { name: 'customer_id', type: 'string' },
-            ],
-            joins: [],
-            measures: [],
-            descriptions: { db: 'Orders table from scan' },
-        };
-        const written = [];
-        const scoped = {
-            getManifestEntry: vi.fn().mockImplementation(async (_connectionId, sourceName) => {
-                return sourceName === 'orders' ? manifestOrders : null;
-            }),
-            isManifestBacked: vi.fn().mockImplementation(async (_connectionId, sourceName) => {
-                return sourceName === 'orders';
-            }),
-            loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
-            loadSource: vi.fn().mockResolvedValue(null),
-            writeSource: vi.fn().mockImplementation(async (_connectionId, source) => {
-                written.push(source);
-                return { warnings: [] };
-            }),
-        };
-        const semanticLayerService = {
-            forWorktree: vi.fn().mockReturnValue(scoped),
-            getManifestEntry: vi.fn(),
-            isManifestBacked: vi.fn(),
-            loadAllSources: vi.fn(),
-            loadSource: vi.fn(),
-            writeSource: vi.fn(),
-        };
-        const result = await importMetricflowSemanticModels({ semanticLayerService }, {
-            connectionId: 'warehouse-1',
-            parseResult: {
-                ...parseResult(),
-                semanticModels: [
-                    parseResult().semanticModels[0],
-                    {
-                        name: 'customers',
-                        description: null,
-                        modelRef: 'customers',
-                        dimensions: [{ name: 'id', column: 'id', type: 'string' }],
-                        measures: [],
-                        entities: [],
-                        defaultTimeDimension: null,
-                    },
-                ],
-                crossModelMetrics: [],
-            },
-            targetSchema: null,
-            hostTables: [
-                {
-                    id: 'orders-table',
-                    name: 'orders',
-                    catalog: null,
-                    db: null,
-                    columns: [
-                        { id: 'c1', name: 'customer_id' },
-                        { id: 'c2', name: 'id' },
-                    ],
-                },
-                { id: 'customers-table', name: 'customers', catalog: null, db: null, columns: [{ id: 'c3', name: 'id' }] },
-            ],
-            workdir: '/tmp/session-worktree',
-        });
-        expect(written[0]).toMatchObject({
-            name: 'orders',
-            joins: [{ to: 'customers', on: 'orders.customer_id = customers.id', relationship: 'many_to_one' }],
-            descriptions: { dbt: 'Orders' },
-        });
-        expect(written[0]).not.toHaveProperty('table');
-        expect(written[0]).not.toHaveProperty('columns');
-        expect(result.sourcesUpdated).toBe(1);
-        expect(result.relationshipsImported).toBe(1);
-    });
-    it('drops joins whose keys are absent from manifest-backed source columns', async () => {
-        const scoped = {
-            getManifestEntry: vi.fn().mockResolvedValue({
-                name: 'orders',
-                table: 'analytics.orders',
-                grain: ['id'],
-                columns: [{ name: 'id', type: 'string' }],
-                joins: [],
-                measures: [],
-            }),
-            isManifestBacked: vi.fn().mockImplementation(async (_connectionId, sourceName) => {
-                return sourceName === 'orders';
-            }),
-            loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
-            loadSource: vi.fn().mockResolvedValue(null),
-            writeSource: vi.fn().mockResolvedValue({ warnings: [] }),
-        };
-        const semanticLayerService = {
-            forWorktree: vi.fn().mockReturnValue(scoped),
-            getManifestEntry: vi.fn(),
-            isManifestBacked: vi.fn(),
-            loadAllSources: vi.fn(),
-            loadSource: vi.fn(),
-            writeSource: vi.fn(),
-        };
-        const result = await importMetricflowSemanticModels({ semanticLayerService }, {
-            connectionId: 'warehouse-1',
-            parseResult: { ...parseResult(), crossModelMetrics: [] },
-            targetSchema: null,
-            hostTables: [
-                { id: 'orders-table', name: 'orders', catalog: null, db: null, columns: [{ id: 'c1', name: 'id' }] },
-                { id: 'customers-table', name: 'customers', catalog: null, db: null, columns: [{ id: 'c2', name: 'id' }] },
-            ],
-            workdir: '/tmp/session-worktree',
-        });
-        expect(scoped.writeSource).toHaveBeenCalledWith('warehouse-1', expect.not.objectContaining({ joins: expect.anything() }), expect.any(String), expect.any(String), expect.any(String), { skipValidation: true });
-        expect(result.relationshipsImported).toBe(0);
-    });
-    it('repairs earlier sources when a later related model fails to write', async () => {
-        const written = [];
-        const scoped = {
-            getManifestEntry: vi.fn().mockResolvedValue(null),
-            isManifestBacked: vi.fn().mockResolvedValue(false),
-            loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
-            loadSource: vi.fn().mockResolvedValue(null),
-            writeSource: vi
-                .fn()
-                .mockImplementationOnce(async (_connectionId, source) => {
-                written.push(source);
-                return { warnings: [] };
-            })
-                .mockRejectedValueOnce(new Error('disk full'))
-                .mockImplementation(async (_connectionId, source) => {
-                written.push(source);
-                return { warnings: [] };
-            }),
-        };
-        const semanticLayerService = {
-            forWorktree: vi.fn().mockReturnValue(scoped),
-            getManifestEntry: vi.fn(),
-            isManifestBacked: vi.fn(),
-            loadAllSources: vi.fn(),
-            loadSource: vi.fn(),
-            writeSource: vi.fn(),
-        };
-        const result = await importMetricflowSemanticModels({ semanticLayerService }, {
-            connectionId: 'warehouse-1',
-            parseResult: {
-                ...parseResult(),
-                semanticModels: [
-                    parseResult().semanticModels[0],
-                    {
-                        name: 'customers',
-                        description: null,
-                        modelRef: 'customers',
-                        dimensions: [{ name: 'id', column: 'id', type: 'string' }],
-                        measures: [],
-                        entities: [],
-                        defaultTimeDimension: null,
-                    },
-                ],
-                crossModelMetrics: [],
-            },
-            targetSchema: null,
-            hostTables: [
-                {
-                    id: 'orders-table',
-                    name: 'orders',
-                    catalog: null,
-                    db: null,
-                    columns: [
-                        { id: 'c1', name: 'customer_id' },
-                        { id: 'c2', name: 'id' },
-                    ],
-                },
-                { id: 'customers-table', name: 'customers', catalog: null, db: null, columns: [{ id: 'c3', name: 'id' }] },
-            ],
-            workdir: '/tmp/session-worktree',
-        });
-        expect(result.sourcesCreated).toBe(1);
-        expect(result.sourcesSkipped).toBe(1);
-        expect(result.relationshipsImported).toBe(0);
-        expect(result.errors).toContain("Failed to import semantic model 'customers': disk full");
-        expect(written.filter((source) => source.name === 'orders')).toHaveLength(2);
-        expect(written[written.length - 1]).toMatchObject({ name: 'orders', joins: [] });
-    });
-});

package/dist/context/ingest/adapters/metricflow/metricflow.adapter.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/context/ingest/adapters/metricflow/metricflow.adapter.test.js

@@ -1,212 +0,0 @@
-import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
-import { makeLocalGitRepo } from '../../../test/make-local-git-repo.js';
-import { MetricflowSourceAdapter } from './metricflow.adapter.js';
-import { readMetricflowProjectionConfig, writeMetricflowProjectionConfig } from './projection-config.js';
-function compileOnlyRequiredDepsCheck() {
-    // @ts-expect-error MetricflowSourceAdapter requires an explicit cache home.
-    new MetricflowSourceAdapter();
-}
-void compileOnlyRequiredDepsCheck;
-async function makeRepo(tmpRoot, files) {
-    const fixtureDir = join(tmpRoot, 'fixture-src');
-    for (const [path, content] of Object.entries(files)) {
-        const dest = join(fixtureDir, path);
-        await mkdir(join(dest, '..'), { recursive: true });
-        await writeFile(dest, content, 'utf-8');
-    }
-    return makeLocalGitRepo(fixtureDir, join(tmpRoot, 'origin'));
-}
-function metricflowParseResult() {
-    return {
-        semanticModels: [
-            {
-                name: 'orders',
-                description: 'Orders',
-                modelRef: 'orders',
-                dimensions: [{ name: 'status', column: 'status', type: 'string', label: 'Status' }],
-                measures: [{ type: 'simple', name: 'order_count', column: 'id', aggregation: 'count' }],
-                entities: [{ name: 'customer', type: 'foreign', expr: 'customer_id' }],
-                defaultTimeDimension: null,
-            },
-        ],
-        crossModelMetrics: [],
-        relationships: [],
-        warnings: ['parser warning'],
-    };
-}
-describe('MetricflowSourceAdapter', () => {
-    let tmpRoot;
-    let stagedDir;
-    let adapter;
-    beforeEach(async () => {
-        tmpRoot = await mkdtemp(join(tmpdir(), 'mf-adapter-'));
-        stagedDir = join(tmpRoot, 'stage');
-        adapter = new MetricflowSourceAdapter({ homeDir: join(tmpRoot, 'cache-home') });
-    });
-    afterEach(async () => {
-        await rm(tmpRoot, { recursive: true, force: true });
-    });
-    it('declares the expected source key and skill list', () => {
-        expect(adapter.source).toBe('metricflow');
-        expect(adapter.skillNames).toEqual(['metricflow_ingest']);
-    });
-    it('returns configured target warehouse connection ids', async () => {
-        const metricflow = new MetricflowSourceAdapter({
-            homeDir: join(tmpRoot, 'cache-home'),
-            targetConnectionIds: ['warehouse', 'analytics', 'warehouse'],
-        });
-        await expect(metricflow.listTargetConnectionIds?.(stagedDir)).resolves.toEqual(['analytics', 'warehouse']);
-    });
-    it('detects a staged dir with a semantic_models YAML', async () => {
-        await mkdir(join(stagedDir, 'models'), { recursive: true });
-        await writeFile(join(stagedDir, 'models/orders.yml'), 'semantic_models:\n  - {name: orders, model: x, measures: [{name: c, agg: count, expr: id}]}\n', 'utf-8');
-        expect(await adapter.detect(stagedDir)).toBe(true);
-    });
-    it('rejects a staged dir with no MetricFlow-shaped YAML', async () => {
-        await mkdir(stagedDir, { recursive: true });
-        await writeFile(join(stagedDir, 'dbt_project.yml'), 'name: proj\n', 'utf-8');
-        expect(await adapter.detect(stagedDir)).toBe(false);
-    });
-    it('chunk: first-run on a minimal single-model dir emits one WU', async () => {
-        await mkdir(join(stagedDir, 'models'), { recursive: true });
-        await writeFile(join(stagedDir, 'models/orders.yml'), 'semantic_models:\n  - {name: orders, model: x, measures: [{name: c, agg: count, expr: id}]}\n', 'utf-8');
-        const result = await adapter.chunk(stagedDir);
-        expect(result.workUnits).toHaveLength(1);
-        expect(result.workUnits[0].unitKey).toBe('metricflow-orders');
-    });
-    it('attaches deep parse artifacts to the chunk result', async () => {
-        await mkdir(stagedDir, { recursive: true });
-        await writeFile(join(stagedDir, 'semantic_models.yml'), [
-            'semantic_models:',
-            '  - name: orders',
-            "    model: ref('orders')",
-            '    dimensions: []',
-            '    measures:',
-            '      - name: order_count',
-            '        agg: count',
-            "        expr: '1'",
-        ].join('\n'));
-        const chunk = await adapter.chunk(stagedDir);
-        expect(chunk.parseArtifacts).toMatchObject({
-            semanticModels: [{ name: 'orders', modelRef: 'orders' }],
-            crossModelMetrics: [],
-            relationships: [],
-            warnings: [],
-        });
-    });
-    it('fetches repo YAML files into the staged directory using a per-connection cache', async () => {
-        const repo = await makeRepo(tmpRoot, {
-            'dbt_project.yml': 'name: analytics\n',
-            'models/orders.yml': 'semantic_models:\n  - name: orders\n    model: ref("orders")\n',
-            'models/readme.md': '# ignored\n',
-        });
-        await adapter.fetch?.({
-            repoUrl: repo.repoUrl,
-            branch: 'main',
-            path: null,
-            authToken: null,
-            parsedTargetTables: {},
-        }, stagedDir, { connectionId: 'warehouse-1', sourceKey: 'metricflow' });
-        await expect(readFile(join(stagedDir, 'models/orders.yml'), 'utf-8')).resolves.toContain('semantic_models');
-        expect(await adapter.detect(stagedDir)).toBe(true);
-    });
-    it('persists parsed target tables for deterministic projection during fetch', async () => {
-        const repo = await makeRepo(tmpRoot, {
-            'dbt_project.yml': 'name: analytics\n',
-            'models/orders.yml': 'semantic_models:\n  - name: orders\n    model: ref("orders")\n',
-        });
-        await adapter.fetch?.({
-            repoUrl: repo.repoUrl,
-            branch: 'main',
-            path: null,
-            authToken: null,
-            parsedTargetTables: {
-                orders: {
-                    ok: true,
-                    catalog: null,
-                    schema: 'analytics',
-                    name: 'orders',
-                    canonicalTable: 'analytics.orders',
-                },
-            },
-        }, stagedDir, { connectionId: 'warehouse-1', sourceKey: 'metricflow' });
-        await expect(readMetricflowProjectionConfig(stagedDir)).resolves.toMatchObject({
-            parsedTargetTables: {
-                orders: {
-                    ok: true,
-                    schema: 'analytics',
-                    name: 'orders',
-                },
-            },
-        });
-    });
-    it('projects parsed MetricFlow semantic models in the integration worktree', async () => {
-        await writeMetricflowProjectionConfig(stagedDir, {
-            parsedTargetTables: {
-                orders: {
-                    ok: true,
-                    catalog: null,
-                    schema: 'analytics',
-                    name: 'orders',
-                    canonicalTable: 'analytics.orders',
-                },
-            },
-        });
-        const scoped = {
-            getManifestEntry: vi.fn().mockResolvedValue(null),
-            isManifestBacked: vi.fn().mockResolvedValue(false),
-            loadAllSources: vi.fn().mockResolvedValue({ sources: [], loadErrors: [] }),
-            loadSource: vi.fn().mockResolvedValue(null),
-            writeSource: vi.fn().mockResolvedValue({ warnings: [] }),
-        };
-        const semanticLayerService = {
-            forWorktree: vi.fn().mockReturnValue(scoped),
-            getManifestEntry: vi.fn(),
-            isManifestBacked: vi.fn(),
-            loadAllSources: vi.fn(),
-            loadSource: vi.fn(),
-            writeSource: vi.fn(),
-        };
-        const result = await adapter.project?.({
-            connectionId: 'warehouse-1',
-            sourceKey: 'metricflow',
-            syncId: 'sync-1',
-            jobId: 'job-1',
-            runId: 'run-1',
-            stagedDir,
-            workdir: '/tmp/metricflow-integration',
-            parseArtifacts: metricflowParseResult(),
-            semanticLayerService: semanticLayerService,
-        });
-        expect(semanticLayerService.forWorktree).toHaveBeenCalledWith('/tmp/metricflow-integration');
-        expect(scoped.writeSource).toHaveBeenCalledWith('warehouse-1', expect.objectContaining({ name: 'orders' }), 'dbt MetricFlow', expect.any(String), 'dbt MetricFlow sync: create source orders', { skipValidation: true });
-        expect(result).toMatchObject({
-            warnings: ['parser warning'],
-            errors: [],
-            touchedSources: [{ connectionId: 'warehouse-1', sourceName: 'orders' }],
-            changedWikiPageKeys: [],
-        });
-    });
-    it('returns a projection error when parse artifacts are missing', async () => {
-        const result = await adapter.project?.({
-            connectionId: 'warehouse-1',
-            sourceKey: 'metricflow',
-            syncId: 'sync-1',
-            jobId: 'job-1',
-            runId: 'run-1',
-            stagedDir,
-            workdir: '/tmp/metricflow-integration',
-            parseArtifacts: undefined,
-            semanticLayerService: {},
-        });
-        expect(result).toMatchObject({
-            warnings: [],
-            errors: ['MetricFlow deterministic projection requires parseArtifacts from chunk()'],
-            touchedSources: [],
-            changedWikiPageKeys: [],
-        });
-    });
-});

package/dist/context/ingest/adapters/metricflow/parse.test.d.ts

@@ -1 +0,0 @@
-export {};