@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/detect.d.ts

@@ -0,0 +1 @@
+export declare function detectMetricFlowStagedDir(stagedDir: string): Promise<boolean>;

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/detect.js

@@ -0,0 +1,34 @@
+import { readdir, readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { parse as parseYaml } from 'yaml';
+const YAML_EXT_RE = /\.(ya?ml)$/i;
+export async function detectMetricFlowStagedDir(stagedDir) {
+    const entries = await readdir(stagedDir, { withFileTypes: true, recursive: true });
+    for (const entry of entries) {
+        if (!entry.isFile() || !YAML_EXT_RE.test(entry.name)) {
+            continue;
+        }
+        const abs = join(entry.parentPath, entry.name);
+        let body;
+        try {
+            body = await readFile(abs, 'utf-8');
+        }
+        catch {
+            continue;
+        }
+        let yaml;
+        try {
+            yaml = parseYaml(body);
+        }
+        catch {
+            continue;
+        }
+        if (yaml && typeof yaml === 'object') {
+            const obj = yaml;
+            if (Array.isArray(obj.semantic_models) || Array.isArray(obj.metrics)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/detect.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/detect.test.js

@@ -0,0 +1,43 @@
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { detectMetricFlowStagedDir } from './detect.js';
+async function touch(stagedDir, relPath, body = '') {
+    const abs = join(stagedDir, relPath);
+    await mkdir(join(abs, '..'), { recursive: true });
+    await writeFile(abs, body, 'utf-8');
+}
+describe('detectMetricFlowStagedDir', () => {
+    let stagedDir;
+    beforeEach(async () => {
+        stagedDir = await mkdtemp(join(tmpdir(), 'mf-detect-'));
+    });
+    afterEach(async () => {
+        await rm(stagedDir, { recursive: true, force: true });
+    });
+    it('returns true when any YAML has top-level semantic_models:', async () => {
+        await touch(stagedDir, 'models/a.yml', 'semantic_models:\n  - {name: a, model: x, measures: []}\n');
+        expect(await detectMetricFlowStagedDir(stagedDir)).toBe(true);
+    });
+    it('returns true when any YAML has top-level metrics:', async () => {
+        await touch(stagedDir, 'metrics/m.yaml', 'metrics:\n  - {name: m, type: simple, type_params: {measure: x}}\n');
+        expect(await detectMetricFlowStagedDir(stagedDir)).toBe(true);
+    });
+    it('returns false for a directory with only dbt_project.yml', async () => {
+        await touch(stagedDir, 'dbt_project.yml', 'name: my_proj\nversion: "1.0.0"\n');
+        expect(await detectMetricFlowStagedDir(stagedDir)).toBe(false);
+    });
+    it('returns false for an empty directory', async () => {
+        expect(await detectMetricFlowStagedDir(stagedDir)).toBe(false);
+    });
+    it('returns false for only broken YAML', async () => {
+        await touch(stagedDir, 'broken.yml', '{ not: valid :::');
+        expect(await detectMetricFlowStagedDir(stagedDir)).toBe(false);
+    });
+    it('ignores non-YAML files and returns false when no YAML qualifies', async () => {
+        await touch(stagedDir, 'readme.md', '# readme');
+        await touch(stagedDir, 'script.py', 'print("hi")');
+        expect(await detectMetricFlowStagedDir(stagedDir)).toBe(false);
+    });
+});

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/fetch.d.ts

@@ -0,0 +1,11 @@
+import type { MetricflowPullConfig } from './pull-config.js';
+export interface FetchMetricflowRepoParams {
+    config: MetricflowPullConfig;
+    cacheDir: string;
+    stagedDir: string;
+}
+export interface FetchMetricflowRepoResult {
+    commitHash: string;
+    filesCopied: number;
+}
+export declare function fetchMetricflowRepo(params: FetchMetricflowRepoParams): Promise<FetchMetricflowRepoResult>;

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/fetch.js

@@ -0,0 +1,51 @@
+import { access, copyFile, mkdir, readdir } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import { cloneOrPull, sanitizeRepoError } from '../../repo-fetch.js';
+const YAML_EXT_RE = /\.ya?ml$/i;
+export async function fetchMetricflowRepo(params) {
+    const { config, cacheDir, stagedDir } = params;
+    const branch = config.branch || 'main';
+    try {
+        const { commitHash } = await cloneOrPull({
+            repoUrl: config.repoUrl,
+            authToken: config.authToken,
+            cacheDir,
+            branch,
+        });
+        const sourceRoot = config.path ? join(cacheDir, config.path) : cacheDir;
+        const filesCopied = await copyYamlFilesRecursive(sourceRoot, stagedDir);
+        return { commitHash, filesCopied };
+    }
+    catch (err) {
+        throw new Error(sanitizeRepoError(err, config.authToken));
+    }
+}
+async function copyYamlFilesRecursive(sourceRoot, destRoot) {
+    if (!(await dirExists(sourceRoot))) {
+        return 0;
+    }
+    await mkdir(destRoot, { recursive: true });
+    const entries = await readdir(sourceRoot, { withFileTypes: true, recursive: true });
+    let copied = 0;
+    for (const entry of entries) {
+        if (!entry.isFile() || !YAML_EXT_RE.test(entry.name)) {
+            continue;
+        }
+        const absSrc = join(entry.parentPath, entry.name);
+        const rel = relative(sourceRoot, absSrc);
+        const dest = join(destRoot, rel);
+        await mkdir(join(dest, '..'), { recursive: true });
+        await copyFile(absSrc, dest);
+        copied++;
+    }
+    return copied;
+}
+async function dirExists(path) {
+    try {
+        await access(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/fetch.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/fetch.test.js

@@ -0,0 +1,97 @@
+import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { makeLocalGitRepo } from '../../../test/make-local-git-repo.js';
+import { fetchMetricflowRepo } from './fetch.js';
+async function exists(path) {
+    try {
+        await access(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function makeRepo(tmpRoot, files) {
+    const fixtureDir = join(tmpRoot, 'fixture-src');
+    for (const [path, content] of Object.entries(files)) {
+        const dest = join(fixtureDir, path);
+        await mkdir(join(dest, '..'), { recursive: true });
+        await writeFile(dest, content, 'utf-8');
+    }
+    return makeLocalGitRepo(fixtureDir, join(tmpRoot, 'origin'));
+}
+describe('fetchMetricflowRepo', () => {
+    let tmpRoot;
+    beforeEach(async () => {
+        tmpRoot = await mkdtemp(join(tmpdir(), 'metricflow-fetch-'));
+    });
+    afterEach(async () => {
+        await rm(tmpRoot, { recursive: true, force: true });
+    });
+    it('clones a dbt repo and stages only YAML files', async () => {
+        const repo = await makeRepo(tmpRoot, {
+            'dbt_project.yml': 'name: analytics\n',
+            'models/orders.yml': 'semantic_models:\n  - name: orders\n    model: ref("orders")\n',
+            'models/readme.md': '# not staged\n',
+            'macros/util.sql': 'select 1\n',
+        });
+        const result = await fetchMetricflowRepo({
+            config: {
+                repoUrl: repo.repoUrl,
+                branch: 'main',
+                path: null,
+                authToken: null,
+                parsedTargetTables: {},
+            },
+            cacheDir: join(tmpRoot, 'cache'),
+            stagedDir: join(tmpRoot, 'stage'),
+        });
+        expect(result.filesCopied).toBe(2);
+        expect(result.commitHash).toMatch(/^[0-9a-f]{40}$/);
+        await expect(readFile(join(tmpRoot, 'stage/dbt_project.yml'), 'utf-8')).resolves.toContain('analytics');
+        await expect(readFile(join(tmpRoot, 'stage/models/orders.yml'), 'utf-8')).resolves.toContain('semantic_models');
+        expect(await exists(join(tmpRoot, 'stage/models/readme.md'))).toBe(false);
+        expect(await exists(join(tmpRoot, 'stage/macros/util.sql'))).toBe(false);
+    });
+    it('honors a configured repo subdirectory', async () => {
+        const repo = await makeRepo(tmpRoot, {
+            'warehouse/dbt_project.yml': 'name: warehouse\n',
+            'warehouse/models/orders.yaml': 'semantic_models:\n  - name: orders\n    model: ref("orders")\n',
+            'outside/ignored.yml': 'semantic_models:\n  - name: ignored\n    model: ref("ignored")\n',
+        });
+        const result = await fetchMetricflowRepo({
+            config: {
+                repoUrl: repo.repoUrl,
+                branch: 'main',
+                path: 'warehouse',
+                authToken: null,
+                parsedTargetTables: {},
+            },
+            cacheDir: join(tmpRoot, 'cache'),
+            stagedDir: join(tmpRoot, 'stage'),
+        });
+        expect(result.filesCopied).toBe(2);
+        await expect(readFile(join(tmpRoot, 'stage/models/orders.yaml'), 'utf-8')).resolves.toContain('orders');
+        expect(await exists(join(tmpRoot, 'stage/outside/ignored.yml'))).toBe(false);
+    });
+    it('returns zero files when the configured subdirectory is absent', async () => {
+        const repo = await makeRepo(tmpRoot, {
+            'dbt_project.yml': 'name: analytics\n',
+        });
+        await mkdir(join(tmpRoot, 'stage'), { recursive: true });
+        const result = await fetchMetricflowRepo({
+            config: {
+                repoUrl: repo.repoUrl,
+                branch: 'main',
+                path: 'missing',
+                authToken: null,
+                parsedTargetTables: {},
+            },
+            cacheDir: join(tmpRoot, 'cache'),
+            stagedDir: join(tmpRoot, 'stage'),
+        });
+        expect(result.filesCopied).toBe(0);
+    });
+});

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/graph.d.ts

@@ -0,0 +1,25 @@
+import type { ParsedMetricFlowProject } from './parse.js';
+export interface MetricFlowComponent {
+    /** Stable integer id, assigned in lexicographic order of `leadName`. */
+    id: number;
+    /** Sorted list of relative paths making up this component. */
+    paths: string[];
+    /** Sorted list of semantic_model names in this component. Empty for metric-only components. */
+    semanticModelNames: string[];
+    /** Sorted list of metric names whose defining file is in this component. */
+    metricNames: string[];
+    /** Lexicographically-first semantic_model name, or first metric name if none. Drives unitKey. */
+    leadName: string;
+}
+export interface MetricFlowGraph {
+    components: MetricFlowComponent[];
+    /** Map semantic_model name → containing component id. */
+    componentByModelName: Map<string, number>;
+    /** Map relative path → containing component id. */
+    componentByPath: Map<string, number>;
+    /** Map semantic_model name → its declaring path. */
+    pathByModelName: Map<string, string>;
+    /** Map semantic_model name → sorted transitive extends ancestor names (used for dependency widening in re-sync). */
+    extendsAncestorsByModelName: Map<string, string[]>;
+}
+export declare function buildMetricFlowGraph(project: ParsedMetricFlowProject): MetricFlowGraph;

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/graph.js

@@ -0,0 +1,163 @@
+class UnionFind {
+    parent = new Map();
+    add(item) {
+        if (!this.parent.has(item)) {
+            this.parent.set(item, item);
+        }
+    }
+    find(item) {
+        this.add(item);
+        let root = item;
+        while (this.parent.get(root) !== root) {
+            const next = this.parent.get(root);
+            if (next === undefined) {
+                throw new Error('union-find parent missing during root traversal');
+            }
+            root = next;
+        }
+        // Path compression — walk again, point each to root.
+        let cursor = item;
+        while (this.parent.get(cursor) !== root) {
+            const next = this.parent.get(cursor);
+            if (next === undefined) {
+                throw new Error('union-find parent missing during path compression');
+            }
+            this.parent.set(cursor, root);
+            cursor = next;
+        }
+        return root;
+    }
+    union(a, b) {
+        const ra = this.find(a);
+        const rb = this.find(b);
+        if (ra !== rb) {
+            this.parent.set(ra, rb);
+        }
+    }
+    roots() {
+        return [...this.parent.keys()].filter((k) => this.find(k) === k);
+    }
+}
+function transitiveAncestors(modelName, direct, visited = new Set()) {
+    if (visited.has(modelName)) {
+        return [];
+    }
+    visited.add(modelName);
+    const parents = direct.get(modelName) ?? [];
+    const out = new Set();
+    for (const parent of parents) {
+        out.add(parent);
+        for (const a of transitiveAncestors(parent, direct, visited)) {
+            out.add(a);
+        }
+    }
+    return [...out].sort();
+}
+export function buildMetricFlowGraph(project) {
+    // Index: semantic_model name → path, measure_name → semantic_model_path.
+    const pathByModelName = new Map();
+    const semanticModelPathToName = new Map();
+    const measureOwnerPath = new Map();
+    for (const sm of project.semanticModels) {
+        pathByModelName.set(sm.name, sm.path);
+        semanticModelPathToName.set(sm.path, sm.name);
+        for (const mName of sm.measureNames) {
+            if (!measureOwnerPath.has(mName)) {
+                measureOwnerPath.set(mName, sm.path);
+            }
+        }
+    }
+    // Union-find keyed by relative path. Every path that carries at least one semantic_model
+    // or at least one metric enters the structure; other YAMLs (e.g. `dbt_project.yml`) are
+    // ignored.
+    const uf = new UnionFind();
+    const participatingPaths = new Set();
+    for (const sm of project.semanticModels) {
+        uf.add(sm.path);
+        participatingPaths.add(sm.path);
+    }
+    for (const m of project.metrics) {
+        uf.add(m.path);
+        participatingPaths.add(m.path);
+    }
+    // (a) extends: unions.
+    for (const sm of project.semanticModels) {
+        for (const parent of sm.extendsFrom) {
+            const parentPath = pathByModelName.get(parent);
+            if (parentPath) {
+                uf.union(sm.path, parentPath);
+            }
+        }
+    }
+    // (b) metric → measure reference unions. For simple/cumulative, union metric file with
+    // the owner semantic_model file of the referenced measure. For derived/ratio/conversion,
+    // each referenced metric name ultimately resolves to a measure; we look it up directly
+    // in the measure index. When a metric's dependsOn item is itself a metric name (derived),
+    // we don't try to chase the chain — the transitive union still happens because the chained
+    // metric's own file will also get unioned to the underlying measure's owner.
+    for (const m of project.metrics) {
+        const candidates = [];
+        if (m.measureRef) {
+            candidates.push(m.measureRef);
+        }
+        candidates.push(...m.dependsOn);
+        for (const name of candidates) {
+            const ownerPath = measureOwnerPath.get(name);
+            if (ownerPath) {
+                uf.union(m.path, ownerPath);
+            }
+        }
+    }
+    // Group participating paths by root.
+    const groups = new Map();
+    for (const path of participatingPaths) {
+        const root = uf.find(path);
+        const list = groups.get(root) ?? [];
+        list.push(path);
+        groups.set(root, list);
+    }
+    // Build component records.
+    const components = [];
+    const componentByPath = new Map();
+    const componentByModelName = new Map();
+    // Compute leadName for each raw group before assigning ids so ordering is stable.
+    const rawComponents = [...groups.values()].map((paths) => {
+        const sortedPaths = [...paths].sort();
+        const smNames = sortedPaths
+            .map((p) => semanticModelPathToName.get(p))
+            .filter((n) => typeof n === 'string')
+            .sort();
+        const metricNames = project.metrics
+            .filter((m) => sortedPaths.includes(m.path))
+            .map((m) => m.name)
+            .sort();
+        const leadName = smNames[0] ?? metricNames[0] ?? sortedPaths[0];
+        return { paths: sortedPaths, semanticModelNames: smNames, metricNames, leadName };
+    });
+    rawComponents.sort((a, b) => a.leadName.localeCompare(b.leadName));
+    rawComponents.forEach((rc, id) => {
+        components.push({ id, ...rc });
+        for (const path of rc.paths) {
+            componentByPath.set(path, id);
+        }
+        for (const name of rc.semanticModelNames) {
+            componentByModelName.set(name, id);
+        }
+    });
+    // Extends ancestor index (used by DiffSet widening).
+    const directExtends = new Map();
+    for (const sm of project.semanticModels) {
+        directExtends.set(sm.name, [...sm.extendsFrom].sort());
+    }
+    const extendsAncestorsByModelName = new Map();
+    for (const sm of project.semanticModels) {
+        extendsAncestorsByModelName.set(sm.name, transitiveAncestors(sm.name, directExtends));
+    }
+    return {
+        components,
+        componentByModelName,
+        componentByPath,
+        pathByModelName,
+        extendsAncestorsByModelName,
+    };
+}

package/node_modules/@ktx/context/dist/ingest/adapters/metricflow/graph.test.d.ts

@@ -0,0 +1 @@
+export {};