@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch-scope.test.js

@@ -0,0 +1,128 @@
+import { describe, expect, it } from 'vitest';
+import { computeFetchScope, hashScope, isPathInMetabaseScope } from './fetch-scope.js';
+const BASE_CONFIG = {
+    metabaseConnectionId: 'a1b2c3d4-e5f6-4789-9abc-def012345678',
+    metabaseDatabaseId: 42,
+    defaultTagNames: [],
+    mapping: {
+        metabaseDatabaseId: 42,
+        metabaseDatabaseName: 'Analytics',
+        metabaseEngine: 'postgres',
+        targetConnectionId: 'b2c3d4e5-f6a7-4890-abcd-ef0123456789',
+    },
+};
+describe('computeFetchScope', () => {
+    it('returns { kind: "all" } for syncMode ALL', () => {
+        const scope = computeFetchScope({
+            ...BASE_CONFIG,
+            syncMode: 'ALL',
+            selections: [{ selectionType: 'item', metabaseObjectId: 5 }],
+        });
+        expect(scope).toEqual({ kind: 'all' });
+    });
+    it('returns { kind: "all-except", ... } for syncMode EXCEPT', () => {
+        const scope = computeFetchScope({
+            ...BASE_CONFIG,
+            syncMode: 'EXCEPT',
+            selections: [
+                { selectionType: 'item', metabaseObjectId: 5 },
+                { selectionType: 'collection', metabaseObjectId: 7 },
+            ],
+        });
+        expect(scope).toEqual({
+            kind: 'all-except',
+            excludeCardIds: new Set([5]),
+            excludeCollectionIds: new Set([7]),
+        });
+    });
+    it('returns { kind: "explicit", ... } for syncMode ONLY', () => {
+        const scope = computeFetchScope({
+            ...BASE_CONFIG,
+            syncMode: 'ONLY',
+            selections: [
+                { selectionType: 'item', metabaseObjectId: 5 },
+                { selectionType: 'item', metabaseObjectId: 11 },
+                { selectionType: 'collection', metabaseObjectId: 7 },
+            ],
+        });
+        expect(scope).toEqual({
+            kind: 'explicit',
+            includeCardIds: new Set([5, 11]),
+            includeCollectionIds: new Set([7]),
+        });
+    });
+    it('returns empty explicit scope for ONLY with no selections', () => {
+        const scope = computeFetchScope({ ...BASE_CONFIG, syncMode: 'ONLY', selections: [] });
+        expect(scope).toEqual({
+            kind: 'explicit',
+            includeCardIds: new Set(),
+            includeCollectionIds: new Set(),
+        });
+    });
+});
+describe('hashScope', () => {
+    it('produces the same hash for identical inputs', () => {
+        const a = hashScope({
+            kind: 'explicit',
+            includeCardIds: new Set([1, 2, 3]),
+            includeCollectionIds: new Set([7]),
+        });
+        const b = hashScope({
+            kind: 'explicit',
+            includeCardIds: new Set([3, 2, 1]),
+            includeCollectionIds: new Set([7]),
+        });
+        expect(a).toBe(b);
+    });
+    it('produces different hashes for different scopes', () => {
+        const a = hashScope({ kind: 'all' });
+        const b = hashScope({
+            kind: 'explicit',
+            includeCardIds: new Set([1]),
+            includeCollectionIds: new Set(),
+        });
+        expect(a).not.toBe(b);
+    });
+    it('produces a 64-char hex string', () => {
+        const fp = hashScope({ kind: 'all' });
+        expect(fp).toMatch(/^[0-9a-f]{64}$/);
+    });
+});
+describe('isPathInMetabaseScope', () => {
+    const allScope = { kind: 'all' };
+    const exceptScope = {
+        kind: 'all-except',
+        excludeCardIds: new Set([100]),
+        excludeCollectionIds: new Set([5]),
+    };
+    const explicitScope = {
+        kind: 'explicit',
+        includeCardIds: new Set([1, 2]),
+        includeCollectionIds: new Set([7]),
+    };
+    it('always includes sync-config.json', () => {
+        expect(isPathInMetabaseScope('sync-config.json', allScope)).toBe(true);
+        expect(isPathInMetabaseScope('sync-config.json', exceptScope)).toBe(true);
+        expect(isPathInMetabaseScope('sync-config.json', explicitScope)).toBe(true);
+    });
+    it('always includes collections/* and databases/*', () => {
+        expect(isPathInMetabaseScope('collections/5.json', explicitScope)).toBe(true);
+        expect(isPathInMetabaseScope('databases/42.json', explicitScope)).toBe(true);
+    });
+    it('for `all` scope, every cards/<id>.json is in scope', () => {
+        expect(isPathInMetabaseScope('cards/1.json', allScope)).toBe(true);
+        expect(isPathInMetabaseScope('cards/999.json', allScope)).toBe(true);
+    });
+    it('for `all-except` scope, excluded card ids are out of scope', () => {
+        expect(isPathInMetabaseScope('cards/100.json', exceptScope)).toBe(false);
+        expect(isPathInMetabaseScope('cards/101.json', exceptScope)).toBe(true);
+    });
+    it('for `explicit` scope, only include-set card ids are in scope', () => {
+        expect(isPathInMetabaseScope('cards/1.json', explicitScope)).toBe(true);
+        expect(isPathInMetabaseScope('cards/2.json', explicitScope)).toBe(true);
+        expect(isPathInMetabaseScope('cards/3.json', explicitScope)).toBe(false);
+    });
+    it('unknown path shapes default to in-scope (conservative)', () => {
+        expect(isPathInMetabaseScope('some-new-dir/whatever.json', explicitScope)).toBe(true);
+    });
+});

package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.d.ts

@@ -0,0 +1,11 @@
+import type { FetchContext } from '../../types.js';
+import type { MetabaseClientFactory } from './client-port.js';
+import type { MetabaseSourceStateReader } from './source-state-port.js';
+export interface FetchMetabaseBundleParams {
+    pullConfig: unknown;
+    stagedDir: string;
+    ctx: FetchContext;
+    clientFactory: MetabaseClientFactory;
+    sourceStateReader: MetabaseSourceStateReader;
+}
+export declare function fetchMetabaseBundle(params: FetchMetabaseBundleParams): Promise<void>;

package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.js

@@ -0,0 +1,235 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { computeFetchScope } from './fetch-scope.js';
+import { serializeCard } from './serialize-card.js';
+import { parseMetabasePullConfig, STAGED_FILES, } from './types.js';
+class IngestInputError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'IngestInputError';
+    }
+}
+const logger = {
+    log: (message) => console.log(message),
+    warn: (message) => console.warn(message),
+};
+function buildCollectionIndex(tree) {
+    const index = new Map();
+    function walk(nodes, parentId) {
+        for (const n of nodes) {
+            index.set(n.id, { id: n.id, name: n.name, parentId });
+            const children = (n.children ?? []);
+            walk(children, n.id);
+        }
+    }
+    walk(tree, null);
+    return index;
+}
+function resolvePath(index, collectionId) {
+    const path = [];
+    let cursor = collectionId;
+    const visited = new Set();
+    while (cursor !== null && cursor !== 'root') {
+        if (visited.has(cursor)) {
+            break;
+        }
+        visited.add(cursor);
+        const node = index.get(cursor);
+        if (!node) {
+            break;
+        }
+        path.unshift(node.name);
+        cursor = node.parentId;
+    }
+    return path;
+}
+export async function fetchMetabaseBundle(params) {
+    const pullConfig = parseMetabasePullConfig(params.pullConfig);
+    const syncState = await params.sourceStateReader.getSourceState(pullConfig.metabaseConnectionId);
+    const mapping = syncState.mappings.find((m) => m.metabaseDatabaseId === pullConfig.metabaseDatabaseId && m.syncEnabled);
+    if (!mapping?.targetConnectionId) {
+        throw new IngestInputError(`no sync-enabled mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId}`);
+    }
+    if (mapping.targetConnectionId !== params.ctx.connectionId) {
+        throw new IngestInputError(`mapping for database ${pullConfig.metabaseDatabaseId} does not point to connection ${params.ctx.connectionId} (points to ${mapping.targetConnectionId})`);
+    }
+    if (mapping.metabaseDatabaseName === null) {
+        throw new IngestInputError(`mapping for database ${pullConfig.metabaseDatabaseId} on Metabase connection ${pullConfig.metabaseConnectionId} is unhydrated; run \`ktx connection mapping refresh ${pullConfig.metabaseConnectionId}\` to populate metabaseDatabaseName before ingest.`);
+    }
+    const mappingDatabaseName = mapping.metabaseDatabaseName;
+    const client = await params.clientFactory.createClient(pullConfig, params.ctx);
+    try {
+        const stagedForScope = {
+            metabaseConnectionId: pullConfig.metabaseConnectionId,
+            metabaseDatabaseId: pullConfig.metabaseDatabaseId,
+            syncMode: syncState.syncMode,
+            selections: syncState.selections.map((s) => ({
+                selectionType: s.selectionType,
+                metabaseObjectId: s.metabaseObjectId,
+            })),
+            defaultTagNames: syncState.defaultTagNames,
+            mapping: {
+                metabaseDatabaseId: mapping.metabaseDatabaseId,
+                metabaseDatabaseName: mappingDatabaseName,
+                metabaseEngine: mapping.metabaseEngine,
+                targetConnectionId: mapping.targetConnectionId,
+            },
+        };
+        const scope = computeFetchScope(stagedForScope);
+        const collectionTree = await client.getCollectionTree();
+        const collectionIndex = buildCollectionIndex(collectionTree);
+        await mkdir(join(params.stagedDir, STAGED_FILES.cardsDir), { recursive: true });
+        await mkdir(join(params.stagedDir, STAGED_FILES.collectionsDir), { recursive: true });
+        await mkdir(join(params.stagedDir, STAGED_FILES.databasesDir), { recursive: true });
+        const cardIdsToFetch = await resolveCardIdsToFetch(client, scope, pullConfig.metabaseDatabaseId, logger);
+        const referencedCollectionIds = new Set();
+        let writtenCards = 0;
+        const fetched = new Set();
+        const queue = [...cardIdsToFetch];
+        const unresolvedCards = [];
+        while (queue.length > 0) {
+            const cardId = queue.shift();
+            if (cardId === undefined) {
+                continue;
+            }
+            if (fetched.has(cardId)) {
+                continue;
+            }
+            fetched.add(cardId);
+            let fullCard;
+            try {
+                fullCard = await client.getCard(cardId);
+            }
+            catch (e) {
+                logger.warn(`failed to load card ${cardId}: ${e instanceof Error ? e.message : String(e)}`);
+                continue;
+            }
+            if (fullCard.database_id !== pullConfig.metabaseDatabaseId) {
+                continue;
+            }
+            if (fullCard.archived) {
+                continue;
+            }
+            const resolvedResult = await client.getResolvedSql(fullCard).then((sql) => ({ ok: true, sql }), (err) => ({ ok: false, err }));
+            if (!resolvedResult.ok || resolvedResult.sql === null) {
+                const reason = classifyResolutionFailure(resolvedResult);
+                const errorMessage = resolvedResult.ok
+                    ? undefined
+                    : resolvedResult.err instanceof Error
+                        ? resolvedResult.err.message
+                        : String(resolvedResult.err);
+                unresolvedCards.push({
+                    cardId,
+                    name: fullCard.name,
+                    reason,
+                    errorMessage,
+                });
+                logger.warn(`[metabase.fetch] card ${cardId} ("${fullCard.name}") dropped; reason=${reason}`);
+                continue;
+            }
+            const resolved = resolvedResult.sql;
+            const collectionPath = fullCard.collection_id && fullCard.collection_id !== 'root'
+                ? resolvePath(collectionIndex, fullCard.collection_id)
+                : [];
+            const staged = serializeCard({
+                card: fullCard,
+                resolvedSql: resolved.resolvedSql,
+                templateTags: resolved.templateTags ?? [],
+                collectionPath,
+                resolutionStatus: resolved.resolutionStatus,
+            });
+            await writeFile(join(params.stagedDir, STAGED_FILES.cardsDir, `${fullCard.id}.json`), JSON.stringify(staged, null, 2), 'utf-8');
+            writtenCards += 1;
+            if (typeof fullCard.collection_id === 'number') {
+                referencedCollectionIds.add(fullCard.collection_id);
+            }
+            if (scope.kind === 'explicit') {
+                for (const refId of staged.referencedCardIds) {
+                    if (!fetched.has(refId)) {
+                        queue.push(refId);
+                    }
+                }
+            }
+        }
+        for (const colId of referencedCollectionIds) {
+            const node = collectionIndex.get(colId);
+            if (!node) {
+                continue;
+            }
+            const file = {
+                metabaseId: node.id,
+                name: node.name,
+                parentId: node.parentId ?? 'root',
+            };
+            await writeFile(join(params.stagedDir, STAGED_FILES.collectionsDir, `${colId}.json`), JSON.stringify(file, null, 2), 'utf-8');
+        }
+        const databaseFile = {
+            metabaseDatabaseId: mapping.metabaseDatabaseId,
+            metabaseDatabaseName: mappingDatabaseName,
+            metabaseEngine: mapping.metabaseEngine,
+            targetConnectionId: mapping.targetConnectionId,
+        };
+        await writeFile(join(params.stagedDir, STAGED_FILES.databasesDir, `${mapping.metabaseDatabaseId}.json`), JSON.stringify(databaseFile, null, 2), 'utf-8');
+        await writeFile(join(params.stagedDir, STAGED_FILES.syncConfig), JSON.stringify(stagedForScope, null, 2), 'utf-8');
+        if (unresolvedCards.length > 0) {
+            await writeFile(join(params.stagedDir, STAGED_FILES.unresolvedCards), JSON.stringify(unresolvedCards, null, 2), 'utf-8');
+        }
+        logger.log(`wrote ${writtenCards} cards for database ${pullConfig.metabaseDatabaseId} -> ${mapping.targetConnectionId} (scope=${scope.kind}); unresolved=${unresolvedCards.length}`);
+    }
+    finally {
+        await client.cleanup();
+    }
+}
+function classifyResolutionFailure(r) {
+    if (r.ok && r.sql === null) {
+        return 'api_500';
+    }
+    if (!r.ok) {
+        const msg = r.err instanceof Error ? r.err.message : String(r.err);
+        if (msg.includes('Cycle detected')) {
+            return 'cycle';
+        }
+        if (msg.includes('no native query')) {
+            return 'missing_native';
+        }
+    }
+    return 'unknown';
+}
+/**
+ * Resolve the initial set of card ids to fetch based on the scope. For `all`
+ * and `all-except`, this fans out to `getAllCards()` and filters by
+ * `database_id` + `excludeCardIds` / `excludeCollectionIds`. For `explicit`,
+ * this walks the selection: direct item ids + members of selected collections
+ * (via `getCollectionItems`). The closure over `{{#N}}` references is applied
+ * later in the main fetch loop.
+ */
+async function resolveCardIdsToFetch(client, scope, metabaseDatabaseId, logger) {
+    if (scope.kind === 'all' || scope.kind === 'all-except') {
+        const all = await client.getAllCards();
+        const matching = all.filter((c) => !c.archived && c.database_id === metabaseDatabaseId);
+        if (scope.kind === 'all') {
+            return matching.map((c) => c.id);
+        }
+        return matching
+            .filter((c) => !scope.excludeCardIds.has(c.id))
+            .filter((c) => typeof c.collection_id !== 'number' || !scope.excludeCollectionIds.has(c.collection_id))
+            .map((c) => c.id);
+    }
+    const ids = new Set(scope.includeCardIds);
+    for (const colId of scope.includeCollectionIds) {
+        let items;
+        try {
+            items = await client.getCollectionItems(colId);
+        }
+        catch (e) {
+            logger.warn(`failed to list collection ${colId}: ${e instanceof Error ? e.message : String(e)}`);
+            continue;
+        }
+        for (const item of items) {
+            if (item.model === 'card' || item.model === 'dataset' || item.model === 'metric') {
+                ids.add(item.id);
+            }
+        }
+    }
+    return [...ids];
+}

package/node_modules/@ktx/context/dist/ingest/adapters/metabase/fetch.test.d.ts

@@ -0,0 +1 @@
+export {};