@kaelio/ktx 0.0.0 → 0.1.0-rc.1

package/node_modules/@ktx/context/dist/ingest/local-ingest.js

@@ -0,0 +1,302 @@
+import { randomUUID } from 'node:crypto';
+import { cp, mkdir, rm } from 'node:fs/promises';
+import { isAbsolute, resolve } from 'node:path';
+import { ktxLocalStateDbPath } from '../project/index.js';
+import { planMetabaseFanoutChildren } from './adapters/metabase/fanout-planner.js';
+import { KtxYamlMetabaseSourceStateReader, LocalMetabaseDiscoveryCache } from './adapters/metabase/local-source-state-store.js';
+import { localPullConfigForAdapter } from './local-adapters.js';
+import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js';
+import { buildSyncId } from './raw-sources-paths.js';
+import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js';
+class LocalIngestPhase {
+    async updateProgress() { }
+    startPhase() {
+        return new LocalIngestPhase();
+    }
+}
+function safeSegment(kind, value) {
+    if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
+        throw new Error(`Unsafe ${kind}: ${value}`);
+    }
+    return value;
+}
+function assertConfigured(project, adapter, connectionId) {
+    if (!project.config.connections[connectionId]) {
+        throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
+    }
+    if (!project.config.ingest.adapters.includes(adapter)) {
+        throw new Error(`Adapter "${adapter}" is not enabled in ktx.yaml`);
+    }
+}
+function findAdapter(adapters, source) {
+    const adapter = adapters.find((candidate) => candidate.source === source);
+    if (!adapter) {
+        throw new Error(`Adapter "${source}" is not available for local ingest`);
+    }
+    return adapter;
+}
+function localJobContext(jobId, memoryFlow) {
+    return {
+        jobId,
+        ...(memoryFlow ? { memoryFlow } : {}),
+        startPhase() {
+            return new LocalIngestPhase();
+        },
+    };
+}
+async function copySourceDirToUpload(sourceDir, uploadDir) {
+    if (!isAbsolute(sourceDir)) {
+        throw new Error('sourceDir must be an absolute path');
+    }
+    await rm(uploadDir, { recursive: true, force: true });
+    await mkdir(uploadDir, { recursive: true });
+    await cp(resolve(sourceDir), uploadDir, { recursive: true });
+}
+async function runScheduledPullJob(options) {
+    const runtime = createLocalBundleIngestRuntime(options);
+    const jobId = options.jobId ?? runtime.nextJobId();
+    const result = await runtime.runner.run({
+        jobId,
+        connectionId: options.connectionId,
+        sourceKey: options.adapter.source,
+        trigger: options.trigger ?? 'manual_resync',
+        bundleRef: { kind: 'scheduled_pull', config: options.pullConfig },
+    }, localJobContext(jobId, options.memoryFlow));
+    const report = await runtime.store.findByJobId(jobId);
+    if (!report) {
+        throw new Error(`Local ingest report for job "${jobId}" was not created`);
+    }
+    return { result, report };
+}
+export async function runLocalIngest(options) {
+    const adapterName = safeSegment('adapter', options.adapter);
+    const connectionId = safeSegment('connection id', options.connectionId);
+    assertConfigured(options.project, adapterName, connectionId);
+    const adapter = findAdapter(options.adapters, adapterName);
+    const pullConfig = options.sourceDir
+        ? undefined
+        : await localPullConfigForAdapter(options.project, adapter, connectionId, options.pullConfigOptions);
+    const runtime = createLocalBundleIngestRuntime(options);
+    const jobId = options.jobId ?? runtime.nextJobId();
+    const bundleRef = options.sourceDir
+        ? { kind: 'upload', uploadId: jobId }
+        : { kind: 'scheduled_pull', config: pullConfig };
+    if (options.sourceDir) {
+        await copySourceDirToUpload(options.sourceDir, runtime.storage.resolveUploadDir(jobId));
+    }
+    else {
+        return runScheduledPullJob({
+            project: options.project,
+            adapters: options.adapters,
+            adapter,
+            connectionId,
+            pullConfig,
+            trigger: options.trigger,
+            jobId,
+            memoryFlow: options.memoryFlow,
+            agentRunner: options.agentRunner,
+            llmRuntime: options.llmRuntime,
+            memoryModel: options.memoryModel,
+            semanticLayerCompute: options.semanticLayerCompute,
+            queryExecutor: options.queryExecutor,
+            logger: options.logger,
+        });
+    }
+    const result = await runtime.runner.run({
+        jobId,
+        connectionId,
+        sourceKey: adapter.source,
+        trigger: options.trigger ?? (options.sourceDir ? 'upload' : 'manual_resync'),
+        bundleRef,
+    }, localJobContext(jobId, options.memoryFlow));
+    const report = await runtime.store.findByJobId(jobId);
+    if (!report) {
+        throw new Error(`Local ingest report for job "${jobId}" was not created`);
+    }
+    return { result, report };
+}
+function metabaseFanoutStatus(children) {
+    const succeeded = children.filter((child) => child.report.body.failedWorkUnits.length === 0).length;
+    if (succeeded === children.length) {
+        return 'all_succeeded';
+    }
+    if (succeeded === 0) {
+        return 'all_failed';
+    }
+    return 'partial_failure';
+}
+function metabaseFanoutTotals(children) {
+    return {
+        workUnits: children.reduce((sum, child) => sum + child.report.body.workUnits.length, 0),
+        failedWorkUnits: children.reduce((sum, child) => sum + child.report.body.failedWorkUnits.length, 0),
+    };
+}
+const METABASE_FETCH_FAILURE_UNIT = 'metabase-fetch';
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function metabaseChildJobId(metabaseDatabaseId) {
+    return `local-metabase-${metabaseDatabaseId}-${randomUUID()}`;
+}
+async function recordLocalMetabaseChildFailure(options) {
+    const store = new SqliteBundleIngestStore({ dbPath: ktxLocalStateDbPath(options.project) });
+    const syncId = buildSyncId(new Date(), options.jobId);
+    const diffSummary = { added: 0, modified: 0, deleted: 0, unchanged: 0 };
+    const reason = errorMessage(options.error);
+    const run = await store.create({
+        jobId: options.jobId,
+        connectionId: options.targetConnectionId,
+        sourceKey: 'metabase',
+        syncId,
+        trigger: options.trigger ?? 'manual_resync',
+        scopeFingerprint: null,
+    });
+    await store.markFailed(run.id);
+    const body = {
+        syncId,
+        diffSummary,
+        commitSha: null,
+        workUnits: [
+            {
+                unitKey: METABASE_FETCH_FAILURE_UNIT,
+                rawFiles: [],
+                status: 'failed',
+                reason,
+                actions: [],
+                touchedSlSources: [],
+            },
+        ],
+        failedWorkUnits: [METABASE_FETCH_FAILURE_UNIT],
+        reconciliationSkipped: true,
+        conflictsResolved: [],
+        evictionsApplied: [],
+        unmappedFallbacks: [],
+        artifactResolutions: [],
+        evictionInputs: [],
+        unresolvedCards: [],
+        supersededBy: null,
+        overrideOf: null,
+        provenanceRows: [],
+        toolTranscripts: [],
+        postProcessor: {
+            sourceKey: 'metabase',
+            status: 'failed',
+            errors: [reason],
+            warnings: [],
+            touchedSources: [],
+        },
+    };
+    const report = await store.create({
+        runId: run.id,
+        jobId: options.jobId,
+        connectionId: options.targetConnectionId,
+        sourceKey: 'metabase',
+        body,
+    });
+    return {
+        result: {
+            jobId: options.jobId,
+            runId: run.id,
+            syncId,
+            diffSummary,
+            workUnitCount: 1,
+            failedWorkUnits: [METABASE_FETCH_FAILURE_UNIT],
+            artifactsWritten: 0,
+            commitSha: null,
+        },
+        report,
+    };
+}
+export async function runLocalMetabaseIngest(options) {
+    if (options.sourceDir) {
+        throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter');
+    }
+    const metabaseConnectionId = safeSegment('metabase connection id', options.metabaseConnectionId);
+    assertConfigured(options.project, 'metabase', metabaseConnectionId);
+    const adapter = findAdapter(options.adapters, 'metabase');
+    const sourceStateReader = new KtxYamlMetabaseSourceStateReader(options.project, {
+        discoveryCache: new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(options.project) }),
+    });
+    const state = await sourceStateReader.getSourceState(metabaseConnectionId);
+    const childPlans = planMetabaseFanoutChildren({
+        metabaseConnectionId,
+        mappings: state.mappings,
+    });
+    options.progress?.onMetabaseFanoutPlanned?.({
+        metabaseConnectionId,
+        children: childPlans.map((childPlan) => ({
+            metabaseDatabaseId: childPlan.metabaseDatabaseId,
+            targetConnectionId: childPlan.targetConnectionId,
+        })),
+    });
+    const children = [];
+    for (const childPlan of childPlans) {
+        const targetConnectionId = safeSegment('target connection id', childPlan.targetConnectionId);
+        if (!options.project.config.connections[targetConnectionId]) {
+            throw new Error(`Target connection "${targetConnectionId}" is not configured in ktx.yaml`);
+        }
+        const childJobId = options.jobIdFactory?.() ?? metabaseChildJobId(childPlan.metabaseDatabaseId);
+        options.progress?.onMetabaseChildStarted?.({
+            metabaseConnectionId,
+            metabaseDatabaseId: childPlan.metabaseDatabaseId,
+            targetConnectionId,
+            jobId: childJobId,
+        });
+        let child;
+        try {
+            child = await runScheduledPullJob({
+                project: options.project,
+                adapters: options.adapters,
+                adapter,
+                connectionId: targetConnectionId,
+                pullConfig: childPlan.pullConfig,
+                trigger: options.trigger,
+                jobId: childJobId,
+                memoryFlow: options.memoryFlow,
+                agentRunner: options.agentRunner,
+                llmRuntime: options.llmRuntime,
+                memoryModel: options.memoryModel,
+                semanticLayerCompute: options.semanticLayerCompute,
+                queryExecutor: options.queryExecutor,
+                logger: options.logger,
+            });
+        }
+        catch (error) {
+            child = await recordLocalMetabaseChildFailure({
+                project: options.project,
+                jobId: childJobId,
+                targetConnectionId,
+                metabaseDatabaseId: childPlan.metabaseDatabaseId,
+                trigger: options.trigger,
+                error,
+            });
+        }
+        options.progress?.onMetabaseChildCompleted?.({
+            metabaseConnectionId,
+            metabaseDatabaseId: childPlan.metabaseDatabaseId,
+            targetConnectionId,
+            jobId: child.report.jobId,
+            status: child.report.body.failedWorkUnits.length > 0 ? 'failed' : 'done',
+        });
+        children.push({
+            jobId: child.report.jobId,
+            metabaseConnectionId,
+            metabaseDatabaseId: childPlan.metabaseDatabaseId,
+            targetConnectionId,
+            result: child.result,
+            report: child.report,
+        });
+    }
+    return {
+        metabaseConnectionId,
+        children,
+        status: metabaseFanoutStatus(children),
+        totals: metabaseFanoutTotals(children),
+    };
+}
+export async function getLocalIngestStatus(project, id) {
+    return new SqliteBundleIngestStore({ dbPath: ktxLocalStateDbPath(project) }).findReportByAnyId(id);
+}
+export async function getLatestLocalIngestStatus(project) {
+    return new SqliteBundleIngestStore({ dbPath: ktxLocalStateDbPath(project) }).findLatestReport();
+}

package/node_modules/@ktx/context/dist/ingest/local-mapping-reconcile.d.ts

@@ -0,0 +1,2 @@
+import { type KtxLocalProject } from '../project/index.js';
+export declare function seedLocalMappingStateFromKtxYaml(project: KtxLocalProject, connectionId: string): Promise<void>;

package/node_modules/@ktx/context/dist/ingest/local-mapping-reconcile.js

@@ -0,0 +1,26 @@
+import { ktxLocalStateDbPath, parseConnectionMappingBootstrap, } from '../project/index.js';
+import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js';
+function lookerMappings(bootstrap) {
+    return Object.entries(bootstrap.connectionMappings)
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([lookerConnectionName, ktxConnectionId]) => ({ lookerConnectionName, ktxConnectionId }));
+}
+export async function seedLocalMappingStateFromKtxYaml(project, connectionId) {
+    const connection = project.config.connections[connectionId];
+    if (!connection) {
+        return;
+    }
+    const bootstrap = parseConnectionMappingBootstrap(connectionId, connection);
+    if (!bootstrap) {
+        return;
+    }
+    if (bootstrap.adapter === 'metabase') {
+        return;
+    }
+    if (bootstrap.adapter === 'looker') {
+        await new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) }).applyYamlBootstrap({
+            lookerConnectionId: connectionId,
+            mappings: lookerMappings(bootstrap),
+        });
+    }
+}

package/node_modules/@ktx/context/dist/ingest/local-mapping-reconcile.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/local-mapping-reconcile.test.js

@@ -0,0 +1,61 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { ktxLocalStateDbPath } from '../project/index.js';
+import { LocalLookerRuntimeStore } from './adapters/looker/local-runtime-store.js';
+import { seedLocalMappingStateFromKtxYaml } from './local-mapping-reconcile.js';
+describe('local mapping yaml reconciliation bridge', () => {
+    let tempDir;
+    afterEach(async () => {
+        if (tempDir) {
+            await rm(tempDir, { recursive: true, force: true });
+        }
+    });
+    function projectWithConnections(connections) {
+        return {
+            projectDir: tempDir,
+            config: { connections },
+        };
+    }
+    it('does not copy Metabase mapping intent into local SQLite state', async () => {
+        tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-yaml-seed-'));
+        const project = projectWithConnections({
+            'prod-metabase': {
+                driver: 'metabase',
+                api_url: 'https://metabase.example.com',
+                mappings: {
+                    databaseMappings: { '1': 'prod-warehouse' },
+                    syncEnabled: { '1': true },
+                    syncMode: 'ONLY',
+                    selections: { collections: [12], items: [] },
+                    defaultTagNames: ['ktx'],
+                },
+            },
+            'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
+        });
+        await expect(seedLocalMappingStateFromKtxYaml(project, 'prod-metabase')).resolves.toBeUndefined();
+    });
+    it('seeds Looker local mappings from ktx.yaml mapping intent', async () => {
+        tempDir = await mkdtemp(join(tmpdir(), 'ktx-looker-yaml-seed-'));
+        const project = projectWithConnections({
+            'prod-looker': {
+                driver: 'looker',
+                base_url: 'https://looker.example.com',
+                client_id: 'client',
+                mappings: { connectionMappings: { analytics: 'prod-warehouse' } },
+            },
+            'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
+        });
+        await seedLocalMappingStateFromKtxYaml(project, 'prod-looker');
+        const store = new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) });
+        await expect(store.listConnectionMappings('prod-looker')).resolves.toMatchObject([
+            { lookerConnectionName: 'analytics', ktxConnectionId: 'prod-warehouse', source: 'ktx.yaml' },
+        ]);
+    });
+    it('does nothing for connections without mapping bootstrap intent', async () => {
+        tempDir = await mkdtemp(join(tmpdir(), 'ktx-no-yaml-seed-'));
+        const project = projectWithConnections({ warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' } });
+        await expect(seedLocalMappingStateFromKtxYaml(project, 'warehouse')).resolves.toBeUndefined();
+    });
+});

package/node_modules/@ktx/context/dist/ingest/local-metabase-ingest.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@ktx/context/dist/ingest/local-metabase-ingest.test.js

@@ -0,0 +1,227 @@
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { initKtxProject } from '../project/index.js';
+import { LocalMetabaseDiscoveryCache } from './adapters/metabase/local-source-state-store.js';
+import { getLocalIngestStatus, runLocalMetabaseIngest } from './local-ingest.js';
+class TestAgentRunner {
+    runLoop = vi.fn(async (params) => {
+        if (params.userPrompt.includes('metabase-db-2')) {
+            return { stopReason: 'error', error: new Error('database 2 failed') };
+        }
+        return { stopReason: 'natural' };
+    });
+}
+class FakeMetabaseSourceAdapter {
+    source = 'metabase';
+    skillNames = [];
+    detect() {
+        return Promise.resolve(true);
+    }
+    async fetch(pullConfig, stagedDir, ctx) {
+        const config = pullConfig;
+        await mkdir(join(stagedDir, 'cards'), { recursive: true });
+        await mkdir(join(stagedDir, 'databases'), { recursive: true });
+        await writeFile(join(stagedDir, 'cards', `${config.metabaseDatabaseId}.json`), JSON.stringify({ connectionId: ctx.connectionId, databaseId: config.metabaseDatabaseId }), 'utf-8');
+        await writeFile(join(stagedDir, 'databases', `${config.metabaseDatabaseId}.json`), JSON.stringify({ metabaseConnectionId: config.metabaseConnectionId }), 'utf-8');
+    }
+    async chunk(stagedDir) {
+        const databaseId = Number(stagedDir.match(/metabase-child-(\d+)/)?.[1] ?? 1);
+        return {
+            workUnits: [
+                {
+                    unitKey: `metabase-db-${databaseId}`,
+                    rawFiles: [`cards/${databaseId}.json`],
+                    peerFileIndex: [],
+                    dependencyPaths: [`databases/${databaseId}.json`],
+                },
+            ],
+        };
+    }
+}
+class ThrowingFetchMetabaseSourceAdapter extends FakeMetabaseSourceAdapter {
+    async fetch(pullConfig, stagedDir, ctx) {
+        const config = pullConfig;
+        if (config.metabaseDatabaseId === 2) {
+            throw new Error('Metabase fetch failed for database 2');
+        }
+        await super.fetch(pullConfig, stagedDir, ctx);
+    }
+}
+describe('runLocalMetabaseIngest', () => {
+    let tempDir;
+    let project;
+    beforeEach(async () => {
+        tempDir = await mkdtemp(join(tmpdir(), 'ktx-metabase-fanout-'));
+        project = await initKtxProject({ projectDir: tempDir, force: true });
+        project.config.connections = {
+            'prod-metabase': {
+                driver: 'metabase',
+                api_url: 'https://metabase.example.com',
+                api_key: 'literal-test-key', // pragma: allowlist secret
+            },
+            warehouse_a: { driver: 'postgres', url: 'postgres://localhost/a' },
+            warehouse_b: { driver: 'postgres', url: 'postgres://localhost/b' },
+        };
+        project.config.ingest.adapters = ['metabase'];
+    });
+    afterEach(async () => {
+        await rm(tempDir, { recursive: true, force: true });
+    });
+    async function seedMetabaseState() {
+        project.config.connections['prod-metabase'].mappings = {
+            databaseMappings: { '1': 'warehouse_a', '2': 'warehouse_b' },
+            syncEnabled: { '1': true, '2': true },
+            syncMode: 'ALL',
+            defaultTagNames: ['ktx'],
+            selections: { collections: [], items: [] },
+        };
+        const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
+        await discoveryCache.refreshDiscoveredDatabases({
+            connectionId: 'prod-metabase',
+            discovered: [
+                { id: 1, name: 'Warehouse A', engine: 'postgres', host: 'localhost', dbName: 'a' },
+                { id: 2, name: 'Warehouse B', engine: 'postgres', host: 'localhost', dbName: 'b' },
+            ],
+        });
+    }
+    it('runs one child job per sync-enabled Metabase mapping', async () => {
+        await seedMetabaseState();
+        const agentRunner = new TestAgentRunner();
+        const ids = ['metabase-child-1', 'metabase-child-3'];
+        const result = await runLocalMetabaseIngest({
+            project,
+            adapters: [new FakeMetabaseSourceAdapter()],
+            metabaseConnectionId: 'prod-metabase',
+            agentRunner,
+            jobIdFactory: () => ids.shift() ?? 'metabase-child-extra',
+        });
+        expect(result.metabaseConnectionId).toBe('prod-metabase');
+        expect(result.status).toBe('all_succeeded');
+        expect(result.children.map((child) => child.targetConnectionId)).toEqual(['warehouse_a', 'warehouse_b']);
+        expect(result.children.map((child) => child.metabaseDatabaseId)).toEqual([1, 2]);
+        expect(new Set(result.children.map((child) => child.jobId)).size).toBe(2);
+        await expect(getLocalIngestStatus(project, result.children[0].jobId)).resolves.toMatchObject({
+            jobId: result.children[0].jobId,
+            connectionId: 'warehouse_a',
+            sourceKey: 'metabase',
+        });
+    });
+    it('throws before runner work when there are no sync-enabled mapped rows', async () => {
+        project.config.connections['prod-metabase'].mappings = {
+            databaseMappings: { '1': null },
+            syncEnabled: { '1': true },
+        };
+        await expect(runLocalMetabaseIngest({
+            project,
+            adapters: [new FakeMetabaseSourceAdapter()],
+            metabaseConnectionId: 'prod-metabase',
+            agentRunner: new TestAgentRunner(),
+        })).rejects.toThrow('no sync-enabled mappings with a target connection');
+    });
+    it('seeds yaml-only Metabase mappings before the unhydrated fan-out preflight', async () => {
+        project.config.connections['prod-metabase'].mappings = {
+            databaseMappings: { '1': 'warehouse_a' },
+            syncEnabled: { '1': true },
+        };
+        const result = await runLocalMetabaseIngest({
+            project,
+            adapters: [new FakeMetabaseSourceAdapter()],
+            metabaseConnectionId: 'prod-metabase',
+            agentRunner: new TestAgentRunner(),
+            jobIdFactory: () => 'metabase-child-1',
+        });
+        expect(result.status).toBe('all_succeeded');
+        expect(result.children).toMatchObject([
+            {
+                metabaseConnectionId: 'prod-metabase',
+                metabaseDatabaseId: 1,
+                targetConnectionId: 'warehouse_a',
+            },
+        ]);
+    });
+    it('rejects source-dir uploads through the Metabase fan-out runner', async () => {
+        await expect(runLocalMetabaseIngest({
+            project,
+            adapters: [new FakeMetabaseSourceAdapter()],
+            metabaseConnectionId: 'prod-metabase',
+            agentRunner: new TestAgentRunner(),
+            sourceDir: tempDir,
+        })).rejects.toThrow('source-dir uploads are not supported for the Metabase fan-out adapter');
+    });
+    it('reports partial failure when a child job fails', async () => {
+        await seedMetabaseState();
+        const agentRunner = new TestAgentRunner();
+        const ids = ['metabase-child-1', 'metabase-child-2'];
+        const result = await runLocalMetabaseIngest({
+            project,
+            adapters: [new FakeMetabaseSourceAdapter()],
+            metabaseConnectionId: 'prod-metabase',
+            agentRunner,
+            jobIdFactory: () => ids.shift() ?? 'metabase-child-extra',
+        });
+        expect(result.status).toBe('partial_failure');
+        expect(result.totals).toEqual({ workUnits: 2, failedWorkUnits: 1 });
+        expect(result.children[1]?.report.body.failedWorkUnits).toEqual(['metabase-db-2']);
+    });
+    it('captures fetch-time child failures and continues later mappings', async () => {
+        await seedMetabaseState();
+        project.config.connections.warehouse_c = { driver: 'postgres', url: 'postgres://localhost/c' };
+        project.config.connections['prod-metabase'].mappings = {
+            databaseMappings: { '1': 'warehouse_a', '2': 'warehouse_b', '3': 'warehouse_c' },
+            syncEnabled: { '1': true, '2': true, '3': true },
+            syncMode: 'ALL',
+            defaultTagNames: ['ktx'],
+            selections: { collections: [], items: [] },
+        };
+        const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: join(tempDir, '.ktx', 'db.sqlite') });
+        await discoveryCache.refreshDiscoveredDatabases({
+            connectionId: 'prod-metabase',
+            discovered: [
+                { id: 1, name: 'Warehouse A', engine: 'postgres', host: 'localhost', dbName: 'a' },
+                { id: 2, name: 'Warehouse B', engine: 'postgres', host: 'localhost', dbName: 'b' },
+                { id: 3, name: 'Warehouse C', engine: 'postgres', host: 'localhost', dbName: 'c' },
+            ],
+        });
+        const ids = ['metabase-child-1', 'metabase-child-2', 'metabase-child-3'];
+        const result = await runLocalMetabaseIngest({
+            project,
+            adapters: [new ThrowingFetchMetabaseSourceAdapter()],
+            metabaseConnectionId: 'prod-metabase',
+            agentRunner: new TestAgentRunner(),
+            jobIdFactory: () => ids.shift() ?? 'metabase-child-extra',
+        });
+        expect(result.status).toBe('partial_failure');
+        expect(result.children.map((child) => child.jobId)).toEqual([
+            'metabase-child-1',
+            'metabase-child-2',
+            'metabase-child-3',
+        ]);
+        expect(result.children.map((child) => child.metabaseDatabaseId)).toEqual([1, 2, 3]);
+        expect(result.children.map((child) => child.targetConnectionId)).toEqual(['warehouse_a', 'warehouse_b', 'warehouse_c']);
+        expect(result.totals).toEqual({ workUnits: 3, failedWorkUnits: 1 });
+        const failed = result.children[1];
+        expect(failed.result).toMatchObject({
+            jobId: 'metabase-child-2',
+            failedWorkUnits: ['metabase-fetch'],
+            artifactsWritten: 0,
+            commitSha: null,
+        });
+        expect(failed.report.body.workUnits).toMatchObject([
+            {
+                unitKey: 'metabase-fetch',
+                status: 'failed',
+                reason: 'Metabase fetch failed for database 2',
+            },
+        ]);
+        await expect(getLocalIngestStatus(project, failed.jobId)).resolves.toMatchObject({
+            jobId: 'metabase-child-2',
+            connectionId: 'warehouse_b',
+            sourceKey: 'metabase',
+            body: {
+                failedWorkUnits: ['metabase-fetch'],
+            },
+        });
+    });
+});

package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.d.ts

@@ -0,0 +1,53 @@
+import type { KtxLocalProject } from '../project/index.js';
+import type { MemoryFlowEventSink } from './memory-flow/types.js';
+import type { IngestTrigger, SourceAdapter, WorkUnit } from './types.js';
+export type LocalIngestStatus = 'running' | 'done' | 'error';
+export interface LocalIngestDiffPaths {
+    added: string[];
+    modified: string[];
+    deleted: string[];
+    unchanged: string[];
+}
+export interface LocalIngestRunRecord {
+    runId: string;
+    jobId: string;
+    status: LocalIngestStatus;
+    adapter: string;
+    connectionId: string;
+    sourceDir: string | null;
+    syncId: string;
+    startedAt: string;
+    completedAt: string;
+    progress: number;
+    done: boolean;
+    previousRunId: string | null;
+    diffSummary: {
+        added: number;
+        modified: number;
+        deleted: number;
+        unchanged: number;
+    };
+    diffPaths: LocalIngestDiffPaths;
+    workUnitCount: number;
+    rawFileCount: number;
+    workUnits: Array<Pick<WorkUnit, 'unitKey' | 'rawFiles' | 'peerFileIndex' | 'dependencyPaths'>>;
+    evictionDeletedRawPaths: string[];
+    errors: string[];
+}
+export type LocalIngestReport = LocalIngestRunRecord & {
+    rawContentHashes: Record<string, string>;
+};
+export interface RunLocalStageOnlyIngestOptions {
+    project: KtxLocalProject;
+    adapters: SourceAdapter[];
+    adapter: string;
+    connectionId: string;
+    sourceDir?: string;
+    trigger?: IngestTrigger;
+    jobId?: string;
+    now?: () => Date;
+    dryRun?: boolean;
+    memoryFlow?: MemoryFlowEventSink;
+}
+export declare function runLocalStageOnlyIngest(options: RunLocalStageOnlyIngestOptions): Promise<LocalIngestRunRecord>;
+export declare function getLocalStageOnlyIngestStatus(project: KtxLocalProject, runId: string): Promise<LocalIngestRunRecord | null>;