@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/dist/ingest-report-file.d.ts

@@ -0,0 +1,2 @@
+import { type IngestReportSnapshot } from '@ktx/context/ingest';
+export declare function readIngestReportSnapshotFile(reportFile: string): Promise<IngestReportSnapshot>;

package/dist/ingest-report-file.js

@@ -0,0 +1,20 @@
+import { readFile } from 'node:fs/promises';
+import { parseIngestReportSnapshot } from '@ktx/context/ingest';
+export async function readIngestReportSnapshotFile(reportFile) {
+    const raw = await readFile(reportFile, 'utf-8');
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Invalid JSON in ingest report file ${reportFile}: ${message}`);
+    }
+    try {
+        return parseIngestReportSnapshot(parsed);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Invalid ingest report file ${reportFile}: ${message}`);
+    }
+}

package/dist/ingest-report-file.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/ingest-report-file.test.js

@@ -0,0 +1,63 @@
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { readIngestReportSnapshotFile } from './ingest-report-file.js';
+function reportSnapshot() {
+    return {
+        id: 'report-1',
+        runId: 'run-1',
+        jobId: 'job-1',
+        connectionId: 'warehouse',
+        sourceKey: 'metabase',
+        createdAt: '2026-04-30T12:00:00.000Z',
+        body: {
+            syncId: 'sync-1',
+            diffSummary: { added: 1, modified: 0, deleted: 0, unchanged: 0 },
+            commitSha: null,
+            workUnits: [],
+            failedWorkUnits: [],
+            reconciliationSkipped: true,
+            conflictsResolved: [],
+            evictionsApplied: [],
+            unmappedFallbacks: [],
+            evictionInputs: [],
+            unresolvedCards: [],
+            supersededBy: null,
+            overrideOf: null,
+            provenanceRows: [],
+            toolTranscripts: [],
+        },
+    };
+}
+describe('readIngestReportSnapshotFile', () => {
+    let tempDir;
+    beforeEach(async () => {
+        tempDir = await mkdtemp(join(tmpdir(), 'ktx-report-file-'));
+    });
+    afterEach(async () => {
+        await rm(tempDir, { recursive: true, force: true });
+    });
+    it('reads and parses an ingest report JSON file', async () => {
+        const reportPath = join(tempDir, 'report.json');
+        await writeFile(reportPath, `${JSON.stringify(reportSnapshot(), null, 2)}\n`, 'utf-8');
+        const report = await readIngestReportSnapshotFile(reportPath);
+        expect(report).toMatchObject({
+            id: 'report-1',
+            runId: 'run-1',
+            jobId: 'job-1',
+            connectionId: 'warehouse',
+            sourceKey: 'metabase',
+        });
+    });
+    it('reports invalid JSON with the file path', async () => {
+        const reportPath = join(tempDir, 'invalid.json');
+        await writeFile(reportPath, '{not json', 'utf-8');
+        await expect(readIngestReportSnapshotFile(reportPath)).rejects.toThrow(`Invalid JSON in ingest report file ${reportPath}`);
+    });
+    it('reports schema failures with the file path', async () => {
+        const reportPath = join(tempDir, 'wrong-shape.json');
+        await writeFile(reportPath, JSON.stringify({ id: 'report-1' }), 'utf-8');
+        await expect(readIngestReportSnapshotFile(reportPath)).rejects.toThrow(`Invalid ingest report file ${reportPath}`);
+    });
+});

package/dist/ingest.d.ts

@@ -0,0 +1,53 @@
+import { type RunLocalIngestOptions, runLocalIngest, runLocalMetabaseIngest } from '@ktx/context/ingest';
+import { readIngestReportSnapshotFile } from './ingest-report-file.js';
+import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
+import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
+import { type KtxMemoryFlowStdin } from './memory-flow-interactive.js';
+import { renderMemoryFlowTui, startLiveMemoryFlowTui } from './memory-flow-tui.js';
+export type KtxIngestOutputMode = 'plain' | 'json' | 'viz';
+type KtxIngestInputMode = 'auto' | 'disabled';
+export type KtxIngestArgs = {
+    command: 'run';
+    projectDir: string;
+    connectionId: string;
+    adapter: string;
+    sourceDir?: string;
+    databaseIntrospectionUrl?: string;
+    cliVersion?: string;
+    runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
+    debugLlmRequestFile?: string;
+    outputMode: KtxIngestOutputMode;
+    inputMode?: KtxIngestInputMode;
+} | {
+    command: 'status' | 'replay' | 'watch';
+    projectDir: string;
+    runId?: string;
+    reportFile?: string;
+    outputMode: KtxIngestOutputMode;
+    inputMode?: KtxIngestInputMode;
+};
+interface KtxIngestIo {
+    stdin?: KtxMemoryFlowStdin;
+    stdout: {
+        isTTY?: boolean;
+        columns?: number;
+        write(chunk: string): void;
+    };
+    stderr: {
+        write(chunk: string): void;
+    };
+}
+interface KtxIngestDeps {
+    jobIdFactory?: () => string;
+    now?: () => Date;
+    createAdapters?: typeof createKtxCliLocalIngestAdapters;
+    runLocalIngest?: typeof runLocalIngest;
+    runLocalMetabaseIngest?: typeof runLocalMetabaseIngest;
+    readReportFile?: typeof readIngestReportSnapshotFile;
+    renderStoredMemoryFlow?: typeof renderMemoryFlowTui;
+    startLiveMemoryFlow?: typeof startLiveMemoryFlowTui;
+    env?: NodeJS.ProcessEnv;
+    localIngestOptions?: Pick<RunLocalIngestOptions, 'agentRunner' | 'llmProvider' | 'memoryModel' | 'semanticLayerCompute' | 'queryExecutor' | 'logger' | 'pullConfigOptions'>;
+}
+export declare function runKtxIngest(args: KtxIngestArgs, io?: KtxIngestIo, deps?: KtxIngestDeps): Promise<number>;
+export {};

package/dist/ingest.js

@@ -0,0 +1,288 @@
+import { buildMemoryFlowViewModel, createMemoryFlowLiveBuffer, formatMemoryFlowFinalSummary, getLatestLocalIngestStatus, getLocalIngestStatus, ingestReportToMemoryFlowReplay, renderMemoryFlowReplay, runLocalIngest, runLocalMetabaseIngest, } from '@ktx/context/ingest';
+import { loadKtxProject } from '@ktx/context/project';
+import { readIngestReportSnapshotFile } from './ingest-report-file.js';
+import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
+import { renderMemoryFlowInteractively } from './memory-flow-interactive.js';
+import { renderMemoryFlowTui, startLiveMemoryFlowTui, } from './memory-flow-tui.js';
+import { resolveVizFallback, warnVizFallbackOnce } from './viz-fallback.js';
+import { profileMark } from './startup-profile.js';
+profileMark('module:ingest');
+function reportStatus(report) {
+    return report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
+}
+function reportActionCounts(report) {
+    const actions = report.body.workUnits.flatMap((workUnit) => workUnit.actions);
+    return {
+        wikiCount: actions.filter((action) => action.target === 'wiki').length,
+        slCount: actions.filter((action) => action.target === 'sl').length,
+    };
+}
+function writeReportStatus(report, io) {
+    const counts = reportActionCounts(report);
+    io.stdout.write(`Report: ${report.id}\n`);
+    io.stdout.write(`Run: ${report.runId}\n`);
+    io.stdout.write(`Job: ${report.jobId}\n`);
+    io.stdout.write(`Status: ${reportStatus(report)}\n`);
+    io.stdout.write(`Adapter: ${report.sourceKey}\n`);
+    io.stdout.write(`Connection: ${report.connectionId}\n`);
+    io.stdout.write(`Sync: ${report.body.syncId}\n`);
+    io.stdout.write(`Diff: +${report.body.diffSummary.added}/~${report.body.diffSummary.modified}/-${report.body.diffSummary.deleted}/=${report.body.diffSummary.unchanged}\n`);
+    io.stdout.write(`Work units: ${report.body.workUnits.length}\n`);
+    io.stdout.write(`Saved memory: ${counts.wikiCount} wiki, ${counts.slCount} SL\n`);
+    io.stdout.write(`Provenance rows: ${report.body.provenanceRows.length}\n`);
+}
+function writeMetabaseFanoutStatus(result, io) {
+    io.stdout.write(`Metabase fan-out: ${result.status}\n`);
+    io.stdout.write(`Source: ${result.metabaseConnectionId}\n`);
+    io.stdout.write(`Children: ${result.children.length}\n`);
+    if (result.totals) {
+        io.stdout.write(`Work units: ${result.totals.workUnits}\n`);
+        io.stdout.write(`Failed work units: ${result.totals.failedWorkUnits}\n`);
+    }
+    for (const child of result.children) {
+        const status = reportStatus(child.report);
+        io.stdout.write(`- target=${child.targetConnectionId} database=${child.metabaseDatabaseId} status=${status} job=${child.jobId}\n`);
+    }
+}
+function pluralize(count, singular, plural = `${singular}s`) {
+    return `${count} ${count === 1 ? singular : plural}`;
+}
+function createMetabaseFanoutProgress(connectionId, io) {
+    io.stdout.write(`Metabase ingest: ${connectionId}\n`);
+    io.stdout.write('Checking mappings and scheduled-pull targets...\n');
+    return {
+        onMetabaseFanoutPlanned(event) {
+            io.stdout.write(`Targets: ${pluralize(event.children.length, 'mapped database')}\n`);
+            for (const child of event.children) {
+                io.stdout.write(`- database=${child.metabaseDatabaseId} target=${child.targetConnectionId} status=queued\n`);
+            }
+        },
+        onMetabaseChildStarted(event) {
+            io.stdout.write(`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=running job=${event.jobId}\n`);
+        },
+        onMetabaseChildCompleted(event) {
+            io.stdout.write(`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=${event.status} job=${event.jobId}\n`);
+        },
+    };
+}
+function writeReportJson(report, io) {
+    io.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+}
+function assertReportMatchesReplayId(report, requestedId, reportFile) {
+    const validIds = [report.id, report.runId, report.jobId];
+    if (!validIds.includes(requestedId)) {
+        throw new Error(`Report file ${reportFile} does not match ingest replay id "${requestedId}"; expected one of ${validIds.join(', ')}`);
+    }
+}
+async function readStoredIngestReport(project, runId) {
+    return runId ? await getLocalIngestStatus(project, runId) : await getLatestLocalIngestStatus(project);
+}
+function isInteractiveTerminal(io) {
+    return io.stdout.isTTY === true;
+}
+function terminalWidth(io) {
+    return io.stdout.columns ?? process.stdout.columns;
+}
+function isTuiCapableIo(io) {
+    return (io.stdin?.isTTY === true &&
+        io.stdout.isTTY === true &&
+        typeof io.stdin.on === 'function' &&
+        typeof io.stdin.setRawMode === 'function' &&
+        typeof io.stdout.write === 'function');
+}
+function effectiveIngestOutputMode(outputMode, io, env, options = {}) {
+    if (outputMode !== 'viz') {
+        return outputMode;
+    }
+    const fallback = resolveVizFallback(io, env, { requireInput: options.requireInput ?? false });
+    if (!fallback.shouldDegrade) {
+        return outputMode;
+    }
+    warnVizFallbackOnce(io, fallback);
+    return 'plain';
+}
+function writeMemoryFlowInput(input, io, options = {}) {
+    if (options.clear) {
+        io.stdout.write('\u001b[2J\u001b[H');
+    }
+    const view = buildMemoryFlowViewModel(input);
+    io.stdout.write(renderMemoryFlowReplay(view, { terminalWidth: terminalWidth(io) }));
+}
+function initialRunMemoryFlowInput(args, runId) {
+    return {
+        runId,
+        connectionId: args.connectionId,
+        adapter: args.adapter,
+        status: 'running',
+        sourceDir: args.sourceDir ?? null,
+        syncId: 'pending',
+        errors: [],
+        events: [],
+        plannedWorkUnits: [],
+        details: { actions: [], provenance: [], transcripts: [] },
+    };
+}
+function managedDaemonOptionsForIngestRun(args, io) {
+    if (args.databaseIntrospectionUrl || !args.cliVersion || !args.runtimeInstallPolicy) {
+        return undefined;
+    }
+    return {
+        cliVersion: args.cliVersion,
+        installPolicy: args.runtimeInstallPolicy,
+        io,
+    };
+}
+async function writeReportRecord(report, outputMode, io, options = {}) {
+    if (outputMode === 'json') {
+        writeReportJson(report, io);
+        return;
+    }
+    const resolvedOutputMode = effectiveIngestOutputMode(outputMode, io, options.env ?? process.env, {
+        requireInput: options.interactive === true,
+    });
+    if (resolvedOutputMode === 'viz') {
+        const input = ingestReportToMemoryFlowReplay(report, { provenanceRowCount: report.body.provenanceRows.length });
+        if (options.interactive === true) {
+            if (io.stdin?.isTTY === true) {
+                const renderStoredMemoryFlow = options.renderStoredMemoryFlow ?? renderMemoryFlowTui;
+                if (isTuiCapableIo(io) && (await renderStoredMemoryFlow(input, io))) {
+                    return;
+                }
+                await renderMemoryFlowInteractively(input, io);
+                return;
+            }
+            writeMemoryFlowInput(input, io);
+            return;
+        }
+        writeMemoryFlowInput(input, io);
+        return;
+    }
+    writeReportStatus(report, io);
+}
+export async function runKtxIngest(args, io = process, deps = {}) {
+    try {
+        const project = await loadKtxProject({ projectDir: args.projectDir });
+        const env = deps.env ?? process.env;
+        if (args.command === 'run') {
+            const createAdapters = deps.createAdapters ?? createKtxCliLocalIngestAdapters;
+            const executeLocalIngest = deps.runLocalIngest ?? runLocalIngest;
+            const localIngestOptions = deps.localIngestOptions ?? {};
+            const managedDaemon = managedDaemonOptionsForIngestRun(args, io);
+            const adapterOptions = {
+                ...(localIngestOptions.pullConfigOptions ?? {}),
+                ...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}),
+                ...(managedDaemon ? { managedDaemon } : {}),
+                ...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}),
+            };
+            if (args.adapter === 'metabase' && args.sourceDir) {
+                throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter');
+            }
+            if (args.adapter === 'metabase') {
+                const executeMetabaseFanout = deps.runLocalMetabaseIngest ?? runLocalMetabaseIngest;
+                const progress = args.outputMode === 'json' ? undefined : createMetabaseFanoutProgress(args.connectionId, io);
+                const result = await executeMetabaseFanout({
+                    project,
+                    adapters: createAdapters(project, adapterOptions),
+                    metabaseConnectionId: args.connectionId,
+                    ...localIngestOptions,
+                    trigger: 'manual_resync',
+                    jobIdFactory: deps.jobIdFactory,
+                    ...(progress ? { progress } : {}),
+                });
+                if (args.outputMode === 'json') {
+                    io.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+                }
+                else {
+                    writeMetabaseFanoutStatus(result, io);
+                }
+                return 0;
+            }
+            const jobId = deps.jobIdFactory?.();
+            let liveTui = null;
+            const runOutputMode = effectiveIngestOutputMode(args.outputMode, io, env, {
+                requireInput: (args.inputMode ?? 'auto') === 'auto',
+            });
+            const shouldUseLiveViz = runOutputMode === 'viz' && (args.inputMode ?? 'auto') === 'auto' && isInteractiveTerminal(io);
+            const initialMemoryFlow = shouldUseLiveViz ? initialRunMemoryFlowInput(args, jobId ?? 'pending') : undefined;
+            let latestMemoryFlowSnapshot = initialMemoryFlow ?? null;
+            if (initialMemoryFlow && isTuiCapableIo(io)) {
+                const startLiveMemoryFlow = deps.startLiveMemoryFlow ?? startLiveMemoryFlowTui;
+                liveTui = await startLiveMemoryFlow(initialMemoryFlow, io);
+            }
+            const memoryFlow = initialMemoryFlow
+                ? createMemoryFlowLiveBuffer(initialMemoryFlow, {
+                    onChange: (snapshot) => {
+                        latestMemoryFlowSnapshot = snapshot;
+                        if (liveTui && !liveTui.isClosed()) {
+                            liveTui.update(snapshot);
+                            return;
+                        }
+                        if (!liveTui) {
+                            writeMemoryFlowInput(snapshot, io, { clear: true });
+                        }
+                    },
+                })
+                : undefined;
+            try {
+                const result = await executeLocalIngest({
+                    project,
+                    adapters: createAdapters(project, adapterOptions),
+                    adapter: args.adapter,
+                    connectionId: args.connectionId,
+                    sourceDir: args.sourceDir,
+                    trigger: 'manual_resync',
+                    jobId,
+                    ...localIngestOptions,
+                    pullConfigOptions: adapterOptions,
+                    ...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}),
+                    ...(memoryFlow ? { memoryFlow } : {}),
+                });
+                if (memoryFlow) {
+                    latestMemoryFlowSnapshot = memoryFlow.snapshot();
+                    liveTui?.close();
+                    liveTui = null;
+                    io.stdout.write(formatMemoryFlowFinalSummary(latestMemoryFlowSnapshot));
+                    return 0;
+                }
+                await writeReportRecord(result.report, runOutputMode, io, {
+                    interactive: (args.inputMode ?? 'auto') === 'auto',
+                    renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
+                    env,
+                });
+                return 0;
+            }
+            finally {
+                liveTui?.close();
+            }
+        }
+        if (args.reportFile) {
+            const readReportFile = deps.readReportFile ?? readIngestReportSnapshotFile;
+            const report = await readReportFile(args.reportFile);
+            if (args.runId) {
+                assertReportMatchesReplayId(report, args.runId, args.reportFile);
+            }
+            await writeReportRecord(report, args.outputMode, io, {
+                interactive: (args.inputMode ?? 'auto') === 'auto',
+                renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
+                env,
+            });
+            return 0;
+        }
+        const report = await readStoredIngestReport(project, args.runId);
+        if (!report) {
+            throw new Error(args.runId
+                ? `Local ingest run or report "${args.runId}" was not found`
+                : 'No local ingest reports were found. Run `ktx ingest --all` first.');
+        }
+        await writeReportRecord(report, args.outputMode, io, {
+            interactive: (args.inputMode ?? 'auto') === 'auto',
+            renderStoredMemoryFlow: deps.renderStoredMemoryFlow,
+            env,
+        });
+        return 0;
+    }
+    catch (error) {
+        io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+        return 1;
+    }
+}

package/dist/ingest.test.d.ts

@@ -0,0 +1 @@
+export {};