@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/dist/context-build-view.js

@@ -0,0 +1,309 @@
+import { spawn } from 'node:child_process';
+import { mkdirSync, openSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { buildPublicIngestPlan, executePublicIngestTarget } from './public-ingest.js';
+import { formatDuration } from './demo-metrics.js';
+import { profileMark } from './startup-profile.js';
+profileMark('module:context-build-view');
+const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+const ESC = String.fromCharCode(0x1b);
+// --- Rendering ---
+function green(text) {
+    return `${ESC}[32m${text}${ESC}[39m`;
+}
+function red(text) {
+    return `${ESC}[31m${text}${ESC}[39m`;
+}
+function cyan(text) {
+    return `${ESC}[36m${text}${ESC}[39m`;
+}
+function dim(text) {
+    return `${ESC}[2m${text}${ESC}[22m`;
+}
+function statusIcon(status, frame, styled) {
+    if (!styled) {
+        switch (status) {
+            case 'done':
+                return '✓';
+            case 'failed':
+                return '✗';
+            case 'running':
+                return SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋';
+            default:
+                return '·';
+        }
+    }
+    switch (status) {
+        case 'done':
+            return green('✓');
+        case 'failed':
+            return red('✗');
+        case 'running':
+            return cyan(SPINNER_FRAMES[frame % SPINNER_FRAMES.length] ?? '⠋');
+        default:
+            return dim('·');
+    }
+}
+function targetDetail(target, styled) {
+    if (target.status === 'done') {
+        const parts = [];
+        if (target.summaryText)
+            parts.push(target.summaryText);
+        parts.push(formatDuration(target.elapsedMs));
+        return parts.join(' · ');
+    }
+    if (target.status === 'failed') {
+        return styled ? red('failed') : 'failed';
+    }
+    if (target.status === 'running') {
+        return target.detailLine ?? (target.target.operation === 'scan' ? 'scanning...' : 'ingesting...');
+    }
+    return styled ? dim('queued') : 'queued';
+}
+function columnWidth(state) {
+    const all = [...state.primarySources, ...state.contextSources];
+    return Math.max(12, ...all.map((t) => t.target.connectionId.length)) + 2;
+}
+function renderTargetLine(target, frame, styled, width) {
+    return `    ${statusIcon(target.status, frame, styled)} ${target.target.connectionId.padEnd(width)} ${targetDetail(target, styled)}`;
+}
+function renderTargetGroup(label, targets, frame, styled, width) {
+    if (targets.length === 0)
+        return [];
+    return ['', `  ${label}:`, ...targets.map((t) => renderTargetLine(t, frame, styled, width))];
+}
+function resumeCommand(projectDir) {
+    return projectDir ? `ktx setup --project-dir ${projectDir}` : 'ktx setup';
+}
+export function renderContextBuildView(state, options = {}) {
+    const styled = options.styled ?? true;
+    const width = columnWidth(state);
+    const lines = [
+        '',
+        'Building KTX context',
+        '─────────────────────',
+        ...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width),
+        ...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width),
+        '',
+    ];
+    const hasActive = [...state.primarySources, ...state.contextSources].some((t) => t.status === 'running' || t.status === 'queued');
+    if (options.showHint && hasActive) {
+        const hint = `  d to detach · ${resumeCommand(options.projectDir)} to resume`;
+        lines.push(styled ? dim(hint) : hint);
+        lines.push('');
+    }
+    return `${lines.join('\n')}\n`;
+}
+// --- IO Capture ---
+const ESC_K_RE = new RegExp(`${ESC.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\[K`, 'g');
+export function extractProgressMessage(chunk) {
+    const cleaned = chunk.replace(/^\r/, '').replace(ESC_K_RE, '').replace(/\n$/, '').trim();
+    const match = cleaned.match(/^\[(\d+)%\]\s*(.+)$/);
+    return match ? `[${match[1]}%] ${match[2]}` : null;
+}
+export function parseScanSummary(output) {
+    const match = output.match(/(\d+) changes? across (\d+) tables?/);
+    return match ? `${match[2]} tables` : null;
+}
+export function parseIngestSummary(output) {
+    const parts = [];
+    const workUnits = output.match(/Work units: (\d+)/);
+    if (workUnits)
+        parts.push(`${workUnits[1]} work units`);
+    const savedMemory = output.match(/Saved memory: (.+)/);
+    if (savedMemory)
+        parts.push(savedMemory[1]);
+    return parts.length > 0 ? parts.join(' · ') : null;
+}
+function createCaptureIo(onProgress, isTTY) {
+    let buffer = '';
+    return {
+        io: {
+            stdout: {
+                isTTY,
+                write(chunk) {
+                    buffer += chunk;
+                    const progress = extractProgressMessage(chunk);
+                    if (progress)
+                        onProgress(progress);
+                },
+            },
+            stderr: {
+                write(chunk) {
+                    buffer += chunk;
+                },
+            },
+        },
+        captured: () => buffer,
+    };
+}
+// --- Repaint ---
+function createRepainter(io) {
+    let lastLineCount = 0;
+    return {
+        paint(content) {
+            if (lastLineCount > 0) {
+                io.stdout.write(`${ESC}[${lastLineCount}A\r`);
+            }
+            io.stdout.write(content);
+            io.stdout.write(`${ESC}[J`);
+            lastLineCount = (content.match(/\n/g) ?? []).length;
+        },
+    };
+}
+// --- Background build ---
+function resolveKtxEntryScript() {
+    const argv1 = process.argv[1];
+    if (argv1 && (argv1.endsWith('.js') || argv1.endsWith('.ts') || argv1.endsWith('.mjs'))) {
+        return argv1;
+    }
+    return null;
+}
+function spawnBackgroundBuild(projectDir) {
+    const entryScript = resolveKtxEntryScript();
+    if (!entryScript)
+        return null;
+    const resolvedDir = resolve(projectDir);
+    const logDir = join(resolvedDir, '.ktx', 'setup');
+    mkdirSync(logDir, { recursive: true });
+    const logPath = join(logDir, 'context-build.log');
+    const logFd = openSync(logPath, 'w');
+    const child = spawn(process.execPath, [entryScript, 'setup', 'context', 'build', '--project-dir', resolvedDir, '--no-input'], { detached: true, stdio: ['ignore', logFd, logFd] });
+    child.unref();
+    return { logPath };
+}
+// --- Keystroke handling ---
+function defaultSetupKeystroke(onDetach, onCtrlC) {
+    const stdin = process.stdin;
+    if (!stdin.isTTY || typeof stdin.setRawMode !== 'function') {
+        return null;
+    }
+    stdin.setRawMode(true);
+    stdin.resume();
+    const onData = (data) => {
+        const char = data.toString();
+        if (char === 'd' || char === 'D')
+            onDetach();
+        else if (char === '\x03')
+            onCtrlC();
+    };
+    stdin.on('data', onData);
+    return () => {
+        stdin.off('data', onData);
+        if (typeof stdin.setRawMode === 'function')
+            stdin.setRawMode(false);
+        stdin.pause();
+    };
+}
+// --- Orchestration ---
+function makeTargetState(target) {
+    return { target, status: 'queued', detailLine: null, summaryText: null, startedAt: null, elapsedMs: 0 };
+}
+export function initViewState(targets) {
+    return {
+        primarySources: targets.filter((t) => t.operation === 'scan').map(makeTargetState),
+        contextSources: targets.filter((t) => t.operation === 'source-ingest').map(makeTargetState),
+        frame: 0,
+    };
+}
+export async function runContextBuild(project, args, io, deps = {}) {
+    const plan = buildPublicIngestPlan(project, { projectDir: args.projectDir, all: true });
+    const state = initViewState(plan.targets);
+    const isTTY = io.stdout.isTTY === true;
+    const nowFn = deps.now ?? (() => Date.now());
+    const repainter = isTTY ? createRepainter(io) : null;
+    const viewOpts = { styled: true, projectDir: args.projectDir };
+    const paint = (hint) => repainter?.paint(renderContextBuildView(state, { ...viewOpts, showHint: hint }));
+    paint(true);
+    let spinnerInterval = null;
+    if (repainter) {
+        spinnerInterval = setInterval(() => {
+            state.frame++;
+            for (const t of [...state.primarySources, ...state.contextSources]) {
+                if (t.status === 'running' && t.startedAt !== null) {
+                    t.elapsedMs = nowFn() - t.startedAt;
+                }
+            }
+            paint(true);
+        }, 140);
+    }
+    const orderedTargets = [...state.primarySources, ...state.contextSources];
+    const execTarget = deps.executeTarget ?? executePublicIngestTarget;
+    let detached = false;
+    let cleanupKeystroke = null;
+    if (isTTY || deps.setupKeystroke) {
+        const cleanup = () => {
+            if (spinnerInterval)
+                clearInterval(spinnerInterval);
+            cleanupKeystroke?.();
+        };
+        cleanupKeystroke = (deps.setupKeystroke ?? defaultSetupKeystroke)(() => {
+            cleanup();
+            deps.onDetach?.();
+            const bg = spawnBackgroundBuild(args.projectDir);
+            io.stdout.write('\n\nContext build continuing in the background.\n');
+            if (bg)
+                io.stdout.write(`Log: ${bg.logPath}\n`);
+            io.stdout.write(`Status: ktx setup context status --project-dir ${resolve(args.projectDir)}\n`);
+            io.stdout.write(`Resume: ${resumeCommand(args.projectDir)}\n`);
+            process.exit(0);
+        }, () => {
+            cleanup();
+            io.stdout.write('\n\nContext build stopped. Nothing is running in the background.\n');
+            io.stdout.write(`Resume: ${resumeCommand(args.projectDir)}\n`);
+            process.exit(130);
+        });
+    }
+    const runArgs = {
+        command: 'run',
+        projectDir: args.projectDir,
+        all: true,
+        json: false,
+        inputMode: args.inputMode,
+        scanMode: args.scanMode,
+        detectRelationships: args.detectRelationships,
+    };
+    let hasFailure = false;
+    try {
+        for (const targetState of orderedTargets) {
+            if (detached)
+                break;
+            targetState.status = 'running';
+            targetState.startedAt = nowFn();
+            paint(true);
+            const capture = createCaptureIo((message) => {
+                targetState.detailLine = message;
+                paint(true);
+            }, false);
+            const result = await execTarget(targetState.target, runArgs, capture.io, {});
+            targetState.elapsedMs = nowFn() - (targetState.startedAt ?? nowFn());
+            const failed = result.steps.some((s) => s.status === 'failed');
+            targetState.status = failed ? 'failed' : 'done';
+            targetState.detailLine = null;
+            if (!failed) {
+                targetState.summaryText =
+                    targetState.target.operation === 'scan'
+                        ? parseScanSummary(capture.captured())
+                        : parseIngestSummary(capture.captured());
+            }
+            if (failed)
+                hasFailure = true;
+            paint(true);
+        }
+    }
+    finally {
+        if (spinnerInterval)
+            clearInterval(spinnerInterval);
+        cleanupKeystroke?.();
+    }
+    if (detached) {
+        return { exitCode: 0, detached: true };
+    }
+    if (!repainter) {
+        io.stdout.write(renderContextBuildView(state, { styled: false }));
+    }
+    else {
+        paint(false);
+    }
+    return { exitCode: hasFailure ? 1 : 0, detached: false };
+}

package/dist/context-build-view.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/context-build-view.test.js

@@ -0,0 +1,225 @@
+import { buildDefaultKtxProjectConfig } from '@ktx/context/project';
+import { describe, expect, it, vi } from 'vitest';
+import { extractProgressMessage, initViewState, parseIngestSummary, parseScanSummary, renderContextBuildView, runContextBuild, } from './context-build-view.js';
+function makeIo(options = {}) {
+    let stdout = '';
+    let stderr = '';
+    return {
+        io: {
+            stdout: {
+                isTTY: options.isTTY,
+                write: (chunk) => {
+                    stdout += chunk;
+                },
+            },
+            stderr: {
+                write: (chunk) => {
+                    stderr += chunk;
+                },
+            },
+        },
+        stdout: () => stdout,
+        stderr: () => stderr,
+    };
+}
+function projectWithConnections(connections) {
+    return {
+        projectDir: '/tmp/project',
+        config: {
+            ...buildDefaultKtxProjectConfig('warehouse'),
+            connections,
+        },
+    };
+}
+function successResult(connectionId, driver, operation) {
+    return {
+        connectionId,
+        driver,
+        steps: [
+            { operation: 'scan', status: operation === 'scan' ? 'done' : 'skipped' },
+            { operation: 'source-ingest', status: operation === 'source-ingest' ? 'done' : 'skipped' },
+            { operation: 'enrich', status: 'skipped' },
+            { operation: 'memory-update', status: operation === 'source-ingest' ? 'done' : 'skipped' },
+        ],
+    };
+}
+function failedResult(connectionId, driver, operation) {
+    return {
+        connectionId,
+        driver,
+        steps: [
+            { operation: 'scan', status: operation === 'scan' ? 'failed' : 'skipped', detail: `${connectionId} failed at scan.` },
+            { operation: 'source-ingest', status: operation === 'source-ingest' ? 'failed' : 'skipped' },
+            { operation: 'enrich', status: 'skipped' },
+            { operation: 'memory-update', status: 'not-run' },
+        ],
+    };
+}
+describe('extractProgressMessage', () => {
+    it('extracts percentage and message from scan progress', () => {
+        expect(extractProgressMessage('\r[45%] Scanning tables...')).toBe('[45%] Scanning tables...');
+    });
+    it('extracts from permanent progress lines', () => {
+        expect(extractProgressMessage('[100%] Done\n')).toBe('[100%] Done');
+    });
+    it('returns null for non-progress output', () => {
+        expect(extractProgressMessage('KTX scan completed\n')).toBeNull();
+    });
+});
+describe('parseScanSummary', () => {
+    it('extracts table count from scan output', () => {
+        expect(parseScanSummary('Semantic layer comparison found 5 changes across 42 tables')).toBe('42 tables');
+    });
+    it('handles singular form', () => {
+        expect(parseScanSummary('found 1 change across 1 table')).toBe('1 tables');
+    });
+    it('returns null when no match', () => {
+        expect(parseScanSummary('No changes detected')).toBeNull();
+    });
+});
+describe('parseIngestSummary', () => {
+    it('extracts work units and saved memory', () => {
+        expect(parseIngestSummary('Work units: 5\nSaved memory: 3 wiki, 2 SL')).toBe('5 work units · 3 wiki, 2 SL');
+    });
+    it('extracts work units alone when no saved memory', () => {
+        expect(parseIngestSummary('Work units: 5\nStatus: done')).toBe('5 work units');
+    });
+    it('extracts saved memory alone when no work units', () => {
+        expect(parseIngestSummary('Saved memory: 3 wiki, 2 SL')).toBe('3 wiki, 2 SL');
+    });
+    it('returns null when no match', () => {
+        expect(parseIngestSummary('Status: done')).toBeNull();
+    });
+});
+describe('initViewState', () => {
+    it('partitions targets into primary and context sources', () => {
+        const state = initViewState([
+            { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
+            { connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
+        ]);
+        expect(state.primarySources).toHaveLength(1);
+        expect(state.primarySources[0].target.connectionId).toBe('warehouse');
+        expect(state.contextSources).toHaveLength(1);
+        expect(state.contextSources[0].target.connectionId).toBe('dbt-main');
+        expect(state.frame).toBe(0);
+    });
+});
+describe('renderContextBuildView', () => {
+    it('renders all-queued state', () => {
+        const state = initViewState([
+            { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
+            { connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
+        ]);
+        const output = renderContextBuildView(state, { styled: false });
+        expect(output).toContain('Building KTX context');
+        expect(output).toContain('Primary sources:');
+        expect(output).toContain('warehouse');
+        expect(output).toContain('queued');
+        expect(output).toContain('Context sources:');
+        expect(output).toContain('dbt-main');
+    });
+    it('renders completed state with summary', () => {
+        const state = initViewState([
+            { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
+        ]);
+        state.primarySources[0].status = 'done';
+        state.primarySources[0].elapsedMs = 72000;
+        state.primarySources[0].summaryText = '42 tables';
+        const output = renderContextBuildView(state, { styled: false });
+        expect(output).toContain('42 tables');
+        expect(output).toContain('1m12s');
+    });
+    it('renders failed state', () => {
+        const state = initViewState([
+            { connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
+        ]);
+        state.primarySources[0].status = 'failed';
+        const output = renderContextBuildView(state, { styled: false });
+        expect(output).toContain('✗');
+        expect(output).toContain('failed');
+    });
+    it('omits empty groups', () => {
+        const state = initViewState([
+            { connectionId: 'dbt-main', driver: 'dbt', operation: 'source-ingest', adapter: 'dbt', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
+        ]);
+        const output = renderContextBuildView(state, { styled: false });
+        expect(output).not.toContain('Primary sources:');
+        expect(output).toContain('Context sources:');
+    });
+});
+describe('runContextBuild', () => {
+    it('executes scan targets before source-ingest targets', async () => {
+        const io = makeIo();
+        const project = projectWithConnections({
+            dbt_main: { driver: 'dbt' },
+            warehouse: { driver: 'postgres' },
+        });
+        const callOrder = [];
+        const executeTarget = vi.fn(async (target) => {
+            callOrder.push(target.connectionId);
+            return successResult(target.connectionId, target.driver, target.operation);
+        });
+        const result = await runContextBuild(project, { projectDir: '/tmp/project', inputMode: 'disabled' }, io.io, { executeTarget, now: () => 1000 });
+        expect(result).toEqual({ exitCode: 0, detached: false });
+        expect(callOrder).toEqual(['warehouse', 'dbt_main']);
+    });
+    it('returns exit code 1 when any target fails', async () => {
+        const io = makeIo();
+        const project = projectWithConnections({
+            warehouse: { driver: 'postgres' },
+        });
+        const executeTarget = vi.fn(async (target) => failedResult(target.connectionId, target.driver, target.operation));
+        const result = await runContextBuild(project, { projectDir: '/tmp/project', inputMode: 'disabled' }, io.io, { executeTarget, now: () => 1000 });
+        expect(result).toEqual({ exitCode: 1, detached: false });
+    });
+    it('renders final view for non-TTY output', async () => {
+        const io = makeIo();
+        const project = projectWithConnections({
+            warehouse: { driver: 'postgres' },
+            dbt_main: { driver: 'dbt' },
+        });
+        const executeTarget = vi.fn(async (target) => successResult(target.connectionId, target.driver, target.operation));
+        await runContextBuild(project, { projectDir: '/tmp/project', inputMode: 'disabled' }, io.io, { executeTarget, now: () => 1000 });
+        const output = io.stdout();
+        expect(output).toContain('Building KTX context');
+        expect(output).toContain('Primary sources:');
+        expect(output).toContain('warehouse');
+        expect(output).toContain('Context sources:');
+        expect(output).toContain('dbt_main');
+    });
+    it('passes scan mode and detect relationships through to target execution', async () => {
+        const io = makeIo();
+        const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
+        const executeTarget = vi.fn(async (target) => successResult(target.connectionId, target.driver, target.operation));
+        await runContextBuild(project, { projectDir: '/tmp/project', inputMode: 'disabled', scanMode: 'enriched', detectRelationships: true }, io.io, { executeTarget, now: () => 1000 });
+        expect(executeTarget).toHaveBeenCalledWith(expect.objectContaining({ connectionId: 'warehouse', operation: 'scan' }), expect.objectContaining({ scanMode: 'enriched', detectRelationships: true }), expect.anything(), {});
+    });
+    it('exits immediately with paused message when d is pressed', async () => {
+        const mockExit = vi.spyOn(process, 'exit').mockImplementation(() => {
+            throw new Error('process.exit');
+        });
+        const io = makeIo();
+        const project = projectWithConnections({
+            warehouse: { driver: 'postgres' },
+            dbt_main: { driver: 'dbt' },
+        });
+        let triggerDetach = null;
+        const executeTarget = vi.fn(async (target) => {
+            if (target.connectionId === 'warehouse')
+                triggerDetach?.();
+            return successResult(target.connectionId, target.driver, target.operation);
+        });
+        await expect(runContextBuild(project, { projectDir: '/tmp/project', inputMode: 'disabled' }, io.io, {
+            executeTarget,
+            now: () => 1000,
+            setupKeystroke: (onDetach) => {
+                triggerDetach = onDetach;
+                return () => { };
+            },
+        })).rejects.toThrow('process.exit');
+        expect(mockExit).toHaveBeenCalledWith(0);
+        expect(io.stdout()).toContain('Context build continuing in the background.');
+        expect(io.stdout()).toContain('Resume: ktx setup --project-dir /tmp/project');
+        mockExit.mockRestore();
+    });
+});

package/dist/demo-assets.d.ts

@@ -0,0 +1,29 @@
+import type { MemoryFlowReplayInput } from '@ktx/context/ingest/memory-flow';
+interface DemoProjectResult {
+    projectDir: string;
+    configPath: string;
+    databasePath: string;
+    replayPath: string;
+}
+interface EnsureDemoProjectOptions {
+    projectDir: string;
+    force: boolean;
+}
+type DemoProjectStateStatus = 'missing' | 'ready' | 'corrupt';
+interface DemoProjectState {
+    status: DemoProjectStateStatus;
+    projectDir: string;
+    missing: string[];
+}
+export declare const DEMO_CONNECTION_ID = "orbit_demo";
+export declare const DEMO_ADAPTER = "live-database";
+export declare const DEMO_REPLAY_FILE = "replay.memory-flow.v1.json";
+export declare const DEMO_FULL_JOB_ID = "demo-full-ingest";
+export declare function defaultDemoProjectDir(): string;
+export declare function inspectDemoProjectState(projectDir: string): Promise<DemoProjectState>;
+export declare function resetDemoProject(options: EnsureDemoProjectOptions): Promise<DemoProjectResult>;
+export declare function ensureDemoProject(options: EnsureDemoProjectOptions): Promise<DemoProjectResult>;
+export declare function ensureSeededDemoProject(options: EnsureDemoProjectOptions): Promise<DemoProjectResult>;
+export declare function loadPackagedDemoReplay(): Promise<MemoryFlowReplayInput>;
+export declare function loadProjectDemoReplay(projectDir: string): Promise<MemoryFlowReplayInput>;
+export {};