@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/dist/setup-context.js

@@ -0,0 +1,564 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { access, mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { cancel, isCancel, select } from '@clack/prompts';
+import { loadKtxProject, markKtxSetupStepComplete, serializeKtxProjectConfig, } from '@ktx/context/project';
+import { buildPublicIngestPlan } from './public-ingest.js';
+import { runContextBuild } from './context-build-view.js';
+import { withMenuOptionsSpacing } from './prompt-navigation.js';
+import { withSetupInterruptConfirmation } from './setup-interrupt.js';
+const SETUP_CONTEXT_STATE_PATH = ['.ktx', 'setup', 'context-build.json'];
+const LIVE_DATABASE_ADAPTER = 'live-database';
+const SCAN_REPORT_FILE = 'scan-report.json';
+function createPromptAdapter() {
+    return {
+        async select(options) {
+            const value = await withSetupInterruptConfirmation(() => select(withMenuOptionsSpacing(options)));
+            if (isCancel(value)) {
+                cancel('Setup cancelled.');
+                return 'back';
+            }
+            return String(value);
+        },
+        cancel(message) {
+            cancel(message);
+        },
+    };
+}
+function statePath(projectDir) {
+    return join(resolve(projectDir), ...SETUP_CONTEXT_STATE_PATH);
+}
+async function pathExists(path) {
+    try {
+        await access(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export function contextBuildCommands(projectDir, runId) {
+    const resolvedProjectDir = resolve(projectDir);
+    const runIdArg = runId ? ` ${runId}` : '';
+    return {
+        build: `ktx setup context build --project-dir ${resolvedProjectDir}`,
+        watch: `ktx setup context watch${runIdArg} --project-dir ${resolvedProjectDir}`,
+        status: `ktx setup context status${runIdArg} --project-dir ${resolvedProjectDir}`,
+        stop: `ktx setup context stop${runIdArg} --project-dir ${resolvedProjectDir}`,
+        resume: `ktx setup --project-dir ${resolvedProjectDir}`,
+    };
+}
+function notStartedState(projectDir) {
+    return {
+        status: 'not_started',
+        primarySourceConnectionIds: [],
+        contextSourceConnectionIds: [],
+        reportIds: [],
+        artifactPaths: [],
+        retryableFailedTargets: [],
+        commands: contextBuildCommands(projectDir),
+    };
+}
+function normalizeState(projectDir, value) {
+    if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+        return notStartedState(projectDir);
+    }
+    const record = value;
+    const status = record.status ?? 'not_started';
+    const runId = typeof record.runId === 'string' && record.runId.length > 0 ? record.runId : undefined;
+    return {
+        ...(runId ? { runId } : {}),
+        status,
+        ...(typeof record.startedAt === 'string' ? { startedAt: record.startedAt } : {}),
+        ...(typeof record.updatedAt === 'string' ? { updatedAt: record.updatedAt } : {}),
+        ...(typeof record.completedAt === 'string' ? { completedAt: record.completedAt } : {}),
+        primarySourceConnectionIds: Array.isArray(record.primarySourceConnectionIds)
+            ? record.primarySourceConnectionIds.filter((item) => typeof item === 'string')
+            : [],
+        contextSourceConnectionIds: Array.isArray(record.contextSourceConnectionIds)
+            ? record.contextSourceConnectionIds.filter((item) => typeof item === 'string')
+            : [],
+        reportIds: Array.isArray(record.reportIds)
+            ? record.reportIds.filter((item) => typeof item === 'string')
+            : [],
+        artifactPaths: Array.isArray(record.artifactPaths)
+            ? record.artifactPaths.filter((item) => typeof item === 'string')
+            : [],
+        retryableFailedTargets: Array.isArray(record.retryableFailedTargets)
+            ? record.retryableFailedTargets.filter((item) => typeof item === 'string')
+            : [],
+        commands: contextBuildCommands(projectDir, runId),
+        ...(typeof record.failureReason === 'string' ? { failureReason: record.failureReason } : {}),
+    };
+}
+export async function readKtxSetupContextState(projectDir) {
+    const filePath = statePath(projectDir);
+    if (!(await pathExists(filePath))) {
+        return notStartedState(projectDir);
+    }
+    return normalizeState(projectDir, JSON.parse(await readFile(filePath, 'utf-8')));
+}
+export async function writeKtxSetupContextState(projectDir, state) {
+    const resolvedProjectDir = resolve(projectDir);
+    await mkdir(join(resolvedProjectDir, '.ktx', 'setup'), { recursive: true });
+    const normalized = normalizeState(resolvedProjectDir, {
+        ...state,
+        commands: contextBuildCommands(resolvedProjectDir, state.runId),
+    });
+    await writeFile(statePath(resolvedProjectDir), `${JSON.stringify(normalized, null, 2)}\n`, 'utf-8');
+}
+export function setupContextStatusFromState(state, options = { completedStep: false }) {
+    const status = options.completedStep && state.status === 'not_started' ? 'completed' : state.status;
+    const ready = options.completedStep && status === 'completed';
+    return {
+        ready,
+        status,
+        ...(state.runId ? { runId: state.runId } : {}),
+        ...(state.runId ? { watchCommand: state.commands.watch, statusCommand: state.commands.status } : {}),
+        retryCommand: state.commands.build,
+        ...(state.failureReason ? { detail: state.failureReason } : {}),
+    };
+}
+function runIdFactory() {
+    return `setup-context-local-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+}
+function listContextTargets(project) {
+    if (Object.keys(project.config.connections).length === 0) {
+        return { primarySourceConnectionIds: [], contextSourceConnectionIds: [] };
+    }
+    const plan = buildPublicIngestPlan(project, { projectDir: project.projectDir, all: true });
+    return {
+        primarySourceConnectionIds: plan.targets
+            .filter((target) => target.operation === 'scan')
+            .map((target) => target.connectionId),
+        contextSourceConnectionIds: plan.targets
+            .filter((target) => target.operation === 'source-ingest')
+            .map((target) => target.connectionId),
+    };
+}
+function missingCapabilities(project) {
+    const missing = [];
+    const llm = project.config.llm;
+    if (llm.provider.backend === 'none' || !llm.models.default) {
+        missing.push('Models are not ready.');
+    }
+    const embeddings = project.config.ingest.embeddings;
+    if (embeddings.backend === 'none' ||
+        embeddings.backend === 'deterministic' ||
+        !embeddings.model ||
+        embeddings.dimensions <= 0) {
+        missing.push('Embeddings are not ready.');
+    }
+    if (project.config.scan.enrichment.mode === 'none') {
+        missing.push('Scan enrichment is not configured.');
+    }
+    return missing;
+}
+async function hasFileWithExtension(root, extensions, options = {}) {
+    if (!(await pathExists(root))) {
+        return false;
+    }
+    const entries = await readdir(root, { withFileTypes: true });
+    for (const entry of entries) {
+        const entryPath = join(root, entry.name);
+        if (entry.isDirectory()) {
+            if (options.ignoredDirectoryNames?.has(entry.name)) {
+                continue;
+            }
+            if (await hasFileWithExtension(entryPath, extensions, options)) {
+                return true;
+            }
+            continue;
+        }
+        if (extensions.has(entry.name.slice(entry.name.lastIndexOf('.')))) {
+            return true;
+        }
+    }
+    return false;
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function stringValue(value) {
+    return typeof value === 'string' && value.length > 0 ? value : null;
+}
+function stringArrayValue(value) {
+    return Array.isArray(value) ? value.filter((item) => typeof item === 'string') : [];
+}
+async function readJsonFile(path) {
+    try {
+        return JSON.parse(await readFile(path, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+async function readLatestScanReport(projectDir, connectionId) {
+    const scanRoot = join(projectDir, 'raw-sources', connectionId, LIVE_DATABASE_ADAPTER);
+    if (!(await pathExists(scanRoot))) {
+        return null;
+    }
+    const reports = [];
+    for (const entry of await readdir(scanRoot, { withFileTypes: true })) {
+        if (!entry.isDirectory()) {
+            continue;
+        }
+        const report = await readJsonFile(join(scanRoot, entry.name, SCAN_REPORT_FILE));
+        if (!isRecord(report)) {
+            continue;
+        }
+        reports.push({ sortKey: stringValue(report.createdAt) ?? entry.name, report });
+    }
+    reports.sort((left, right) => left.sortKey.localeCompare(right.sortKey));
+    return reports.at(-1)?.report ?? null;
+}
+function scanReportHasCompletedDescriptionEnrichment(report, connectionId) {
+    if (!isRecord(report)) {
+        return false;
+    }
+    if (report.connectionId !== connectionId || report.mode !== 'enriched' || report.dryRun === true) {
+        return false;
+    }
+    if (!isRecord(report.enrichment) || !isRecord(report.enrichmentState) || !isRecord(report.artifactPaths)) {
+        return false;
+    }
+    const completedStages = stringArrayValue(report.enrichmentState.completedStages);
+    return (report.enrichment.tableDescriptions === 'completed' &&
+        report.enrichment.columnDescriptions === 'completed' &&
+        report.enrichment.embeddings === 'completed' &&
+        completedStages.includes('descriptions') &&
+        completedStages.includes('embeddings') &&
+        stringArrayValue(report.artifactPaths.manifestShards).length > 0);
+}
+async function verifyPrimarySourceScans(projectDir, connectionIds) {
+    const details = [];
+    for (const connectionId of connectionIds) {
+        const report = await readLatestScanReport(projectDir, connectionId);
+        if (!scanReportHasCompletedDescriptionEnrichment(report, connectionId)) {
+            details.push(`${connectionId}: enriched database scan with AI descriptions has not completed.`);
+        }
+    }
+    return { ready: details.length === 0, details };
+}
+async function defaultVerifyContextReady(projectDir) {
+    const project = await loadKtxProject({ projectDir });
+    const targets = listContextTargets(project);
+    const primarySourceScans = await verifyPrimarySourceScans(projectDir, targets.primarySourceConnectionIds);
+    const semanticLayerContextReady = await hasFileWithExtension(join(projectDir, 'semantic-layer'), new Set(['.yaml', '.yml']), {
+        ignoredDirectoryNames: new Set(['_schema']),
+    });
+    const wikiReady = await hasFileWithExtension(join(projectDir, 'knowledge'), new Set(['.md']));
+    const contextSourceReady = targets.contextSourceConnectionIds.length === 0 || semanticLayerContextReady || wikiReady;
+    const ready = primarySourceScans.ready && contextSourceReady;
+    const semanticSearchReady = semanticLayerContextReady || primarySourceScans.ready;
+    const details = [];
+    if (!primarySourceScans.ready) {
+        details.push(...primarySourceScans.details);
+    }
+    if (!contextSourceReady) {
+        details.push('No semantic-layer or wiki assets were found after the context build.');
+    }
+    return {
+        ready,
+        agentContextReady: ready,
+        semanticSearchReady,
+        details: ready
+            ? [
+                `Agent context: ${ready ? 'ready' : 'not ready'}`,
+                `Semantic search: ${semanticSearchReady ? 'ready' : 'not ready'}`,
+            ]
+            : details,
+    };
+}
+async function markContextComplete(projectDir) {
+    const project = await loadKtxProject({ projectDir });
+    await writeFile(project.configPath, serializeKtxProjectConfig(markKtxSetupStepComplete(project.config, 'context')), 'utf-8');
+}
+function writeBuildHeader(projectDir, runId, io) {
+    const commands = contextBuildCommands(projectDir, runId);
+    io.stdout.write('\nKTX context build\n');
+    io.stdout.write(`Run: ${runId}\n`);
+    io.stdout.write(`Project: ${resolve(projectDir)}\n\n`);
+    io.stdout.write('Detach: press d to leave this running.\n');
+    io.stdout.write(`Resume: ${commands.watch}\n`);
+    io.stdout.write(`Status: ${commands.status}\n\n`);
+}
+function writeMissingCapabilities(missing, io) {
+    io.stderr.write('KTX cannot build agent-ready context yet.\n\n');
+    io.stderr.write('Missing:\n');
+    for (const item of missing) {
+        io.stderr.write(`  ${item}\n`);
+    }
+    io.stderr.write('\nFix this in setup before building context.\n');
+}
+function writeSkippedContext(projectDir, io) {
+    io.stdout.write('\nKTX is configured, but context has not been built yet.\n\n');
+    io.stdout.write('Agents were not connected because KTX has not prepared searchable context for them.\n\n');
+    io.stdout.write(`Resume setup:\n  ktx setup --project-dir ${resolve(projectDir)}\n\n`);
+    io.stdout.write(`Build context directly:\n  ktx setup context build --project-dir ${resolve(projectDir)}\n\n`);
+    io.stdout.write(`Check status:\n  ktx status --project-dir ${resolve(projectDir)}\n`);
+}
+function writeSuccess(readiness, targets, io) {
+    io.stdout.write('\nKTX context is ready for agents.\n\n');
+    io.stdout.write('Primary sources:\n');
+    if (targets.primarySourceConnectionIds.length === 0) {
+        io.stdout.write('  none\n');
+    }
+    else {
+        for (const connectionId of targets.primarySourceConnectionIds) {
+            io.stdout.write(`  ${connectionId}: enriched scan complete\n`);
+        }
+    }
+    io.stdout.write('\nContext sources:\n');
+    if (targets.contextSourceConnectionIds.length === 0) {
+        io.stdout.write('  none\n');
+    }
+    else {
+        for (const connectionId of targets.contextSourceConnectionIds) {
+            io.stdout.write(`  ${connectionId}: memory update complete\n`);
+        }
+    }
+    io.stdout.write('\nVerification:\n');
+    io.stdout.write(`  Agent context: ${readiness.agentContextReady ? 'ready' : 'not ready'}\n`);
+    io.stdout.write(`  Semantic search: ${readiness.semanticSearchReady ? 'ready' : 'not ready'}\n`);
+}
+function writeExistingContextSuccess(readiness, io) {
+    io.stdout.write('\nKTX context is ready for agents.\n\n');
+    io.stdout.write('Existing context artifacts were found from setup ingest.\n\n');
+    io.stdout.write('Verification:\n');
+    io.stdout.write(`  Agent context: ${readiness.agentContextReady ? 'ready' : 'not ready'}\n`);
+    io.stdout.write(`  Semantic search: ${readiness.semanticSearchReady ? 'ready' : 'not ready'}\n`);
+}
+async function promptForBuild(prompts) {
+    return (await prompts.select({
+        message: 'Build KTX context for agents?\n\n' +
+            'KTX is fully configured and ready to build context. This may take a few minutes to a few hours.',
+        options: [
+            { value: 'build', label: 'Build context now (recommended)' },
+            { value: 'skip', label: 'Leave context unbuilt and exit setup' },
+            { value: 'back', label: 'Back' },
+        ],
+    }));
+}
+async function runBuild(args, io, deps, project, targets) {
+    const now = deps.now ?? (() => new Date());
+    const runId = deps.runIdFactory?.() ?? runIdFactory();
+    const startedAt = now().toISOString();
+    const runningState = {
+        runId,
+        status: 'running',
+        startedAt,
+        updatedAt: startedAt,
+        primarySourceConnectionIds: targets.primarySourceConnectionIds,
+        contextSourceConnectionIds: targets.contextSourceConnectionIds,
+        reportIds: [],
+        artifactPaths: [],
+        retryableFailedTargets: [],
+        commands: contextBuildCommands(args.projectDir, runId),
+    };
+    await writeKtxSetupContextState(args.projectDir, runningState);
+    const contextBuild = deps.runContextBuild ?? runContextBuild;
+    const buildResult = await contextBuild(project, {
+        projectDir: args.projectDir,
+        inputMode: args.inputMode,
+        scanMode: 'enriched',
+        detectRelationships: true,
+    }, io, {
+        onDetach: () => {
+            const resolvedDir = resolve(args.projectDir);
+            mkdirSync(join(resolvedDir, '.ktx', 'setup'), { recursive: true });
+            const detachedState = normalizeState(resolvedDir, {
+                ...runningState,
+                status: 'detached',
+                updatedAt: new Date().toISOString(),
+            });
+            writeFileSync(statePath(resolvedDir), `${JSON.stringify(detachedState, null, 2)}\n`);
+        },
+    });
+    if (buildResult.detached) {
+        const updatedAt = now().toISOString();
+        await writeKtxSetupContextState(args.projectDir, { ...runningState, status: 'detached', updatedAt });
+        return { status: 'detached', projectDir: args.projectDir, runId };
+    }
+    if (buildResult.exitCode !== 0) {
+        const updatedAt = now().toISOString();
+        await writeKtxSetupContextState(args.projectDir, {
+            ...runningState,
+            status: 'failed',
+            updatedAt,
+            retryableFailedTargets: [...targets.primarySourceConnectionIds, ...targets.contextSourceConnectionIds],
+            failureReason: 'Context build failed.',
+        });
+        return { status: 'failed', projectDir: args.projectDir };
+    }
+    const readiness = await (deps.verifyContextReady ?? defaultVerifyContextReady)(args.projectDir);
+    if (!readiness.ready) {
+        const updatedAt = now().toISOString();
+        await writeKtxSetupContextState(args.projectDir, {
+            ...runningState,
+            status: 'failed',
+            updatedAt,
+            retryableFailedTargets: readiness.failedTargets ?? [],
+            failureReason: readiness.details.join(' '),
+        });
+        io.stderr.write('KTX context build did not pass agent-readiness verification.\n');
+        for (const detail of readiness.details) {
+            io.stderr.write(`  ${detail}\n`);
+        }
+        return { status: 'failed', projectDir: args.projectDir };
+    }
+    await markContextComplete(project.projectDir);
+    const completedAt = now().toISOString();
+    await writeKtxSetupContextState(args.projectDir, {
+        ...runningState,
+        status: 'completed',
+        updatedAt: completedAt,
+        completedAt,
+        retryableFailedTargets: [],
+    });
+    writeSuccess(readiness, targets, io);
+    return { status: 'ready', projectDir: args.projectDir, runId };
+}
+async function completeExistingContext(args, io, deps, targets) {
+    const readiness = await (deps.verifyContextReady ?? defaultVerifyContextReady)(args.projectDir);
+    if (!readiness.ready) {
+        return null;
+    }
+    const now = deps.now ?? (() => new Date());
+    const completedAt = now().toISOString();
+    const runId = deps.runIdFactory?.() ?? runIdFactory();
+    await markContextComplete(args.projectDir);
+    await writeKtxSetupContextState(args.projectDir, {
+        runId,
+        status: 'completed',
+        startedAt: completedAt,
+        updatedAt: completedAt,
+        completedAt,
+        primarySourceConnectionIds: targets.primarySourceConnectionIds,
+        contextSourceConnectionIds: targets.contextSourceConnectionIds,
+        reportIds: [],
+        artifactPaths: [],
+        retryableFailedTargets: [],
+        commands: contextBuildCommands(args.projectDir, runId),
+    });
+    writeExistingContextSuccess(readiness, io);
+    return { status: 'ready', projectDir: args.projectDir, runId };
+}
+export async function runKtxSetupContextStep(args, io, deps = {}) {
+    try {
+        const project = await loadKtxProject({ projectDir: args.projectDir });
+        const existingState = await readKtxSetupContextState(args.projectDir);
+        if (project.config.setup?.completed_steps.includes('context') === true && existingState.status === 'completed') {
+            return { status: 'ready', projectDir: args.projectDir, runId: existingState.runId ?? 'setup-context-completed' };
+        }
+        if ((existingState.status === 'running' || existingState.status === 'detached') &&
+            args.inputMode !== 'disabled') {
+            const prompts = deps.prompts ?? createPromptAdapter();
+            const choice = await prompts.select({
+                message: 'A context build is running in the background.\n\n' +
+                    'You can wait for it to finish, check its status, or start a fresh build.',
+                options: [
+                    { value: 'status', label: 'Check status' },
+                    { value: 'rebuild', label: 'Start a fresh context build' },
+                    { value: 'back', label: 'Back' },
+                ],
+            });
+            if (choice === 'status') {
+                const commands = contextBuildCommands(args.projectDir, existingState.runId);
+                io.stdout.write(`\nRun: ${commands.status}\n`);
+                io.stdout.write(`Log: ${join(resolve(args.projectDir), '.ktx', 'setup', 'context-build.log')}\n`);
+                return { status: 'detached', projectDir: args.projectDir, runId: existingState.runId ?? '' };
+            }
+            if (choice === 'back') {
+                return { status: 'back', projectDir: args.projectDir };
+            }
+        }
+        const targets = listContextTargets(project);
+        if (targets.primarySourceConnectionIds.length === 0 && targets.contextSourceConnectionIds.length === 0) {
+            if (args.allowEmpty === true) {
+                return { status: 'skipped', projectDir: args.projectDir };
+            }
+            io.stderr.write('No primary or context sources are configured for a KTX context build.\n');
+            return { status: 'failed', projectDir: args.projectDir };
+        }
+        const missing = missingCapabilities(project);
+        if (missing.length > 0) {
+            writeMissingCapabilities(missing, io);
+            return { status: 'missing-input', projectDir: args.projectDir };
+        }
+        if (args.forcePrompt !== true && args.prompt !== false && deps.verifyContextReady === undefined) {
+            const existingContextResult = await completeExistingContext(args, io, deps, targets);
+            if (existingContextResult) {
+                return existingContextResult;
+            }
+        }
+        if (args.inputMode !== 'disabled' && args.prompt !== false) {
+            const choice = await promptForBuild(deps.prompts ?? createPromptAdapter());
+            if (choice === 'back') {
+                return { status: 'back', projectDir: args.projectDir };
+            }
+            if (choice === 'skip') {
+                writeSkippedContext(args.projectDir, io);
+                return { status: 'skipped', projectDir: args.projectDir };
+            }
+        }
+        return await runBuild(args, io, deps, project, targets);
+    }
+    catch (error) {
+        io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+        return { status: 'failed', projectDir: args.projectDir };
+    }
+}
+function stateMatchesRunId(state, runId) {
+    return !runId || state.runId === runId;
+}
+function statusPayload(state) {
+    return setupContextStatusFromState(state, { completedStep: state.status === 'completed' });
+}
+function writeContextStatus(state, io) {
+    io.stdout.write(`KTX context built: ${state.status === 'completed' ? 'yes' : state.status.replaceAll('_', ' ')}\n`);
+    if (state.runId) {
+        io.stdout.write(`Run: ${state.runId}\n`);
+        io.stdout.write(`Watch: ${state.commands.watch}\n`);
+        io.stdout.write(`Status: ${state.commands.status}\n`);
+    }
+    if (state.failureReason) {
+        io.stdout.write(`Detail: ${state.failureReason}\n`);
+    }
+}
+export async function runKtxSetupContextCommand(args, io, deps = {}) {
+    if (args.command === 'build') {
+        const result = await runKtxSetupContextStep({ projectDir: args.projectDir, inputMode: args.inputMode, prompt: false }, io, deps);
+        return result.status === 'ready' || result.status === 'skipped' ? 0 : 1;
+    }
+    const state = await readKtxSetupContextState(args.projectDir);
+    if (!stateMatchesRunId(state, args.runId)) {
+        io.stderr.write(`KTX setup context run "${args.runId}" was not found.\n`);
+        return 1;
+    }
+    if (args.command === 'status') {
+        if (args.json) {
+            io.stdout.write(`${JSON.stringify(statusPayload(state), null, 2)}\n`);
+        }
+        else {
+            writeContextStatus(state, io);
+        }
+        return 0;
+    }
+    if (args.command === 'watch') {
+        io.stdout.write('KTX context build\n');
+        writeContextStatus(state, io);
+        return 0;
+    }
+    const updatedAt = new Date().toISOString();
+    const nextState = {
+        ...state,
+        status: state.status === 'completed' ? 'completed' : 'paused',
+        updatedAt,
+    };
+    await writeKtxSetupContextState(args.projectDir, nextState);
+    io.stdout.write(state.status === 'completed'
+        ? 'KTX context build already completed.\n'
+        : 'KTX context build pause requested. Resume with setup when ready.\n');
+    return 0;
+}

package/dist/setup-context.test.d.ts

@@ -0,0 +1 @@
+export {};