@kaelio/ktx 0.0.0 → 0.1.0-rc.0

package/dist/completion.js

@@ -0,0 +1,272 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+const KTX_COMPLETION_BLOCK_START = '# >>> ktx completion >>>';
+const KTX_COMPLETION_BLOCK_END = '# <<< ktx completion <<<';
+const KTX_COMPLETION_BLOCK_PATTERN = new RegExp(`\\n?${escapeRegExp(KTX_COMPLETION_BLOCK_START)}[\\s\\S]*?${escapeRegExp(KTX_COMPLETION_BLOCK_END)}\\n?`, 'g');
+export function zshCompletionScript() {
+    const zshWords = '$' + '{words[@]}';
+    const zshCompletionCapture = [
+        '$',
+        `{(@f)$("${'$'}{ktx_completion_command[@]}" dev __complete --shell zsh --position "$CURRENT" -- "${zshWords}" 2>/dev/null)}`,
+    ].join('');
+    const zshCompletionsCount = '$' + '{#completions[@]}';
+    const zshCompletionCommand = '$' + '(eval "print -r -- $' + '{KTX_COMPLETION_COMMAND:-ktx}")';
+    return [
+        '#compdef ktx',
+        '',
+        '_ktx() {',
+        '  local -a completions',
+        '  local -a ktx_completion_command',
+        `  ktx_completion_command=("\${(@z)${zshCompletionCommand}}")`,
+        `  completions=("${zshCompletionCapture}")`,
+        `  if (( ${zshCompletionsCount} )); then`,
+        "    _describe 'ktx completions' completions",
+        '  else',
+        '    _files',
+        '  fi',
+        '}',
+        '',
+        'compdef _ktx ktx',
+        '',
+    ].join('\n');
+}
+export async function installZshCompletion() {
+    const homeDir = process.env.HOME || homedir();
+    const zshConfigDir = process.env.ZDOTDIR || homeDir;
+    const completionDir = join(homeDir, '.zfunc');
+    const completionPath = join(completionDir, '_ktx');
+    const zshrcPath = join(zshConfigDir, '.zshrc');
+    await mkdir(completionDir, { recursive: true });
+    await mkdir(dirname(zshrcPath), { recursive: true });
+    await writeFile(completionPath, zshCompletionScript(), 'utf-8');
+    const existingZshrc = await readOptionalTextFile(zshrcPath);
+    const nextZshrc = updateZshrcCompletionBlock(existingZshrc);
+    await writeFile(zshrcPath, nextZshrc, 'utf-8');
+    return { completionPath, zshrcPath };
+}
+export function completeCommanderInput(program, request) {
+    const words = completionWordsForPosition(request.words, request.position);
+    const tokens = stripProgramName(program, words);
+    const current = tokens.at(-1) ?? '';
+    const previous = tokens.slice(0, -1);
+    const state = resolveCommandState(program, previous);
+    return candidatesForState(state, current).map(formatZshCandidate);
+}
+function completionWordsForPosition(words, position) {
+    if (!Number.isInteger(position) || position < 1) {
+        return words;
+    }
+    return words.slice(0, position);
+}
+function stripProgramName(program, words) {
+    const [first, ...rest] = words;
+    if (!first) {
+        return [];
+    }
+    return first === program.name() || first.endsWith(`/${program.name()}`) ? rest : words;
+}
+function resolveCommandState(program, tokens) {
+    let command = program;
+    let positionalIndex = 0;
+    let pendingOption;
+    let positionalOnly = false;
+    for (let index = 0; index < tokens.length; index += 1) {
+        const token = tokens[index];
+        if (pendingOption) {
+            pendingOption = undefined;
+            continue;
+        }
+        if (token === '--') {
+            positionalOnly = true;
+            continue;
+        }
+        if (!positionalOnly && token.startsWith('-')) {
+            const option = findOption(command, optionNameFromToken(token));
+            if (option && !token.includes('=') && optionTakesValue(option)) {
+                if (index === tokens.length - 1) {
+                    pendingOption = option;
+                }
+                else if (option.required || !tokens[index + 1]?.startsWith('-')) {
+                    index += 1;
+                }
+            }
+            continue;
+        }
+        const child = findVisibleSubcommand(command, token);
+        if (child) {
+            command = child;
+            positionalIndex = 0;
+            continue;
+        }
+        positionalIndex += 1;
+    }
+    return { command, pendingOption, positionalIndex };
+}
+function candidatesForState(state, current) {
+    const optionValue = splitOptionValueToken(current);
+    if (optionValue) {
+        const option = findOption(state.command, optionValue.optionName);
+        return choiceCandidates(option?.argChoices, optionValue.valuePrefix, optionValue.optionPrefix);
+    }
+    if (state.pendingOption) {
+        return choiceCandidates(state.pendingOption.argChoices, current);
+    }
+    if (current.startsWith('-')) {
+        return visibleOptions(state.command)
+            .map(optionCandidate)
+            .filter((candidate) => candidate.value.startsWith(current));
+    }
+    const commandCandidates = visibleSubcommands(state.command)
+        .map(commandCandidate)
+        .filter((candidate) => candidate.value.startsWith(current));
+    const argument = state.command.registeredArguments[state.positionalIndex];
+    return [...commandCandidates, ...choiceCandidates(argument?.argChoices, current)];
+}
+function visibleSubcommands(command) {
+    return command.commands.filter((subcommand) => subcommand._hidden !== true);
+}
+function findVisibleSubcommand(command, name) {
+    return visibleSubcommands(command).find((subcommand) => subcommand.name() === name || subcommand.aliases().includes(name));
+}
+function visibleOptions(command) {
+    const options = [];
+    const seen = new Set();
+    for (const current of commandChain(command)) {
+        for (const option of current.options) {
+            if (option.hidden) {
+                continue;
+            }
+            const key = option.long ?? option.short ?? option.flags;
+            if (seen.has(key)) {
+                continue;
+            }
+            seen.add(key);
+            options.push(option);
+        }
+    }
+    return options;
+}
+function commandChain(command) {
+    const chain = [];
+    let current = command;
+    while (current) {
+        chain.unshift(current);
+        current = current.parent;
+    }
+    return chain;
+}
+function findOption(command, name) {
+    return visibleOptions(command).find((option) => option.long === name || option.short === name);
+}
+function optionTakesValue(option) {
+    return option.required || option.optional;
+}
+function optionNameFromToken(token) {
+    return token.split('=', 1)[0] ?? token;
+}
+function splitOptionValueToken(token) {
+    const separatorIndex = token.indexOf('=');
+    if (!token.startsWith('-') || separatorIndex < 0) {
+        return null;
+    }
+    return {
+        optionName: token.slice(0, separatorIndex),
+        optionPrefix: token.slice(0, separatorIndex + 1),
+        valuePrefix: token.slice(separatorIndex + 1),
+    };
+}
+function commandCandidate(command) {
+    return {
+        value: command.name(),
+        description: command.summary() || command.description(),
+    };
+}
+function optionCandidate(option) {
+    return {
+        value: option.long ?? option.short ?? option.flags,
+        description: option.description,
+    };
+}
+function choiceCandidates(choices, prefix, completionPrefix = '') {
+    return (choices ?? [])
+        .filter((choice) => choice.startsWith(prefix))
+        .map((choice) => ({ value: `${completionPrefix}${choice}` }));
+}
+function formatZshCandidate(candidate) {
+    if (!candidate.description) {
+        return escapeZshCompletion(candidate.value);
+    }
+    return `${escapeZshCompletion(candidate.value)}:${escapeZshDescription(candidate.description)}`;
+}
+function escapeZshCompletion(value) {
+    return value.replace(/\\/g, '\\\\').replace(/:/g, '\\:');
+}
+function escapeZshDescription(value) {
+    return value.replace(/\s+/g, ' ').replace(/\\/g, '\\\\').replace(/:/g, '\\:').trim();
+}
+async function readOptionalTextFile(path) {
+    try {
+        return await readFile(path, 'utf-8');
+    }
+    catch (error) {
+        if (isNodeError(error) && error.code === 'ENOENT') {
+            return '';
+        }
+        throw error;
+    }
+}
+function updateZshrcCompletionBlock(contents) {
+    const withoutManagedBlock = contents.replace(KTX_COMPLETION_BLOCK_PATTERN, normalizeTrailingNewline);
+    const hasCompinit = /^.*\bcompinit\b.*$/m.test(withoutManagedBlock);
+    const block = zshrcCompletionBlock({ includeCompinit: !hasCompinit });
+    if (!hasCompinit) {
+        return appendBlock(withoutManagedBlock, block);
+    }
+    const compinitMatch = /^.*\bcompinit\b.*$/m.exec(withoutManagedBlock);
+    if (!compinitMatch || compinitMatch.index === undefined) {
+        return appendBlock(withoutManagedBlock, block);
+    }
+    return [
+        withoutManagedBlock.slice(0, compinitMatch.index),
+        block,
+        '\n',
+        withoutManagedBlock.slice(compinitMatch.index),
+    ].join('');
+}
+function zshrcCompletionBlock(options) {
+    return [
+        KTX_COMPLETION_BLOCK_START,
+        '_ktx_completion_command() {',
+        '  local dir="$PWD"',
+        '  while [[ "$dir" != "/" ]]; do',
+        `    if [[ -f "$dir/package.json" ]] && command grep -q '"name": "ktx-workspace"' "$dir/package.json" 2>/dev/null; then`,
+        '      print -r -- "node $dir/scripts/run-ktx.mjs --"',
+        '      return',
+        '    fi',
+        '    dir="' + '$' + '{dir:h}"',
+        '  done',
+        '  print -r -- "ktx"',
+        '}',
+        "export KTX_COMPLETION_COMMAND='$(_ktx_completion_command)'",
+        'setopt complete_aliases',
+        'fpath=("$HOME/.zfunc" $fpath)',
+        ...(options.includeCompinit ? ['autoload -Uz compinit', 'compinit'] : []),
+        KTX_COMPLETION_BLOCK_END,
+    ].join('\n');
+}
+function appendBlock(contents, block) {
+    if (!contents.trim()) {
+        return `${block}\n`;
+    }
+    return `${contents.replace(/\s*$/, '\n\n')}${block}\n`;
+}
+function normalizeTrailingNewline(match) {
+    return match.startsWith('\n') || match.endsWith('\n') ? '\n' : '';
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function isNodeError(error) {
+    return error instanceof Error && 'code' in error;
+}

package/dist/connection.d.ts

@@ -0,0 +1,62 @@
+import type { KtxCliIo } from './index.js';
+import { createKtxCliScanConnector } from './local-scan-connectors.js';
+interface KtxNotionConnectionCliConfig {
+    authTokenRef: string;
+    crawlMode: 'all_accessible' | 'selected_roots';
+    rootPageIds: string[];
+    rootDatabaseIds: string[];
+    rootDataSourceIds: string[];
+    maxPagesPerRun?: number;
+    maxKnowledgeCreatesPerRun?: number;
+    maxKnowledgeUpdatesPerRun?: number;
+}
+type KtxConnectionInputMode = 'disabled';
+export type KtxConnectionArgs = {
+    command: 'list';
+    projectDir: string;
+} | {
+    command: 'add';
+    projectDir: string;
+    driver: string;
+    connectionId: string;
+    url?: string;
+    schemas: string[];
+    readonly: boolean;
+    force: boolean;
+    allowLiteralCredentials: boolean;
+    notion?: KtxNotionConnectionCliConfig;
+} | {
+    command: 'test';
+    projectDir: string;
+    connectionId: string;
+} | {
+    command: 'remove';
+    projectDir: string;
+    connectionId: string;
+    force: boolean;
+    inputMode?: KtxConnectionInputMode;
+} | {
+    command: 'map';
+    projectDir: string;
+    sourceConnectionId: string;
+    json: boolean;
+};
+interface KtxConnectionPromptAdapter {
+    confirm(options: {
+        message: string;
+        initialValue?: boolean;
+    }): Promise<boolean>;
+    cancel(message: string): void;
+}
+interface KtxConnectionIo extends KtxCliIo {
+    stdin?: {
+        isTTY?: boolean;
+    };
+}
+interface KtxConnectionDeps {
+    createScanConnector?: typeof createKtxCliScanConnector;
+    runMapping?: (argv: string[], io: KtxCliIo) => Promise<number>;
+    prompts?: KtxConnectionPromptAdapter;
+}
+export declare function runKtxConnection(args: KtxConnectionArgs, io?: KtxConnectionIo, deps?: KtxConnectionDeps): Promise<number>;
+export {};

package/dist/connection.js

@@ -0,0 +1,261 @@
+import { cancel, confirm, isCancel } from '@clack/prompts';
+import { loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
+import { createKtxCliScanConnector } from './local-scan-connectors.js';
+import { profileMark } from './startup-profile.js';
+profileMark('module:connection');
+function assertSafeConnectionId(connectionId) {
+    if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
+        throw new Error(`Unsafe connection id: ${connectionId}`);
+    }
+}
+function isCredentialReference(value) {
+    return value.startsWith('env:') || value.startsWith('file:');
+}
+function literalCredentialWarning(connectionId) {
+    return `Warning: writing a literal credential URL to ktx.yaml for connection "${connectionId}". Prefer env:NAME or file:/path references.`;
+}
+function createClackConnectionPromptAdapter() {
+    return {
+        async confirm(options) {
+            const value = await confirm(options);
+            return isCancel(value) ? false : value;
+        },
+        cancel(message) {
+            cancel(message);
+        },
+    };
+}
+function isInteractiveConnectionIo(args, io) {
+    return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
+}
+async function cleanupConnector(connector) {
+    if (connector?.cleanup) {
+        await connector.cleanup();
+    }
+}
+async function testNativeConnection(project, connectionId, createScanConnector) {
+    let connector = null;
+    try {
+        connector = await createScanConnector(project, connectionId);
+        const snapshot = await connector.introspect({
+            connectionId,
+            driver: connector.driver,
+            mode: 'structural',
+            dryRun: true,
+            detectRelationships: false,
+        }, { runId: `connection-test-${connectionId}` });
+        return {
+            driver: connector.driver,
+            tableCount: snapshot.tables.length,
+        };
+    }
+    finally {
+        await cleanupConnector(connector);
+    }
+}
+function createBufferedIo() {
+    let stdout = '';
+    let stderr = '';
+    return {
+        stdout: {
+            write(chunk) {
+                stdout += chunk;
+            },
+        },
+        stderr: {
+            write(chunk) {
+                stderr += chunk;
+            },
+        },
+        stdoutText() {
+            return stdout;
+        },
+        stderrText() {
+            return stderr;
+        },
+    };
+}
+function splitOutputLines(output) {
+    return output
+        .split('\n')
+        .map((line) => line.trim())
+        .filter(Boolean);
+}
+async function runLowLevelMapping(args, argv, io, deps) {
+    if (deps.runMapping) {
+        return await deps.runMapping(argv, io);
+    }
+    const { runKtxConnectionMapping } = await import('./commands/connection-mapping.js');
+    return await runKtxConnectionMapping(args, io);
+}
+function parseMappingListJson(output) {
+    const trimmed = output.trim();
+    if (!trimmed) {
+        return [];
+    }
+    const parsed = JSON.parse(trimmed);
+    return Array.isArray(parsed) ? parsed : [];
+}
+async function runPublicConnectionMap(args, io, deps) {
+    const refreshIo = createBufferedIo();
+    const refreshArgs = {
+        command: 'refresh',
+        projectDir: args.projectDir,
+        connectionId: args.sourceConnectionId,
+        autoAccept: true,
+    };
+    const refreshCode = await runLowLevelMapping(refreshArgs, ['refresh', args.sourceConnectionId, '--auto-accept', '--project-dir', args.projectDir], refreshIo, deps);
+    if (refreshCode !== 0) {
+        io.stderr.write(refreshIo.stderrText() ||
+            refreshIo.stdoutText() ||
+            `Failed to refresh mapping metadata for ${args.sourceConnectionId}\n`);
+        return refreshCode;
+    }
+    const validationIo = createBufferedIo();
+    const validationArgs = {
+        command: 'validate',
+        projectDir: args.projectDir,
+        connectionId: args.sourceConnectionId,
+    };
+    const validationCode = await runLowLevelMapping(validationArgs, ['validate', args.sourceConnectionId, '--project-dir', args.projectDir], validationIo, deps);
+    if (validationCode !== 0) {
+        io.stderr.write(validationIo.stderrText() || validationIo.stdoutText() || `Mapping validation failed for ${args.sourceConnectionId}\n`);
+        return validationCode;
+    }
+    const listIo = createBufferedIo();
+    const listArgv = ['list', args.sourceConnectionId, '--project-dir', args.projectDir];
+    const listArgs = {
+        command: 'list',
+        projectDir: args.projectDir,
+        connectionId: args.sourceConnectionId,
+        json: args.json,
+    };
+    const listCode = await runLowLevelMapping(listArgs, args.json ? [...listArgv, '--json'] : listArgv, listIo, deps);
+    if (listCode !== 0) {
+        io.stderr.write(listIo.stderrText() || listIo.stdoutText() || `Failed to list mappings for ${args.sourceConnectionId}\n`);
+        return listCode;
+    }
+    if (args.json) {
+        io.stdout.write(`${JSON.stringify({
+            connectionId: args.sourceConnectionId,
+            refresh: { ok: true, output: splitOutputLines(refreshIo.stdoutText()) },
+            validation: { ok: true, output: splitOutputLines(validationIo.stdoutText()) },
+            mappings: parseMappingListJson(listIo.stdoutText()),
+        }, null, 2)}\n`);
+        return 0;
+    }
+    io.stdout.write(`Mapping: ${args.sourceConnectionId}\n`);
+    io.stdout.write(refreshIo.stdoutText());
+    io.stdout.write(validationIo.stdoutText());
+    io.stdout.write('\nMappings:\n');
+    io.stdout.write(listIo.stdoutText().trim() ? listIo.stdoutText() : 'No mappings found.\n');
+    io.stdout.write('\nNext:\n');
+    io.stdout.write(`  ktx ingest ${args.sourceConnectionId}\n`);
+    io.stdout.write(`  ktx dev mapping list ${args.sourceConnectionId}\n`);
+    return 0;
+}
+export async function runKtxConnection(args, io = process, deps = {}) {
+    try {
+        if (args.command === 'map') {
+            return await runPublicConnectionMap(args, io, deps);
+        }
+        const project = await loadKtxProject({ projectDir: args.projectDir });
+        if (args.command === 'list') {
+            const entries = Object.entries(project.config.connections).sort(([a], [b]) => a.localeCompare(b));
+            if (entries.length === 0) {
+                io.stdout.write('No connections configured. Run `ktx connection add <id> --driver <driver>` to add one.\n');
+                return 0;
+            }
+            const idWidth = Math.max('ID'.length, ...entries.map(([id]) => id.length));
+            const driverWidth = Math.max('DRIVER'.length, ...entries.map(([, c]) => (c.driver ?? 'unknown').length));
+            io.stdout.write(`${'ID'.padEnd(idWidth)}  ${'DRIVER'.padEnd(driverWidth)}\n`);
+            for (const [id, connection] of entries) {
+                io.stdout.write(`${id.padEnd(idWidth)}  ${(connection.driver ?? 'unknown').padEnd(driverWidth)}\n`);
+            }
+            return 0;
+        }
+        if (args.command === 'add') {
+            assertSafeConnectionId(args.connectionId);
+            const hasLiteralCredentialUrl = !!args.url && !isCredentialReference(args.url);
+            if (hasLiteralCredentialUrl && !args.allowLiteralCredentials) {
+                throw new Error('Literal credential URLs require --allow-literal-credentials');
+            }
+            if (hasLiteralCredentialUrl) {
+                io.stderr.write(`${literalCredentialWarning(args.connectionId)}\n`);
+            }
+            if (project.config.connections[args.connectionId] && !args.force) {
+                throw new Error(`Connection "${args.connectionId}" already exists; pass --force to replace it`);
+            }
+            const connectionConfig = args.driver === 'notion' && args.notion
+                ? {
+                    driver: 'notion',
+                    auth_token_ref: args.notion.authTokenRef,
+                    crawl_mode: args.notion.crawlMode,
+                    root_page_ids: args.notion.rootPageIds,
+                    root_database_ids: args.notion.rootDatabaseIds,
+                    root_data_source_ids: args.notion.rootDataSourceIds,
+                    ...(args.notion.maxPagesPerRun !== undefined ? { max_pages_per_run: args.notion.maxPagesPerRun } : {}),
+                    ...(args.notion.maxKnowledgeCreatesPerRun !== undefined
+                        ? { max_knowledge_creates_per_run: args.notion.maxKnowledgeCreatesPerRun }
+                        : {}),
+                    ...(args.notion.maxKnowledgeUpdatesPerRun !== undefined
+                        ? { max_knowledge_updates_per_run: args.notion.maxKnowledgeUpdatesPerRun }
+                        : {}),
+                }
+                : {
+                    driver: args.driver,
+                    ...(args.url ? { url: args.url } : {}),
+                    ...(args.schemas.length > 0 ? { schemas: args.schemas } : {}),
+                    readonly: args.readonly,
+                };
+            const nextConfig = {
+                ...project.config,
+                connections: {
+                    ...project.config.connections,
+                    [args.connectionId]: connectionConfig,
+                },
+            };
+            await project.fileStore.writeFile('ktx.yaml', serializeKtxProjectConfig(nextConfig), 'ktx', 'ktx@example.com', `Update KTX connection: ${args.connectionId}`);
+            io.stdout.write(`Connection: ${args.connectionId}\n`);
+            io.stdout.write(`Driver: ${args.driver}\n`);
+            return 0;
+        }
+        if (args.command === 'remove') {
+            if (!project.config.connections[args.connectionId]) {
+                throw new Error(`Connection "${args.connectionId}" is not configured in ktx.yaml`);
+            }
+            if (!args.force) {
+                if (!isInteractiveConnectionIo(args, io)) {
+                    throw new Error(`connection remove ${args.connectionId} requires --force when input is disabled or not interactive`);
+                }
+                const prompts = deps.prompts ?? createClackConnectionPromptAdapter();
+                const confirmed = await prompts.confirm({
+                    message: `Remove connection "${args.connectionId}" from ktx.yaml? Ingested artifacts will remain in .ktx/.`,
+                    initialValue: false,
+                });
+                if (!confirmed) {
+                    prompts.cancel('Connection removal cancelled.');
+                    return 1;
+                }
+            }
+            const { [args.connectionId]: _removedConnection, ...connections } = project.config.connections;
+            const nextConfig = {
+                ...project.config,
+                connections,
+            };
+            await project.fileStore.writeFile('ktx.yaml', serializeKtxProjectConfig(nextConfig), 'ktx', 'ktx@example.com', `Remove KTX connection: ${args.connectionId}`);
+            io.stdout.write('Connection removed from ktx.yaml.\n');
+            io.stdout.write('Ingested artifacts from this connection remain in .ktx/. Run ktx dev artifacts to inspect.\n');
+            return 0;
+        }
+        const result = await testNativeConnection(project, args.connectionId, deps.createScanConnector ?? createKtxCliScanConnector);
+        io.stdout.write(`Connection test passed: ${args.connectionId}\n`);
+        io.stdout.write(`Driver: ${result.driver}\n`);
+        io.stdout.write(`Tables: ${result.tableCount}\n`);
+        return 0;
+    }
+    catch (error) {
+        io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+        return 1;
+    }
+}

package/dist/connection.test.d.ts

@@ -0,0 +1 @@
+export {};