@kaelio/ktx 0.7.0 → 0.8.0

package/dist/cli-program.js

@@ -1,6 +1,7 @@
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { Command, InvalidArgumentError } from '@commander-js/extra-typings';
+import { registerCompletionCommands } from './commands/completion-commands.js';
 import { registerConnectionCommands } from './commands/connection-commands.js';
 import { registerIngestCommands } from './commands/ingest-commands.js';
 import { registerWikiCommands } from './commands/knowledge-commands.js';
@@ -319,6 +320,11 @@ export function collectCommandFlagsPresent(command) {
 export function buildKtxProgram(options) {
     const program = createBaseProgram(options.packageInfo, options.io);
     program.hook('preAction', async (_thisCommand, actionCommand) => {
+        // The hidden completion command must stay silent and side-effect free: skip
+        // the telemetry notice, command span, and project checks entirely.
+        if (commandPath(actionCommand).includes('__complete')) {
+            return;
+        }
         const telemetry = await import('./telemetry/index.js');
         options.setTelemetryModule?.(telemetry);
         await telemetry.showTelemetryNoticeIfNeeded(options.io, options.packageInfo);
@@ -362,6 +368,7 @@ export function buildKtxProgram(options) {
     registerStatusCommands(program, context);
     registerMcpCommands(program, context);
     registerAdminCommands(program, context);
+    registerCompletionCommands(program, context);
     return program;
 }
 export async function runCommanderKtxCli(argv, io, deps, info, options) {

package/dist/command-schemas.d.ts

@@ -20,8 +20,8 @@ export declare const slQueryCommandSchema: z.ZodObject<{
     }, z.core.$strip>>;
     queryFile: z.ZodOptional<z.ZodString>;
     format: z.ZodEnum<{
-        sql: "sql";
         json: "json";
+        sql: "sql";
     }>;
     execute: z.ZodBoolean;
     cliVersion: z.ZodString;

package/dist/command-tree.js

@@ -5,7 +5,11 @@ export function walkCommandTree(command) {
         description: command.description(),
         aliases: command.aliases(),
         arguments: command.registeredArguments.map(formatArgumentDeclaration),
-        children: command.commands.map((child) => walkCommandTree(child)),
+        // Internal commands (e.g. the shell-completion helper `__complete`) use a
+        // `__` prefix and are omitted from the human-facing command tree.
+        children: command.commands
+            .filter((child) => !child.name().startsWith('__'))
+            .map((child) => walkCommandTree(child)),
     };
 }
 export function formatCommandTree(node) {

package/dist/commands/completion-commands.d.ts

@@ -0,0 +1,3 @@
+import { type Command } from '@commander-js/extra-typings';
+import type { KtxCliCommandContext } from '../cli-program.js';
+export declare function registerCompletionCommands(program: Command, context: KtxCliCommandContext): void;

package/dist/commands/completion-commands.js

@@ -0,0 +1,38 @@
+import { Argument } from '@commander-js/extra-typings';
+import { computeCompletions } from '../completion/complete-engine.js';
+import { completionScript } from '../completion/completion-scripts.js';
+import { createProjectCompletionProviders } from '../completion/dynamic-candidates.js';
+import { profileMark } from '../startup-profile.js';
+profileMark('module:commands/completion-commands');
+export function registerCompletionCommands(program, context) {
+    program
+        .command('completion')
+        .description('Print a shell completion script for ktx')
+        .addArgument(new Argument('<shell>', 'Target shell').choices(['zsh', 'bash']))
+        .addHelpText('after', '\nEnable completion by adding the matching line to your shell startup file:\n' +
+        '  zsh:  eval "$(ktx completion zsh)"\n' +
+        '  bash: eval "$(ktx completion bash)"\n')
+        .action((shell) => {
+        context.io.stdout.write(completionScript(shell));
+    });
+    // Hidden command invoked by the generated shell scripts. It must only ever
+    // print newline-separated candidates to stdout and exit 0, so a TAB press is
+    // never disrupted by an error, a telemetry notice, or a parse failure.
+    program
+        .command('__complete', { hidden: true })
+        .argument('[words...]')
+        .allowUnknownOption(true)
+        .helpOption(false)
+        .action(async (words) => {
+        try {
+            const candidates = await computeCompletions(program, words, createProjectCompletionProviders());
+            if (candidates.length > 0) {
+                context.io.stdout.write(`${candidates.join('\n')}\n`);
+            }
+        }
+        catch {
+            // Swallow: completion must never break the shell.
+        }
+        context.setExitCode(0);
+    });
+}

package/dist/commands/ingest-commands.js

@@ -11,8 +11,6 @@ export function registerIngestCommands(program, context, commandOptions) {
         .usage('[options] [connectionId]')
         .argument('[connectionId]', 'Configured connection id to ingest (omit to ingest all)')
         .option('--all', 'Ingest all configured connections', false)
-        .addOption(new Option('--fast', 'Use deterministic database schema ingest').conflicts('deep'))
-        .addOption(new Option('--deep', 'Use AI-enriched database ingest').conflicts('fast'))
         .addOption(new Option('--query-history', 'Include database query-history usage patterns').conflicts('noQueryHistory'))
         .addOption(new Option('--no-query-history', 'Skip database query-history usage patterns'))
         .option('--query-history-window-days <days>', 'Query-history lookback window for this run', parsePositiveIntegerOption)
@@ -57,8 +55,6 @@ export function registerIngestCommands(program, context, commandOptions) {
             all: selection.kind === 'all',
             json: options.json === true,
             inputMode: options.input === false ? 'disabled' : 'auto',
-            ...(options.fast === true ? { depth: 'fast' } : {}),
-            ...(options.deep === true ? { depth: 'deep' } : {}),
             queryHistory,
             ...(options.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: options.queryHistoryWindowDays } : {}),
             cliVersion: context.packageInfo.version,

package/dist/commands/knowledge-commands.js

@@ -11,9 +11,9 @@ function isDebugEnabled(command) {
     return options.debug === true;
 }
 export function registerWikiCommands(program, context) {
-    program
+    const wiki = program
         .command('wiki')
-        .description('List or search local wiki pages')
+        .description('List, search, or read local wiki pages')
         .usage('[options] [query...]')
         .argument('[query...]', 'Search query; omit to list all pages')
         .option('--user-id <id>', 'Local user id', 'local')
@@ -50,4 +50,17 @@ export function registerWikiCommands(program, context) {
             cliVersion: context.packageInfo.version,
         });
     });
+    wiki
+        .command('read')
+        .description('Read a wiki page file by key')
+        .argument('<key>', 'Wiki page key')
+        .action(async (key, _options, command) => {
+        const parentOpts = command.parent?.opts();
+        await runKnowledgeArgs(context, {
+            command: 'read',
+            projectDir: resolveCommandProjectDir(command),
+            key,
+            userId: parentOpts?.userId ?? 'local',
+        });
+    });
 }

package/dist/commands/setup-commands.js

@@ -195,9 +195,9 @@ export function registerSetupCommands(program, context) {
         .addOption(new Option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML').hideHelp())
         .addOption(new Option('--source-branch <branch>', 'Git branch for source setup').hideHelp())
         .addOption(new Option('--source-subpath <path>', 'Repo subpath for source setup').hideHelp())
-        .addOption(new Option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth').hideHelp())
+        .addOption(new Option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth or Notion integration token').hideHelp())
         .addOption(new Option('--source-url <url>', 'Source service URL for Metabase or Looker').hideHelp())
-        .addOption(new Option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion').hideHelp())
+        .addOption(new Option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase').hideHelp())
         .addOption(new Option('--source-client-id <id>', 'Looker client id').hideHelp())
         .addOption(new Option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref').hideHelp())
         .addOption(new Option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id').hideHelp())

package/dist/commands/sl-commands.js

@@ -63,19 +63,27 @@ export function registerSlCommands(program, context, commandName = 'sl') {
             cliVersion: context.packageInfo.version,
         });
     });
+    sl.command('read')
+        .description('Read a semantic-layer source YAML file')
+        .argument('<sourceName>', 'Semantic-layer source name')
+        .action(async (sourceName, _options, command) => {
+        const parentOpts = command.parent?.opts();
+        await runSlArgs(context, {
+            command: 'read',
+            projectDir: resolveCommandProjectDir(command),
+            connectionId: parentOpts?.connectionId,
+            sourceName,
+        });
+    });
     sl.command('validate')
-        .description('Validate a semantic-layer source (set --connection-id on `ktx sl`)')
+        .description('Validate a semantic-layer source')
         .argument('<sourceName>', 'Semantic-layer source name')
         .action(async (sourceName, _options, command) => {
         const parentOpts = command.parent?.opts();
-        const connectionId = parentOpts?.connectionId;
-        if (connectionId === undefined) {
-            command.error("error: required option '--connection-id <id>' not specified");
-        }
         await runSlArgs(context, {
             command: 'validate',
             projectDir: resolveCommandProjectDir(command),
-            connectionId: connectionId,
+            connectionId: parentOpts?.connectionId,
             sourceName,
         });
     });
@@ -99,10 +107,14 @@ export function registerSlCommands(program, context, commandName = 'sl') {
             throw new Error('sl query requires at least one --measure');
         }
         const parentOpts = command.parent?.opts();
+        const connectionId = parentOpts?.connectionId;
+        if (connectionId === undefined) {
+            command.error("error: required option '--connection-id <id>' not specified");
+        }
         const args = slQueryCommandSchema.parse({
             command: 'query',
             projectDir: resolveCommandProjectDir(command),
-            connectionId: parentOpts?.connectionId,
+            connectionId,
             ...(options.queryFile
                 ? { queryFile: options.queryFile }
                 : {

package/dist/completion/complete-engine.d.ts

@@ -0,0 +1,19 @@
+import type { CommandUnknownOpts } from '@commander-js/extra-typings';
+/**
+ * Dynamic completion candidates that depend on project state (semantic-layer
+ * source names, wiki page keys, connection ids). Injected so the engine stays
+ * pure and unit-testable without touching the filesystem.
+ */
+export interface CompletionProviders {
+    /** Candidate operands for a positional argument of the active command path. */
+    positionalCandidates(commandPath: string[], typedTokens: string[]): Promise<string[]>;
+    /** Candidate values for an option that has no static `choices` (e.g. `--connection-id`). */
+    optionValueCandidates(commandPath: string[], optionFlag: string, typedTokens: string[]): Promise<string[]>;
+}
+/**
+ * Compute completion candidates for the partial last element of `words`
+ * (everything the shell has on the line after `ktx`). The active command and
+ * its flags are derived by walking the live Commander tree, so completion never
+ * drifts from the real command structure.
+ */
+export declare function computeCompletions(program: CommandUnknownOpts, words: string[], providers: CompletionProviders): Promise<string[]>;

package/dist/completion/complete-engine.js

@@ -0,0 +1,128 @@
+function isHiddenCommand(command) {
+    // Completion mirrors `ktx --help`: commands registered with `{ hidden: true }`
+    // (the `__complete` helper and `mcp serve-internal`) are internal and must not
+    // surface. Commander exposes this only through the private `_hidden` field its
+    // own help renderer reads, so a name heuristic like a `__` prefix is not enough.
+    return command._hidden === true;
+}
+function resolveCommand(program, typedTokens) {
+    let command = program;
+    const commandPath = [];
+    for (let index = 0; index < typedTokens.length; index += 1) {
+        const token = typedTokens[index];
+        if (token.startsWith('-')) {
+            // A value-taking option in the `--flag value` form consumes the next token
+            // as its value, so skip that value before matching subcommands. Otherwise a
+            // connection id like `query` would be resolved as the `sl query` subcommand
+            // instead of being treated as the `--connection-id` value. The `--flag=value`
+            // form carries its own value and consumes nothing extra.
+            if (!token.includes('=')) {
+                const option = findOption(command, token);
+                if (option && !option.isBoolean()) {
+                    index += 1;
+                }
+            }
+            continue;
+        }
+        const sub = command.commands.find((candidate) => candidate.name() === token || candidate.aliases().includes(token));
+        if (sub) {
+            command = sub;
+            commandPath.push(sub.name());
+        }
+    }
+    return { command, commandPath };
+}
+function collectOptions(command) {
+    const options = [];
+    let current = command;
+    while (current) {
+        options.push(...current.options);
+        current = current.parent;
+    }
+    return options;
+}
+function findOption(command, flag) {
+    return collectOptions(command).find((option) => option.long === flag || option.short === flag);
+}
+function isRepeatableOption(option) {
+    // Variadic options, and options backed by a collector with an array default
+    // (e.g. `--measure`/`--dimension`), may be supplied more than once.
+    return option.variadic || Array.isArray(option.defaultValue);
+}
+function flagCandidates(command, typedTokens) {
+    const present = new Set(typedTokens.filter((token) => token.startsWith('-')));
+    const candidates = [];
+    for (const option of collectOptions(command)) {
+        if (option.hidden || !option.long) {
+            continue;
+        }
+        if (present.has(option.long) && !isRepeatableOption(option)) {
+            continue;
+        }
+        candidates.push(option.long);
+    }
+    return candidates;
+}
+async function optionValueCandidates(resolved, option, typedTokens, providers) {
+    if (option.argChoices && option.argChoices.length > 0) {
+        return option.argChoices;
+    }
+    return providers.optionValueCandidates(resolved.commandPath, option.long ?? option.name(), typedTokens);
+}
+function dedupeSortFilter(candidates, partial) {
+    const seen = new Set();
+    const matches = [];
+    for (const candidate of candidates) {
+        if (!candidate.startsWith(partial) || seen.has(candidate)) {
+            continue;
+        }
+        seen.add(candidate);
+        matches.push(candidate);
+    }
+    return matches.sort();
+}
+/**
+ * Compute completion candidates for the partial last element of `words`
+ * (everything the shell has on the line after `ktx`). The active command and
+ * its flags are derived by walking the live Commander tree, so completion never
+ * drifts from the real command structure.
+ */
+export async function computeCompletions(program, words, providers) {
+    const partial = words.length > 0 ? (words[words.length - 1] ?? '') : '';
+    const typedTokens = words.slice(0, -1);
+    const resolved = resolveCommand(program, typedTokens);
+    // (a) Option value via the `--opt=value` form.
+    const equalsMatch = /^(--[^=]+)=(.*)$/.exec(partial);
+    if (equalsMatch) {
+        const [, flag, valuePartial] = equalsMatch;
+        const option = findOption(resolved.command, flag);
+        if (!option || option.isBoolean()) {
+            return [];
+        }
+        const values = await optionValueCandidates(resolved, option, typedTokens, providers);
+        return dedupeSortFilter(values.map((value) => `${flag}=${value}`), `${flag}=${valuePartial}`);
+    }
+    // (b) Option value via the `--opt value` form (previous token is a value-taking option).
+    const previous = typedTokens[typedTokens.length - 1];
+    if (previous && previous.startsWith('-') && !partial.startsWith('-')) {
+        const option = findOption(resolved.command, previous);
+        if (option && !option.isBoolean()) {
+            return dedupeSortFilter(await optionValueCandidates(resolved, option, typedTokens, providers), partial);
+        }
+    }
+    // (c) Flag completion.
+    if (partial.startsWith('-')) {
+        return dedupeSortFilter(flagCandidates(resolved.command, typedTokens), partial);
+    }
+    // (d) Positional: subcommand names union static argument choices union dynamic operand candidates.
+    const candidates = resolved.command.commands
+        .filter((sub) => !isHiddenCommand(sub))
+        .map((sub) => sub.name());
+    for (const argument of resolved.command.registeredArguments) {
+        if (argument.argChoices) {
+            candidates.push(...argument.argChoices);
+        }
+    }
+    candidates.push(...(await providers.positionalCandidates(resolved.commandPath, typedTokens)));
+    return dedupeSortFilter(candidates, partial);
+}

package/dist/completion/completion-scripts.d.ts

@@ -0,0 +1 @@
+export declare function completionScript(shell: 'zsh' | 'bash'): string;

package/dist/completion/completion-scripts.js

@@ -0,0 +1,36 @@
+// Static shell completion scripts emitted by `ktx completion <shell>`.
+//
+// Both scripts gather the words on the current command line (excluding the
+// leading `ktx`), append the partial word under the cursor, and delegate to the
+// hidden `ktx __complete` command, which prints newline-separated candidates.
+// All command/flag/entity knowledge lives in `ktx __complete` so these scripts
+// never have to encode the command tree.
+//
+// Lines are single-quoted JS strings so the shell `${...}` expansions are
+// emitted verbatim (a template literal would try to interpolate them).
+const ZSH_SCRIPT = [
+    '#compdef ktx',
+    '_ktx() {',
+    '  local -a candidates',
+    '  local out',
+    '  out="$(ktx __complete -- "${words[@]:1:$((CURRENT-1))}" 2>/dev/null)" || return 0',
+    '  candidates=("${(@f)out}")',
+    '  compadd -- $candidates',
+    '}',
+    'compdef _ktx ktx',
+    '',
+].join('\n');
+const BASH_SCRIPT = [
+    '_ktx() {',
+    '  local cur out',
+    '  cur="${COMP_WORDS[COMP_CWORD]}"',
+    '  out="$(ktx __complete -- "${COMP_WORDS[@]:1:COMP_CWORD}" 2>/dev/null)" || { COMPREPLY=(); return 0; }',
+    "  local IFS=$'\\n'",
+    '  COMPREPLY=($(compgen -W "${out}" -- "$cur"))',
+    '}',
+    'complete -F _ktx ktx',
+    '',
+].join('\n');
+export function completionScript(shell) {
+    return shell === 'zsh' ? ZSH_SCRIPT : BASH_SCRIPT;
+}

package/dist/completion/dynamic-candidates.d.ts

@@ -0,0 +1,6 @@
+import type { CompletionProviders } from './complete-engine.js';
+/**
+ * Project-backed completion providers. Every entry swallows its own errors so a
+ * failed lookup never breaks the shell — completion degrades to commands/flags.
+ */
+export declare function createProjectCompletionProviders(): CompletionProviders;

package/dist/completion/dynamic-candidates.js

@@ -0,0 +1,98 @@
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { resolveKtxProjectDir } from '../project-resolver.js';
+/** Extract an option value from already-typed tokens (`--flag value` or `--flag=value`). */
+function extractOptionValue(tokens, flag) {
+    const prefix = `${flag}=`;
+    for (let index = 0; index < tokens.length; index += 1) {
+        const token = tokens[index];
+        if (token === flag) {
+            const next = tokens[index + 1];
+            if (next !== undefined && !next.startsWith('-')) {
+                return next;
+            }
+        }
+        else if (token.startsWith(prefix)) {
+            return token.slice(prefix.length);
+        }
+    }
+    return undefined;
+}
+/**
+ * Resolve and load the project the user is completing against. Honors a
+ * `--project-dir` typed on the line, then `KTX_PROJECT_DIR`, then the nearest
+ * `ktx.yaml`. Returns null (no completions) when there is no project, without
+ * creating any files.
+ */
+async function loadCompletionProject(typedTokens) {
+    const explicitProjectDir = extractOptionValue(typedTokens, '--project-dir');
+    const projectDir = resolveKtxProjectDir(explicitProjectDir !== undefined ? { explicitProjectDir } : {});
+    if (!existsSync(join(projectDir, 'ktx.yaml'))) {
+        return null;
+    }
+    const { loadKtxProject } = await import('../context/project/project.js');
+    return loadKtxProject({ projectDir });
+}
+async function sourceNames(typedTokens) {
+    const project = await loadCompletionProject(typedTokens);
+    if (!project) {
+        return [];
+    }
+    const connectionId = extractOptionValue(typedTokens, '--connection-id');
+    const { listLocalSlSources } = await import('../context/sl/local-sl.js');
+    const summaries = await listLocalSlSources(project, connectionId !== undefined ? { connectionId } : {});
+    return [...new Set(summaries.map((summary) => summary.name))];
+}
+async function wikiPageKeys(typedTokens) {
+    const project = await loadCompletionProject(typedTokens);
+    if (!project) {
+        return [];
+    }
+    const userId = extractOptionValue(typedTokens, '--user-id');
+    const { listLocalKnowledgePageKeys } = await import('../context/wiki/local-knowledge.js');
+    return listLocalKnowledgePageKeys(project, userId !== undefined ? { userId } : {});
+}
+async function connectionIds(typedTokens) {
+    const project = await loadCompletionProject(typedTokens);
+    if (!project) {
+        return [];
+    }
+    return Object.keys(project.config.connections).sort();
+}
+/**
+ * Project-backed completion providers. Every entry swallows its own errors so a
+ * failed lookup never breaks the shell — completion degrades to commands/flags.
+ */
+export function createProjectCompletionProviders() {
+    return {
+        async positionalCandidates(commandPath, typedTokens) {
+            try {
+                const key = commandPath.join(' ');
+                if (key === 'sl read' || key === 'sl validate') {
+                    return await sourceNames(typedTokens);
+                }
+                if (key === 'wiki read') {
+                    return await wikiPageKeys(typedTokens);
+                }
+                if (key === 'connection test' || key === 'ingest') {
+                    return await connectionIds(typedTokens);
+                }
+                return [];
+            }
+            catch {
+                return [];
+            }
+        },
+        async optionValueCandidates(_commandPath, optionFlag, typedTokens) {
+            try {
+                if (optionFlag === '--connection-id' || optionFlag === '--connection') {
+                    return await connectionIds(typedTokens);
+                }
+                return [];
+            }
+            catch {
+                return [];
+            }
+        },
+    };
+}

package/dist/connection-drivers.d.ts

@@ -0,0 +1,3 @@
+import type { KtxProjectConnectionConfig } from './context/project/config.js';
+export declare function normalizeConnectionDriver(connection: KtxProjectConnectionConfig): string;
+export declare function isDatabaseDriver(driver: string): boolean;

package/dist/connection-drivers.js

@@ -0,0 +1,17 @@
+const KTX_DATABASE_DRIVER_IDS = new Set([
+    'sqlite',
+    'postgres',
+    'mysql',
+    'clickhouse',
+    'sqlserver',
+    'bigquery',
+    'snowflake',
+]);
+export function normalizeConnectionDriver(connection) {
+    return String(connection.driver ?? '')
+        .trim()
+        .toLowerCase();
+}
+export function isDatabaseDriver(driver) {
+    return KTX_DATABASE_DRIVER_IDS.has(driver.trim().toLowerCase());
+}

package/dist/context/ingest/ingest-bundle.runner.d.ts

@@ -9,6 +9,14 @@ export declare class IngestBundleRunner {
     private readonly chainByConnection;
     constructor(deps: IngestBundleRunnerDeps);
     run(job: IngestBundleJob, ctx?: IngestJobContext): Promise<IngestBundleResult>;
+    /**
+     * When profiling is enabled — via the `KTX_PROFILE_INGEST` env var or the
+     * `ingest.profile` config setting — read the job's trace + tool transcripts
+     * and print a rolled-up timing breakdown to stderr. `json` emits the raw
+     * structured profile for coding agents; `table` emits a human summary.
+     * Best-effort: profiling never affects the run outcome.
+     */
+    private maybeEmitIngestProfile;
     protected stageRawFilesStage1: typeof stageRawFilesStage1;
     private syncKnowledgeSlRefsFromActions;
     protected materializeOverrideSnapshot(report: IngestReportSnapshot, ctx: {