@crafter/cli-tree 0.1.0

package/src/archaeology/prompts.ts

@@ -0,0 +1,178 @@
+import type { EnrichedCLINode } from "./types";
+
+export interface PromptContext {
+	cli: string;
+	version?: string;
+	knownCommands?: string[];
+	knownTree?: EnrichedCLINode;
+}
+
+export function surveyPrompt(ctx: PromptContext): string {
+	return `You are a CLI archaeologist. Your task is to produce a hypothesis map of the \`${ctx.cli}\` CLI based on your training knowledge.
+
+${ctx.version ? `Version: ${ctx.version}\n` : ""}Do not invoke the CLI yet. This is survey phase — hypothesize what families of commands exist and what subcommands belong to each family.
+
+Output STRICT JSON matching this schema:
+
+\`\`\`json
+{
+  "cli": "${ctx.cli}",
+  "description": "<one-line description of the CLI>",
+  "families": [
+    {
+      "name": "<family name, e.g. 'container management'>",
+      "description": "<one line>",
+      "subcommands": ["<subcommand-name>", ...]
+    }
+  ],
+  "topLevelFlags": ["--global-flag-1", "--global-flag-2"]
+}
+\`\`\`
+
+Rules:
+- Group related subcommands into families (container, image, network, auth, config, etc).
+- List ONLY subcommand names — no descriptions in this phase.
+- Include hidden/experimental commands you know about.
+- topLevelFlags = flags that work on the root (e.g. --version, --help, --config).
+- Output ONLY the JSON, no prose before or after.`;
+}
+
+export function excavationPrompt(ctx: PromptContext, subcommandPath: string[]): string {
+	const cmd = [ctx.cli, ...subcommandPath].join(" ");
+	return `You are a CLI archaeologist, currently excavating the \`${cmd}\` subcommand.
+
+You will validate your hypothesis by running the real CLI. Use the Bash tool to run:
+
+\`${cmd} --help\`
+
+Then parse its output and produce a structured description of this subcommand.
+
+Output STRICT JSON matching this schema:
+
+\`\`\`json
+{
+  "command": ${JSON.stringify([ctx.cli, ...subcommandPath])},
+  "exists": true|false,
+  "description": "<description from help output>",
+  "subcommands": [
+    { "name": "<sub>", "description": "<desc>" }
+  ],
+  "flags": [
+    {
+      "name": "flag-name",
+      "short": "f",
+      "type": "boolean|string|number|enum",
+      "description": "<desc>",
+      "required": false
+    }
+  ],
+  "args": [
+    { "name": "input", "required": true, "variadic": false }
+  ]
+}
+\`\`\`
+
+Rules:
+- If the command does not exist or --help fails, set exists: false and return empty arrays.
+- Extract every flag and arg from the help output.
+- Do NOT invent anything — only what is in the actual help text.
+- Output ONLY the JSON.`;
+}
+
+export function deepDivePrompt(ctx: PromptContext, subcommandPath: string[]): string {
+	const cmd = [ctx.cli, ...subcommandPath].join(" ");
+	return `You are a CLI archaeologist. You already excavated \`${cmd}\` from --help.
+
+Now dig deeper: based on your training knowledge, what flags exist for this command that are NOT documented in --help? Focus on:
+- Hidden flags (work but undocumented)
+- Experimental flags (marked as experimental in source code)
+- Deprecated flags (still functional but hidden)
+- Flags that only appear in man pages or shell completion scripts
+
+Output STRICT JSON:
+
+\`\`\`json
+{
+  "command": ${JSON.stringify([ctx.cli, ...subcommandPath])},
+  "hiddenFlags": [
+    {
+      "name": "flag-name",
+      "short": "f",
+      "type": "boolean|string|number|enum",
+      "hidden": true,
+      "description": "<what it does>",
+      "source": "source-code|man-page|completion|community-known"
+    }
+  ],
+  "deprecatedFlags": [
+    {
+      "name": "old-flag",
+      "description": "Deprecated in favor of --new-flag",
+      "replacement": "new-flag"
+    }
+  ],
+  "examples": [
+    "${cmd} --example-use-case"
+  ]
+}
+\`\`\`
+
+Rules:
+- Only suggest flags you are confident about. Confidence > 70%.
+- Prefer flags you have seen in real usage, PRs, issues, or man pages.
+- If unsure, omit. This phase is about quality, not quantity.
+- Output ONLY the JSON.`;
+}
+
+export function cartographyPrompt(ctx: PromptContext, subcommandPath: string[]): string {
+	const cmd = [ctx.cli, ...subcommandPath].join(" ");
+	return `You are a CLI archaeologist mapping flag interactions for \`${cmd}\`.
+
+Identify constraints between flags:
+- Mutually exclusive (cannot use together)
+- Requires (using A requires B)
+- Implies (using A sets B automatically)
+
+Output STRICT JSON:
+
+\`\`\`json
+{
+  "command": ${JSON.stringify([ctx.cli, ...subcommandPath])},
+  "constraints": [
+    {
+      "type": "mutually-exclusive|requires|implies",
+      "flags": ["--flag-a", "--flag-b"],
+      "description": "<human explanation>"
+    }
+  ]
+}
+\`\`\`
+
+Rules:
+- Only list constraints you are confident about.
+- Reference real flags from the --help output, not hypothetical ones.
+- Output ONLY the JSON.`;
+}
+
+export function validationInstructions(cli: string): string {
+	return `You are validating an archaeology result for \`${cli}\`.
+
+For each command/flag proposed by the LLM survey phase, you must:
+
+1. Run \`<cli> <subcommand> --help\` via the Bash tool.
+2. Parse the real help output.
+3. Mark each flag as:
+   - "verified" — appears in --help, can confirm it exists
+   - "rejected" — does NOT appear in --help, LLM hallucinated
+   - "unconfirmed" — not in --help but LLM may still be correct (hidden flags)
+
+Return a validation report. Flags marked "rejected" must be removed from the final tree.
+Flags marked "unconfirmed" are kept only if --include-unverified is set.`;
+}
+
+export function extractJsonFromLlmResponse(response: string): unknown {
+	const fenced = response.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
+	const raw = fenced ? fenced[1]! : response;
+	const trimmed = raw.trim();
+	return JSON.parse(trimmed);
+}

package/src/archaeology/types.ts

@@ -0,0 +1,139 @@
+import type { CLINode, Flag, Arg } from "../types";
+
+export type SourceKind =
+	| "help"
+	| "llm-survey"
+	| "llm-deep-dive"
+	| "llm-cartography"
+	| "llm-validated"
+	| "man"
+	| "completion"
+	| "user";
+
+export interface Source {
+	type: SourceKind;
+	confidence: number;
+	verifiedAt?: string;
+	note?: string;
+}
+
+export interface EnrichedFlag extends Flag {
+	hidden?: boolean;
+	deprecated?: boolean;
+	experimental?: boolean;
+	source?: Source;
+	examples?: string[];
+	excludes?: string[];
+	requires?: string[];
+}
+
+export interface EnrichedArg extends Arg {
+	source?: Source;
+}
+
+export interface Example {
+	command: string;
+	description?: string;
+	source?: Source;
+}
+
+export interface FlagConstraint {
+	type: "mutually-exclusive" | "requires" | "implies";
+	flags: string[];
+	description?: string;
+}
+
+export interface EnrichedCLINode {
+	name: string;
+	description?: string;
+	aliases?: string[];
+	hidden?: boolean;
+	deprecated?: boolean;
+	experimental?: boolean;
+	subcommands?: EnrichedCLINode[];
+	flags?: EnrichedFlag[];
+	args?: EnrichedArg[];
+	constraints?: FlagConstraint[];
+	examples?: Example[];
+	source?: Source;
+}
+
+export interface SurveyHypothesis {
+	cli: string;
+	description: string;
+	families: Array<{
+		name: string;
+		description: string;
+		subcommands: string[];
+	}>;
+	topLevelFlags: string[];
+}
+
+export interface DeepDiveProposal {
+	command: string[];
+	flags: Array<{
+		name: string;
+		short?: string;
+		type: "boolean" | "string" | "number" | "enum";
+		hidden?: boolean;
+		deprecated?: boolean;
+		description?: string;
+	}>;
+	examples: string[];
+}
+
+export interface ValidationVerdict {
+	command: string[];
+	exists: boolean;
+	verifiedFlags: string[];
+	rejectedFlags: string[];
+	unconfirmedFlags: string[];
+	rawHelp?: string;
+	error?: string;
+}
+
+export interface ArchaeologyCache {
+	cli: string;
+	cliVersion?: string;
+	createdAt: string;
+	tree: EnrichedCLINode;
+	phases: {
+		survey: boolean;
+		excavation: boolean;
+		deepDive: boolean;
+		cartography: boolean;
+	};
+}
+
+export interface ArchaeologyOptions {
+	cacheDir?: string;
+	skipCache?: boolean;
+	includeUnverified?: boolean;
+	maxDepth?: number;
+	phases?: {
+		survey?: boolean;
+		excavation?: boolean;
+		deepDive?: boolean;
+		cartography?: boolean;
+	};
+}
+
+export function toCLINode(node: EnrichedCLINode): CLINode {
+	const base: CLINode = { name: node.name };
+	if (node.description) base.description = node.description;
+	if (node.aliases) base.aliases = node.aliases;
+	if (node.subcommands) base.subcommands = node.subcommands.map(toCLINode);
+	if (node.flags) base.flags = node.flags.map(f => ({ ...f }));
+	if (node.args) base.args = node.args.map(a => ({ ...a }));
+	return base;
+}
+
+export function fromCLINode(node: CLINode, source: Source = { type: "help", confidence: 1 }): EnrichedCLINode {
+	const base: EnrichedCLINode = { name: node.name, source };
+	if (node.description) base.description = node.description;
+	if (node.aliases) base.aliases = node.aliases;
+	if (node.subcommands) base.subcommands = node.subcommands.map(s => fromCLINode(s, source));
+	if (node.flags) base.flags = node.flags.map(f => ({ ...f, source }));
+	if (node.args) base.args = node.args.map(a => ({ ...a, source }));
+	return base;
+}

package/src/archaeology/validate.ts

@@ -0,0 +1,157 @@
+import { parseHelp } from "../parse";
+import type { CLINode } from "../types";
+import type { ValidationVerdict, EnrichedCLINode, SurveyHypothesis, DeepDiveProposal } from "./types";
+
+export async function runHelpCommand(binary: string, subcommandPath: string[]): Promise<string | null> {
+	try {
+		const proc = Bun.spawn([binary, ...subcommandPath, "--help"], {
+			stdout: "pipe",
+			stderr: "pipe",
+		});
+
+		const timeout = setTimeout(() => proc.kill(), 5000);
+
+		const [stdout, stderr] = await Promise.all([
+			new Response(proc.stdout).text(),
+			new Response(proc.stderr).text(),
+		]);
+
+		clearTimeout(timeout);
+		await proc.exited;
+
+		const output = stdout || stderr;
+		return output.trim() ? output : null;
+	} catch {
+		return null;
+	}
+}
+
+export async function validateSubcommandExists(
+	binary: string,
+	subcommandPath: string[],
+): Promise<boolean> {
+	const output = await runHelpCommand(binary, subcommandPath);
+	if (!output) return false;
+
+	const lower = output.toLowerCase();
+	const hasError =
+		/unknown (command|subcommand)|command not found|no such command/.test(lower) ||
+		/is not a \w+ command/.test(lower) ||
+		/invalid (command|subcommand)/.test(lower) ||
+		/error: unrecognized/.test(lower);
+	return !hasError;
+}
+
+export async function validateSurvey(
+	binary: string,
+	survey: SurveyHypothesis,
+): Promise<{ confirmed: string[]; rejected: string[] }> {
+	const confirmed: string[] = [];
+	const rejected: string[] = [];
+
+	const allSubs = survey.families.flatMap(f => f.subcommands);
+	const unique = Array.from(new Set(allSubs));
+
+	const BATCH = 5;
+	for (let i = 0; i < unique.length; i += BATCH) {
+		const batch = unique.slice(i, i + BATCH);
+		const results = await Promise.all(
+			batch.map(async sub => ({ sub, exists: await validateSubcommandExists(binary, [sub]) })),
+		);
+		for (const { sub, exists } of results) {
+			if (exists) confirmed.push(sub);
+			else rejected.push(sub);
+		}
+	}
+
+	return { confirmed, rejected };
+}
+
+export async function excavateCommand(
+	binary: string,
+	subcommandPath: string[],
+): Promise<CLINode | null> {
+	const output = await runHelpCommand(binary, subcommandPath);
+	if (!output) return null;
+
+	const name = subcommandPath[subcommandPath.length - 1] ?? binary;
+	return parseHelp(name, output);
+}
+
+export async function validateDeepDive(
+	binary: string,
+	proposal: DeepDiveProposal,
+): Promise<ValidationVerdict> {
+	const subcommandPath = proposal.command.slice(1);
+	const realHelp = await runHelpCommand(binary, subcommandPath);
+
+	if (!realHelp) {
+		return {
+			command: proposal.command,
+			exists: false,
+			verifiedFlags: [],
+			rejectedFlags: proposal.flags.map(f => f.name),
+			unconfirmedFlags: [],
+			error: "Command did not produce help output",
+		};
+	}
+
+	const parsed = parseHelp(subcommandPath[subcommandPath.length - 1] ?? binary, realHelp);
+	const realFlagNames = new Set(parsed.flags?.map(f => f.name) ?? []);
+
+	const verifiedFlags: string[] = [];
+	const rejectedFlags: string[] = [];
+	const unconfirmedFlags: string[] = [];
+
+	for (const flag of proposal.flags) {
+		if (realFlagNames.has(flag.name)) {
+			verifiedFlags.push(flag.name);
+		} else if (flag.hidden || flag.deprecated) {
+			unconfirmedFlags.push(flag.name);
+		} else {
+			rejectedFlags.push(flag.name);
+		}
+	}
+
+	return {
+		command: proposal.command,
+		exists: true,
+		verifiedFlags,
+		rejectedFlags,
+		unconfirmedFlags,
+		rawHelp: realHelp,
+	};
+}
+
+export interface ValidationSummary {
+	totalProposed: number;
+	verified: number;
+	rejected: number;
+	unconfirmed: number;
+	hallucinations: string[];
+}
+
+export function summarizeValidation(verdicts: ValidationVerdict[]): ValidationSummary {
+	let verified = 0;
+	let rejected = 0;
+	let unconfirmed = 0;
+	const hallucinations: string[] = [];
+
+	for (const v of verdicts) {
+		verified += v.verifiedFlags.length;
+		rejected += v.rejectedFlags.length;
+		unconfirmed += v.unconfirmedFlags.length;
+
+		for (const flag of v.rejectedFlags) {
+			hallucinations.push(`${v.command.join(" ")} --${flag}`);
+		}
+	}
+
+	return {
+		totalProposed: verified + rejected + unconfirmed,
+		verified,
+		rejected,
+		unconfirmed,
+		hallucinations,
+	};
+}