@inkeep/agents-cli 0.39.4 → 0.40.0

package/dist/commands/pull-v3/index.js

@@ -0,0 +1,510 @@
+import { ManagementApiClient } from "../../api.js";
+import "../../utils/config.js";
+import { initializeCommand } from "../../utils/cli-pipeline.js";
+import { performBackgroundVersionCheck } from "../../utils/background-version-check.js";
+import "../../utils/json-comparison.js";
+import { loadProject } from "../../utils/project-loader.js";
+import { extractSubAgents } from "./utils/component-registry.js";
+import { compareProjects } from "./project-comparator.js";
+import { checkAndPromptForStaleComponentCleanup, cleanupStaleComponents, copyProjectToTemp } from "./component-updater.js";
+import { introspectGenerate } from "./introspect-generator.js";
+import { existsSync, mkdirSync } from "node:fs";
+import { basename, dirname, join, resolve } from "node:path";
+import * as p from "@clack/prompts";
+import chalk from "chalk";
+import { EventEmitter } from "node:events";
+
+//#region src/commands/pull-v3/index.ts
+/**
+* Pull v3 - Clean, efficient project generation
+*
+* Step 1: Validate and compile existing code
+* Step 2: Compare project with DB to detect ALL changes
+* Step 3: Classify changes as new vs modified components
+* Step 4: Generate new components deterministically
+* Step 5: Use LLM to correct modified components
+*/
+EventEmitter.defaultMaxListeners = 20;
+/**
+* Clean up stdin state between interactive prompts to prevent listener leaks
+*/
+function resetStdinState() {
+	process.stdin.removeAllListeners("data");
+	process.stdin.removeAllListeners("keypress");
+	process.stdin.removeAllListeners("end");
+	if (process.stdin.isTTY && process.stdin.isRaw) process.stdin.setRawMode(false);
+	if (!process.stdin.isPaused()) process.stdin.pause();
+}
+/**
+* Create project directory structure
+*/
+function createProjectStructure(projectDir, projectId) {
+	const projectRoot = projectDir;
+	const paths = {
+		projectRoot,
+		agentsDir: join(projectRoot, "agents"),
+		toolsDir: join(projectRoot, "tools"),
+		dataComponentsDir: join(projectRoot, "data-components"),
+		artifactComponentsDir: join(projectRoot, "artifact-components"),
+		statusComponentsDir: join(projectRoot, "status-components"),
+		environmentsDir: join(projectRoot, "environments"),
+		credentialsDir: join(projectRoot, "credentials"),
+		contextConfigsDir: join(projectRoot, "context-configs"),
+		externalAgentsDir: join(projectRoot, "external-agents")
+	};
+	Object.values(paths).forEach((dir) => {
+		if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+	});
+	return paths;
+}
+/**
+* Enrich canDelegateTo references with component type information
+*/
+function enrichCanDelegateToWithTypes(project, debug = false) {
+	const agentIds = new Set(project.agents ? Object.keys(project.agents) : []);
+	const subAgentIds = new Set(Object.keys(extractSubAgents(project)));
+	const externalAgentIds = new Set(project.externalAgents ? Object.keys(project.externalAgents) : []);
+	const enrichCanDelegateToArray = (canDelegateTo, context) => {
+		if (!Array.isArray(canDelegateTo)) return;
+		for (let i = 0; i < canDelegateTo.length; i++) {
+			const item = canDelegateTo[i];
+			if (typeof item !== "string") continue;
+			const id = item;
+			let enrichedItem = null;
+			if (agentIds.has(id)) enrichedItem = { agentId: id };
+			else if (subAgentIds.has(id)) enrichedItem = { subAgentId: id };
+			else if (externalAgentIds.has(id)) enrichedItem = { externalAgentId: id };
+			else continue;
+			canDelegateTo[i] = enrichedItem;
+		}
+	};
+	if (project.agents) {
+		for (const [_, agentData] of Object.entries(project.agents)) if (agentData.subAgents) {
+			for (const [subAgentId, subAgentData] of Object.entries(agentData.subAgents)) if (subAgentData.canDelegateTo) enrichCanDelegateToArray(subAgentData.canDelegateTo, `subAgent:${subAgentId}`);
+		}
+	}
+}
+/**
+* Read existing project from filesystem if it exists
+*/
+async function readExistingProject(projectRoot, debug = false) {
+	if (!existsSync(join(projectRoot, "index.ts"))) return null;
+	try {
+		const { loadProject: loadProject$1 } = await import("../../utils/project-loader.js");
+		const project = await loadProject$1(projectRoot);
+		return await Promise.race([project.getFullDefinition(), new Promise((_, reject) => {
+			setTimeout(() => {
+				reject(/* @__PURE__ */ new Error("getFullDefinition() timed out after 30 seconds - likely circular reference or infinite loop in local project"));
+			}, 3e4);
+		})]);
+	} catch (error) {
+		const errorMessage = error instanceof Error ? error.message : String(error);
+		errorMessage.includes("Credential") && errorMessage.includes("not found");
+		return null;
+	}
+}
+/**
+* Main pull-v3 command
+* @returns PullResult when in batch mode, otherwise void (exits process)
+*/
+async function pullV3Command(options) {
+	if (options.all) {
+		await pullAllProjects(options);
+		return;
+	}
+	const batchMode = options._batchMode ?? false;
+	const originalLogLevel = process.env.LOG_LEVEL;
+	process.env.LOG_LEVEL = "silent";
+	const restoreLogLevel = () => {
+		if (originalLogLevel !== void 0) process.env.LOG_LEVEL = originalLogLevel;
+		else delete process.env.LOG_LEVEL;
+	};
+	if (!batchMode) performBackgroundVersionCheck();
+	console.log(chalk.blue("\nInkeep Pull:"));
+	if (options.introspect) console.log(chalk.gray("  Introspect mode • Complete regeneration • No comparison needed"));
+	else console.log(chalk.gray("  Smart comparison • Detect all changes • Targeted updates"));
+	const s = p.spinner();
+	try {
+		const { config, profile, isCI } = await initializeCommand({
+			configPath: options.config,
+			profileName: options.profile,
+			tag: options.tag,
+			showSpinner: true,
+			spinnerText: "Loading configuration...",
+			logConfig: true,
+			quiet: options.quiet
+		});
+		s.start("Detecting project...");
+		let projectDir;
+		let projectId;
+		let localProjectForId = null;
+		const currentDir = process.cwd();
+		if (existsSync(join(currentDir, "index.ts"))) {
+			projectDir = currentDir;
+			s.start("Loading local project...");
+			try {
+				localProjectForId = await loadProject(projectDir);
+				const localProjectId = localProjectForId.getId();
+				if (options.project) {
+					if (localProjectId !== options.project) {
+						s.stop("Project ID mismatch");
+						console.error(chalk.red(`Local project ID "${localProjectId}" doesn't match --project "${options.project}"`));
+						console.error(chalk.yellow("Either remove --project flag or ensure it matches the local project ID"));
+						if (batchMode) return {
+							success: false,
+							error: "Project ID mismatch"
+						};
+						process.exit(1);
+					}
+				}
+				projectId = localProjectId;
+				s.stop(`Using local project: ${projectId}`);
+			} catch (error) {
+				s.stop("Failed to load local project");
+				throw new Error(`Could not load local project: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		} else {
+			if (!options.project) {
+				s.stop("No index.ts found in current directory");
+				console.error(chalk.yellow("Please run this command from a directory containing index.ts or use --project <project-id>"));
+				if (batchMode) return {
+					success: false,
+					error: "No index.ts found and no --project specified"
+				};
+				process.exit(1);
+			}
+			const projectPath = resolve(currentDir, options.project);
+			if (existsSync(join(projectPath, "index.ts"))) {
+				projectDir = projectPath;
+				s.start("Loading project from specified path...");
+				try {
+					localProjectForId = await loadProject(projectDir);
+					projectId = localProjectForId.getId();
+					s.stop(`Using project from path: ${projectId}`);
+				} catch (error) {
+					s.stop("Failed to load project from path");
+					throw new Error(`Could not load project from ${projectPath}: ${error instanceof Error ? error.message : String(error)}`);
+				}
+			} else {
+				projectId = options.project;
+				projectDir = join(currentDir, projectId);
+				s.stop(`Creating new project directory: ${projectDir}`);
+			}
+		}
+		s.start(`Fetching project: ${projectId}`);
+		const remoteProject = await (await ManagementApiClient.create(config.agentsManageApiUrl, options.config, config.tenantId, projectId, isCI, config.agentsManageApiKey)).getFullProject(projectId);
+		if (options.debug && remoteProject.functions) {
+			console.log(chalk.gray("   📋 Project-level functions from API:"), Object.keys(remoteProject.functions));
+			Object.entries(remoteProject.functions).forEach(([id, data]) => {
+				console.log(chalk.gray(`      ${id}: has name=${!!data.name}, has description=${!!data.description}`));
+			});
+		}
+		if (remoteProject.agents) for (const [agentId, agentData] of Object.entries(remoteProject.agents)) {
+			if (agentData.functionTools) {
+				remoteProject.functionTools = remoteProject.functionTools || {};
+				Object.assign(remoteProject.functionTools, agentData.functionTools);
+				if (options.debug) console.log(chalk.gray(`   Hoisted functionTools from agent ${agentId}: ${Object.keys(agentData.functionTools).join(", ")}`));
+			}
+			if (agentData.functions) {
+				remoteProject.functions = remoteProject.functions || {};
+				Object.entries(agentData.functions).forEach(([funcId, funcData]) => {
+					if (!remoteProject.functions[funcId]) remoteProject.functions[funcId] = {
+						id: funcData.id,
+						inputSchema: funcData.inputSchema,
+						executeCode: funcData.executeCode,
+						dependencies: funcData.dependencies
+					};
+				});
+			}
+		}
+		if (remoteProject.agents && remoteProject.tools) {
+			const projectToolIds = Object.keys(remoteProject.tools);
+			for (const [agentId, agentData] of Object.entries(remoteProject.agents)) if (agentData.tools) {
+				Object.keys(agentData.tools).length;
+				const agentSpecificTools = Object.fromEntries(Object.entries(agentData.tools).filter(([toolId]) => !projectToolIds.includes(toolId)));
+				if (Object.keys(agentSpecificTools).length > 0) agentData.tools = agentSpecificTools;
+				else delete agentData.tools;
+			}
+		}
+		enrichCanDelegateToWithTypes(remoteProject, options.debug);
+		s.message("Project data fetched");
+		if (options.json) {
+			console.log(JSON.stringify(remoteProject, null, 2));
+			restoreLogLevel();
+			return;
+		}
+		const paths = createProjectStructure(projectDir, projectId);
+		if (options.introspect) {
+			console.log(chalk.yellow("\n🔍 Introspect mode: Regenerating all files from scratch"));
+			s.start("Generating all files deterministically...");
+			await introspectGenerate(remoteProject, paths, options.env || "development", options.debug || false);
+			s.stop("All files generated");
+			console.log(chalk.green("\n✅ Project regenerated successfully with introspect mode!"));
+			console.log(chalk.gray(`   📁 Location: ${paths.projectRoot}`));
+			console.log(chalk.gray(`   🌍 Environment: ${options.env || "development"}`));
+			console.log(chalk.gray(`   🚀 Mode: Complete regeneration (no comparison)`));
+			restoreLogLevel();
+			if (batchMode) return { success: true };
+			process.exit(0);
+		}
+		const localProject = await readExistingProject(paths.projectRoot, options.debug);
+		if (!localProject) s.message("No existing project found - treating as new project");
+		else s.message("Existing project loaded");
+		s.start("Building component registry from local files...");
+		const { buildComponentRegistryFromParsing } = await import("./component-parser.js");
+		const localRegistry = buildComponentRegistryFromParsing(paths.projectRoot, options.debug);
+		s.message("Component registry built");
+		if (options.debug) {
+			console.log(chalk.cyan("\n🔍 Component Registry Debug:"));
+			const allComponents = localRegistry.getAllComponents();
+			console.log(chalk.gray("   Total components registered:"), allComponents.length);
+			const nameGroups = /* @__PURE__ */ new Map();
+			for (const comp of allComponents) {
+				if (!nameGroups.has(comp.name)) nameGroups.set(comp.name, []);
+				nameGroups.get(comp.name).push(comp);
+			}
+			for (const [varName, components] of nameGroups.entries()) if (components.length > 1) {
+				console.log(chalk.red(`   ❌ Variable name conflict: "${varName}"`));
+				for (const comp of components) console.log(chalk.gray(`      - ${comp.type}:${comp.id} -> ${comp.filePath}`));
+			} else console.log(chalk.gray(`   ✅ ${varName} (${components[0].type}:${components[0].id})`));
+		}
+		s.start("Comparing projects for changes...");
+		const comparison = await compareProjects(localProject, remoteProject, localRegistry, options.debug);
+		if (!comparison.hasChanges && !options.force) {
+			s.stop();
+			console.log(chalk.green("✅ Project is already up to date"));
+			console.log(chalk.gray("   No differences detected between local and remote projects"));
+			restoreLogLevel();
+			if (batchMode) return {
+				success: true,
+				upToDate: true
+			};
+			process.exit(0);
+		}
+		s.message(`Detected ${comparison.changeCount} differences`);
+		const tempDirName = `.temp-validation-${Date.now()}`;
+		s.start("Preparing temp directory...");
+		let performedCleanup = false;
+		if (localProject) {
+			copyProjectToTemp(paths.projectRoot, tempDirName);
+			console.log(chalk.green(`✅ Existing project copied to temp directory`));
+			s.start("Checking for stale components...");
+			const shouldCleanupStale = await checkAndPromptForStaleComponentCleanup(paths.projectRoot, remoteProject, localRegistry);
+			resetStdinState();
+			if (shouldCleanupStale) {
+				s.start("Cleaning up stale components from temp directory...");
+				await cleanupStaleComponents(paths.projectRoot, tempDirName, remoteProject, localRegistry);
+				console.log(chalk.green(`✅ Stale components cleaned up from temp directory`));
+				performedCleanup = true;
+			}
+		} else {
+			mkdirSync(join(paths.projectRoot, tempDirName), { recursive: true });
+			console.log(chalk.green(`✅ Empty temp directory created for new project`));
+		}
+		s.message("Temp directory prepared");
+		const newComponentCount = Object.values(comparison.componentChanges).reduce((sum, changes) => sum + changes.added.length, 0);
+		let newComponentResults = [];
+		if (newComponentCount > 0) {
+			s.start("Creating new component files in temp directory...");
+			const { createNewComponents } = await import("./new-component-generator.js");
+			newComponentResults = await createNewComponents(comparison, remoteProject, localRegistry, paths, options.env || "development", tempDirName);
+			if (options.debug) {
+				console.log(chalk.cyan("\n🔍 Component Registry After Generation:"));
+				const allComponents = localRegistry.getAllComponents();
+				console.log(chalk.gray("   Total components registered:"), allComponents.length);
+				const nameGroups = /* @__PURE__ */ new Map();
+				for (const comp of allComponents) {
+					if (!nameGroups.has(comp.name)) nameGroups.set(comp.name, []);
+					nameGroups.get(comp.name).push(comp);
+				}
+				for (const [varName, components] of nameGroups.entries()) if (components.length > 1) {
+					console.log(chalk.red(`   ❌ Variable name conflict: "${varName}"`));
+					for (const comp of components) console.log(chalk.gray(`      - ${comp.type}:${comp.id} -> ${comp.filePath}`));
+				} else console.log(chalk.gray(`   ✅ ${varName} (${components[0].type}:${components[0].id})`));
+			}
+			const successful = newComponentResults.filter((r) => r.success);
+			console.log(chalk.green(`✅ Added ${successful.length} new components to temp directory`));
+			s.message("New component files created");
+		}
+		const modifiedCount = Object.values(comparison.componentChanges).reduce((sum, changes) => sum + changes.modified.length, 0);
+		if (modifiedCount > 0) {
+			s.stop();
+			const { updateModifiedComponents } = await import("./component-updater.js");
+			const newComponentsForContext = newComponentResults && newComponentResults.length > 0 ? newComponentResults.filter((result) => result.success).map((result) => ({
+				componentId: result.componentId,
+				componentType: result.componentType,
+				filePath: result.filePath.replace(paths.projectRoot + "/", "")
+			})) : void 0;
+			await updateModifiedComponents(comparison, remoteProject, localRegistry, paths.projectRoot, options.env || "development", options.debug, tempDirName, newComponentsForContext);
+		}
+		s.start("Generating project index file in temp directory...");
+		const { generateProjectIndex } = await import("./project-index-generator.js");
+		await generateProjectIndex(join(paths.projectRoot, tempDirName), remoteProject, localRegistry, projectId);
+		s.message("Project index file created");
+		if (newComponentCount > 0 || modifiedCount > 0 || performedCleanup) {
+			s.stop("Running validation on complete project...");
+			const { validateTempDirectory } = await import("./project-validator.js");
+			const validationResult = await validateTempDirectory(paths.projectRoot, tempDirName, remoteProject, { skipExit: batchMode });
+			if (batchMode) {
+				restoreLogLevel();
+				return {
+					success: validationResult.success,
+					upToDate: validationResult.upToDate
+				};
+			}
+		} else {
+			s.stop();
+			console.log(chalk.green("\n✅ No changes detected - project is up to date"));
+		}
+		restoreLogLevel();
+		if (batchMode) return { success: true };
+		process.exit(0);
+	} catch (error) {
+		s.stop();
+		console.error(chalk.red(`\nError: ${error instanceof Error ? error.message : String(error)}`));
+		if (options.debug && error instanceof Error) console.error(chalk.red(error.stack || ""));
+		restoreLogLevel();
+		if (batchMode) return {
+			success: false,
+			error: error instanceof Error ? error.message : String(error)
+		};
+		process.exit(1);
+	}
+}
+/**
+* Pull all projects for the current tenant
+* Uses smart comparison with LLM merging for existing projects, introspect for new projects
+*/
+async function pullAllProjects(options) {
+	console.log(chalk.blue("\n🔄 Batch Pull: Sequential processing with smart comparison\n"));
+	console.log(chalk.gray("  • Existing projects: Smart comparison + LLM merging + confirmation prompts"));
+	console.log(chalk.gray("  • New projects: Fresh generation with introspect mode\n"));
+	performBackgroundVersionCheck();
+	const { config, profile, isCI } = await initializeCommand({
+		configPath: options.config,
+		profileName: options.profile,
+		tag: options.tag,
+		showSpinner: true,
+		spinnerText: "Loading configuration...",
+		logConfig: true,
+		quiet: options.quiet
+	});
+	const s = p.spinner();
+	try {
+		s.start("Fetching project list from API...");
+		const projects = await (await ManagementApiClient.create(config.agentsManageApiUrl, options.config, config.tenantId, void 0, isCI, config.agentsManageApiKey)).listAllProjects();
+		s.stop(`Found ${projects.length} project(s)`);
+		if (projects.length === 0) {
+			console.log(chalk.yellow("No projects found for this tenant."));
+			process.exit(0);
+		}
+		const existingProjects = [];
+		const newProjects = [];
+		for (const project of projects) if (existsSync(join(join(process.cwd(), project.id), "index.ts"))) existingProjects.push(project);
+		else newProjects.push(project);
+		console.log(chalk.gray("\nProjects to pull:\n"));
+		if (existingProjects.length > 0) {
+			console.log(chalk.cyan("  Existing (smart comparison):"));
+			for (const project of existingProjects) console.log(chalk.gray(`    • ${project.name || project.id} (${project.id})`));
+		}
+		if (newProjects.length > 0) {
+			console.log(chalk.cyan("  New (introspect):"));
+			for (const project of newProjects) console.log(chalk.gray(`    • ${project.name || project.id} (${project.id})`));
+		}
+		console.log();
+		const results = [];
+		const total = projects.length;
+		for (let i = 0; i < projects.length; i++) {
+			const project = projects[i];
+			const progress = `[${i + 1}/${total}]`;
+			console.log(chalk.cyan(`\n${"─".repeat(60)}`));
+			console.log(chalk.cyan(`${progress} Pulling ${project.name || project.id}...`));
+			const result = await pullSingleProject(project.id, project.name, options, config, isCI);
+			results.push(result);
+			if (result.success) console.log(chalk.green(`\n  ✓ ${result.projectName || result.projectId} → ${result.targetDir}`));
+			else console.log(chalk.red(`\n  ✗ ${result.projectName || result.projectId}: ${result.error}`));
+		}
+		const succeeded = results.filter((r) => r.success).length;
+		const failed = results.filter((r) => !r.success).length;
+		console.log(chalk.cyan(`\n${"═".repeat(60)}`));
+		console.log(chalk.cyan("📊 Batch Pull Summary:"));
+		console.log(chalk.green(`  ✓ Succeeded: ${succeeded}`));
+		if (failed > 0) {
+			console.log(chalk.red(`  ✗ Failed: ${failed}`));
+			console.log(chalk.red("\nFailed projects:"));
+			for (const result of results) if (!result.success) console.log(chalk.red(`  • ${result.projectId}: ${result.error}`));
+		}
+		process.exit(failed > 0 ? 1 : 0);
+	} catch (error) {
+		s.stop();
+		console.error(chalk.red(`\nError: ${error instanceof Error ? error.message : String(error)}`));
+		process.exit(1);
+	}
+}
+/**
+* Pull a single project (used by batch operations)
+* Uses smart comparison flow for existing projects, introspect for new projects
+*/
+async function pullSingleProject(projectId, projectName, options, config, isCI) {
+	const targetDir = join(process.cwd(), projectId);
+	const hasExistingProject = existsSync(join(targetDir, "index.ts"));
+	try {
+		if (hasExistingProject) {
+			console.log(chalk.gray(`   📂 Existing project found - using smart comparison mode`));
+			const originalDir = process.cwd();
+			process.chdir(targetDir);
+			try {
+				const result = await pullV3Command({
+					...options,
+					project: projectId,
+					all: false,
+					_batchMode: true
+				});
+				process.chdir(originalDir);
+				if (result && typeof result === "object") return {
+					projectId,
+					projectName,
+					targetDir,
+					success: result.success,
+					error: result.error
+				};
+				return {
+					projectId,
+					projectName,
+					targetDir,
+					success: true
+				};
+			} catch (error) {
+				process.chdir(originalDir);
+				throw error;
+			}
+		}
+		console.log(chalk.gray(`   🆕 New project - using introspect mode`));
+		const originalLogLevel = process.env.LOG_LEVEL;
+		process.env.LOG_LEVEL = "silent";
+		const restoreLogLevel = () => {
+			if (originalLogLevel !== void 0) process.env.LOG_LEVEL = originalLogLevel;
+			else delete process.env.LOG_LEVEL;
+		};
+		const remoteProject = await (await ManagementApiClient.create(config.agentsManageApiUrl, options.config, config.tenantId, projectId, isCI, config.agentsManageApiKey)).getFullProject(projectId);
+		await introspectGenerate(remoteProject, createProjectStructure(targetDir, projectId), options.env || "development", false);
+		restoreLogLevel();
+		return {
+			projectId,
+			projectName: projectName || remoteProject.name,
+			targetDir,
+			success: true
+		};
+	} catch (error) {
+		return {
+			projectId,
+			projectName,
+			targetDir,
+			success: false,
+			error: error instanceof Error ? error.message : String(error)
+		};
+	}
+}
+
+//#endregion
+export { enrichCanDelegateToWithTypes, pullV3Command };