@agentuity/cli 0.0.48 → 0.0.50

package/src/cmd/dev/index.ts

@@ -1,15 +1,15 @@
 /** biome-ignore-all lint/style/useTemplate: its easier */
 import { z } from 'zod';
-import type { BuildMetadata } from '@agentuity/server';
 import { resolve, join } from 'node:path';
 import { bundle } from '../bundle/bundler';
 import { getBuildMetadata } from '../bundle/plugin';
 import { existsSync, type FSWatcher, watch, statSync, readdirSync } from 'node:fs';
 import {
 	getDefaultConfigDir,
-	loadDevelopmentProjectSDKKey,
+	loadProjectSDKKey,
 	saveProjectDir,
 	saveConfig,
+	loadBuildMetadata,
 } from '../../config';
 import { type Config, createCommand } from '../../types';
 import * as tui from '../../tui';
@@ -17,10 +17,22 @@ import { createAgentTemplates, createAPITemplates } from './templates';
 import { generateEndpoint, type DevmodeResponse } from './api';
 import { APIClient, getAPIBaseURL } from '../../api';
 import { download } from './download';
+import { createDevmodeSyncService } from './sync';
+import { getDevmodeDeploymentId } from '../bundle/ast';
+import { BuildMetadata } from '@agentuity/server';
+import { getCommand } from '../../command-prefix';
 
 export const command = createCommand({
 	name: 'dev',
 	description: 'Build and run the development server',
+	tags: ['mutating', 'slow', 'requires-project'],
+	idempotent: true,
+	examples: [
+		getCommand('dev'),
+		getCommand('dev --port 8080'),
+		getCommand('dev --local'),
+		getCommand('dev --no-public'),
+	],
 	schema: {
 		options: z.object({
 			local: z.boolean().optional().describe('Turn on local services (instead of cloud)'),
@@ -43,6 +55,11 @@ export const command = createCommand({
 		const { opts, logger, options, project, projectDir, auth } = ctx;
 		let { config } = ctx;
 
+		// Allow sync with mock service even without devmode endpoint
+		const useMockService = process.env.DEVMODE_SYNC_SERVICE_MOCK === 'true';
+		const apiClient = new APIClient(getAPIBaseURL(config), logger, config);
+		const syncService = createDevmodeSyncService({ logger, apiClient, mock: useMockService });
+
 		const rootDir = projectDir;
 		const appTs = join(rootDir, 'app.ts');
 		const srcDir = join(rootDir, 'src');
@@ -69,12 +86,11 @@ export const command = createCommand({
 		let gravityBin: string | undefined;
 
 		if (auth && project && opts.public) {
-			// Only create apiClient if auth is available
-			const apiClient = new APIClient(getAPIBaseURL(config), logger, config);
+			// Generate devmode endpoint only when using --public
 			const endpoint = await tui.spinner({
 				message: 'Connecting to Gravity',
 				callback: () => {
-					return generateEndpoint(apiClient, project.projectId, config?.devmode?.hostname);
+					return generateEndpoint(apiClient!, project.projectId, config?.devmode?.hostname);
 				},
 				clearOnSuccess: true,
 			});
@@ -87,6 +103,13 @@ export const command = createCommand({
 			devmode = endpoint;
 		}
 
+		logger.debug(
+			'Getting devmode deployment id for projectId: %s, endpointId: %s',
+			project?.projectId,
+			devmode?.id
+		);
+		const deploymentId = getDevmodeDeploymentId(project?.projectId ?? '', devmode?.id ?? '');
+
 		if (devmode) {
 			const configDir = getDefaultConfigDir();
 			const gravityDir = join(configDir, 'gravity');
@@ -179,6 +202,21 @@ export const command = createCommand({
 		const agentuityDir = resolve(rootDir, '.agentuity');
 		const appPath = resolve(agentuityDir, 'app.js');
 
+		// Load existing metadata file to use as previousMetadata for sync
+		// This prevents reinserting agents/evals that haven't changed
+		let previousMetadata: BuildMetadata | undefined;
+		try {
+			previousMetadata = await loadBuildMetadata(agentuityDir);
+			logger.debug(
+				'Loaded previous metadata with %d agent(s)',
+				previousMetadata.agents?.length ?? 0
+			);
+		} catch (_error) {
+			// File doesn't exist yet (first run), that's okay
+			logger.debug('No previous metadata file found, will treat all agents/evals as new');
+			previousMetadata = undefined;
+		}
+
 		// Watch directories instead of files to survive atomic replacements (sed -i, cp)
 		const watches = [rootDir];
 		const watchers: FSWatcher[] = [];
@@ -200,7 +238,7 @@ export const command = createCommand({
 		let gravityClient: Bun.Subprocess | undefined;
 
 		if (gravityBin && devmode && project) {
-			const sdkKey = await loadDevelopmentProjectSDKKey(rootDir);
+			const sdkKey = await loadProjectSDKKey(rootDir);
 			if (!sdkKey) {
 				tui.warning(`Couldn't find the AGENTUITY_SDK_KEY in ${rootDir} .env file`);
 			} else {
@@ -390,6 +428,8 @@ export const command = createCommand({
 							await bundle({
 								rootDir,
 								dev: true,
+								projectId: project?.projectId,
+								deploymentId,
 							});
 							building = false;
 							buildCompletedAt = Date.now();
@@ -419,6 +459,64 @@ export const command = createCommand({
 				metadata = getBuildMetadata();
 				logger.trace('Build metadata retrieved');
 
+				// Sync agents and evals to API if in devmode with auth
+				if (auth && project && apiClient) {
+					try {
+						logger.debug('Loading build metadata for sync...');
+						const currentMetadata = await loadBuildMetadata(agentuityDir);
+						logger.debug(
+							'Found %d agent(s) and %d route(s) in metadata',
+							currentMetadata.agents?.length ?? 0,
+							currentMetadata.routes?.length ?? 0
+						);
+						if (currentMetadata.agents) {
+							for (const agent of currentMetadata.agents) {
+								logger.debug(
+									'Agent: id=%s, name=%s, version=%s, evals=%d',
+									agent.id,
+									agent.name,
+									agent.version,
+									agent.evals?.length ?? 0
+								);
+								if (agent.evals) {
+									for (const evalItem of agent.evals) {
+										logger.debug(
+											'  Eval: id=%s, name=%s, version=%s',
+											evalItem.id,
+											evalItem.name,
+											evalItem.version
+										);
+									}
+								}
+							}
+						}
+						logger.debug('Syncing agents and evals...');
+
+						await syncService.sync(
+							currentMetadata,
+							previousMetadata,
+							project.projectId,
+							deploymentId
+						);
+						previousMetadata = currentMetadata;
+						logger.debug('Sync completed successfully');
+					} catch (error) {
+						logger.error('Failed to sync agents/evals: %s', error);
+						if (error instanceof Error) {
+							logger.error('Error stack: %s', error.stack);
+						}
+						// Don't fail the build, just log the error
+					}
+				} else {
+					logger.trace(
+						'Skipping sync - auth=%s, project=%s, devmode=%s, apiClient=%s',
+						!!auth,
+						!!project,
+						!!devmode,
+						!!apiClient
+					);
+				}
+
 				logger.trace('Starting dev server: %s', appPath);
 				// Use shell to run in a process group for proper cleanup
 				// The 'exec' ensures the shell is replaced by the actual process
@@ -554,12 +652,12 @@ export const command = createCommand({
 
 			const showRoutes = () => {
 				tui.info('API Route Detail');
-				console.table(metadata?.routes, ['method', 'path', 'filename']);
+				tui.table(metadata?.routes ?? [], ['method', 'path', 'filename']);
 			};
 
 			const showAgents = () => {
 				tui.info('Agent Detail');
-				console.table(metadata?.agents, ['name', 'filename', 'description']);
+				tui.table(metadata?.agents ?? [], ['name', 'filename', 'description']);
 			};
 
 			process.stdin.on('data', (data) => {

package/src/cmd/dev/sync.ts

@@ -0,0 +1,414 @@
+import { z } from 'zod';
+import type { Logger, BuildMetadata } from '../../types';
+import type { APIClient } from '../../api';
+
+interface AgentSyncPayload {
+	id: string;
+	name: string;
+	identifier: string;
+	agentId: string;
+	description?: string;
+	version: string;
+	filename: string;
+	projectId: string;
+	subagents?: Array<{
+		id: string;
+		name: string;
+		identifier: string;
+		agentId: string;
+		description?: string;
+		version: string;
+		filename: string;
+		projectId: string;
+	}>;
+}
+
+interface EvalSyncPayload {
+	id: string;
+	name: string;
+	identifier: string;
+	evalId: string;
+	description?: string;
+	version: string;
+	filename: string;
+	projectId: string;
+	agentIdentifier: string;
+}
+
+interface IDevmodeSyncService {
+	sync(
+		currentMetadata: BuildMetadata,
+		previousMetadata: BuildMetadata | undefined,
+		projectId: string,
+		deploymentId: string
+	): Promise<void>;
+}
+
+// Shared diff logic for agents
+function getAgentsToSync(
+	currentAgents: BuildMetadata['agents'],
+	previousAgentIds: Set<string>,
+	projectId: string,
+	logger: Logger
+): { create: AgentSyncPayload[]; delete: string[] } {
+	const agentsToCreate: AgentSyncPayload[] = [];
+	const currentAgentIds = new Set<string>();
+
+	for (const agent of currentAgents || []) {
+		currentAgentIds.add(agent.id);
+		// If ID is not in previous, add to create
+		if (!previousAgentIds.has(agent.id)) {
+			logger.debug(
+				'[CLI AGENT SYNC] Preparing to create: id="%s", name="%s"',
+				agent.id,
+				agent.name
+			);
+
+			// Add projectId to subagents if they exist
+			const subagents = agent.subagents?.map((subagent) => {
+				return {
+					id: subagent.id,
+					name: subagent.name,
+					identifier: subagent.identifier,
+					agentId: subagent.agentId,
+					description: subagent.description,
+					version: subagent.version,
+					filename: subagent.filename,
+					projectId,
+				};
+			});
+
+			agentsToCreate.push({
+				id: agent.id,
+				name: agent.name,
+				identifier: agent.identifier,
+				agentId: agent.agentId,
+				description: agent.description,
+				version: agent.version,
+				filename: agent.filename,
+				projectId,
+				...(subagents && { subagents }),
+			});
+		}
+	}
+
+	// If ID is in previous but not in current, add to delete
+	const agentsToDelete: string[] = [];
+	for (const previousId of previousAgentIds) {
+		if (!currentAgentIds.has(previousId)) {
+			logger.debug('[CLI AGENT SYNC] Preparing to delete: id="%s"', previousId);
+			agentsToDelete.push(previousId);
+		}
+	}
+
+	return { create: agentsToCreate, delete: agentsToDelete };
+}
+
+// Shared diff logic for evals
+function getEvalsToSync(
+	currentMetadata: BuildMetadata,
+	previousEvalIds: Set<string>,
+	projectId: string,
+	logger: Logger
+): { create: EvalSyncPayload[]; delete: string[] } {
+	const evalsToCreate: EvalSyncPayload[] = [];
+	const currentEvalIds = new Set<string>();
+
+	for (const agent of currentMetadata.agents || []) {
+		if (agent.evals) {
+			for (const evalItem of agent.evals) {
+				currentEvalIds.add(evalItem.id);
+				// If ID is not in previous, add to create
+				if (!previousEvalIds.has(evalItem.id)) {
+					logger.debug(
+						'[CLI EVAL SYNC] Preparing to create: id="%s", name="%s"',
+						evalItem.id,
+						evalItem.name
+					);
+
+					evalsToCreate.push({
+						...evalItem,
+						evalId: evalItem.evalId,
+						projectId,
+						agentIdentifier: agent.agentId,
+					});
+				}
+			}
+		}
+	}
+
+	// If ID is in previous but not in current, add to delete
+	const evalsToDelete: string[] = [];
+	for (const previousId of previousEvalIds) {
+		if (!currentEvalIds.has(previousId)) {
+			logger.debug('[CLI EVAL SYNC] Preparing to delete: id="%s"', previousId);
+			evalsToDelete.push(previousId);
+		}
+	}
+
+	return { create: evalsToCreate, delete: evalsToDelete };
+}
+
+class DevmodeSyncService implements IDevmodeSyncService {
+	constructor(
+		private logger: Logger,
+		private apiClient: APIClient
+	) {}
+
+	async sync(
+		currentMetadata: BuildMetadata,
+		previousMetadata: BuildMetadata | undefined,
+		projectId: string,
+		deploymentId: string
+	): Promise<void> {
+		// Build previous agent IDs set
+		const previousAgentIds = new Set<string>();
+		if (previousMetadata) {
+			this.logger.debug(
+				'Previous metadata found with %d agent(s)',
+				previousMetadata.agents?.length ?? 0
+			);
+			for (const agent of previousMetadata.agents || []) {
+				previousAgentIds.add(agent.id);
+			}
+		} else {
+			this.logger.debug('No previous metadata, all agents will be treated as new');
+		}
+
+		// Build previous eval IDs set
+		const previousEvalIds = new Set<string>();
+		if (previousMetadata) {
+			let prevEvalCount = 0;
+			for (const agent of previousMetadata.agents || []) {
+				if (agent.evals) {
+					for (const evalItem of agent.evals) {
+						previousEvalIds.add(evalItem.id);
+						prevEvalCount++;
+					}
+				}
+			}
+			this.logger.debug('Previous metadata found with %d eval(s)', prevEvalCount);
+		} else {
+			this.logger.debug('No previous metadata, all evals will be treated as new');
+		}
+
+		const currentAgents = currentMetadata.agents || [];
+		this.logger.debug('Processing %d current agent(s)', currentAgents.length);
+
+		let currentEvalCount = 0;
+		for (const agent of currentMetadata.agents || []) {
+			if (agent.evals) {
+				currentEvalCount += agent.evals.length;
+			}
+		}
+		this.logger.debug('Processing %d current eval(s)', currentEvalCount);
+
+		// Get agents and evals to sync using shared diff logic
+		const { create: agentsToCreate, delete: agentsToDelete } = getAgentsToSync(
+			currentAgents,
+			previousAgentIds,
+			projectId,
+			this.logger
+		);
+		const { create: evalsToCreate, delete: evalsToDelete } = getEvalsToSync(
+			currentMetadata,
+			previousEvalIds,
+			projectId,
+			this.logger
+		);
+
+		if (agentsToCreate.length > 0 || agentsToDelete.length > 0) {
+			this.logger.debug(
+				'Bulk syncing %d agent(s) to create, %d agent(s) to delete',
+				agentsToCreate.length,
+				agentsToDelete.length
+			);
+		}
+		if (evalsToCreate.length > 0 || evalsToDelete.length > 0) {
+			this.logger.debug(
+				'Bulk syncing %d eval(s) to create, %d eval(s) to delete',
+				evalsToCreate.length,
+				evalsToDelete.length
+			);
+		}
+
+		// Sync both in parallel
+		try {
+			await Promise.all([
+				this.syncAgents(agentsToCreate, agentsToDelete, deploymentId),
+				this.syncEvals(evalsToCreate, evalsToDelete, deploymentId),
+			]);
+
+			if (agentsToCreate.length > 0 || agentsToDelete.length > 0) {
+				this.logger.debug(
+					'Successfully bulk synced %d agent(s) to create, %d agent(s) to delete',
+					agentsToCreate.length,
+					agentsToDelete.length
+				);
+			}
+			if (evalsToCreate.length > 0 || evalsToDelete.length > 0) {
+				this.logger.debug(
+					'Successfully bulk synced %d eval(s) to create, %d eval(s) to delete',
+					evalsToCreate.length,
+					evalsToDelete.length
+				);
+			}
+		} catch (error) {
+			this.logger.error('Failed to bulk sync agents/evals: %s', error);
+			if (error instanceof Error) {
+				this.logger.error('Error details: %s', error.message);
+			}
+			throw error;
+		}
+	}
+
+	private async syncAgents(
+		agents: AgentSyncPayload[],
+		agentsToDelete: string[],
+		deploymentId: string
+	): Promise<void> {
+		if (agents.length === 0 && agentsToDelete.length === 0) {
+			return;
+		}
+
+		const payload = {
+			create: agents,
+			delete: agentsToDelete,
+			deploymentId,
+		};
+		this.logger.trace(
+			'[CLI AGENT SYNC] Sending payload to POST /cli/devmode/agent: %s',
+			JSON.stringify(payload, null, 2)
+		);
+
+		await this.apiClient.request(
+			'POST',
+			'/cli/devmode/agent',
+			z.object({ success: z.boolean() }),
+			payload
+		);
+	}
+
+	private async syncEvals(
+		evals: EvalSyncPayload[],
+		evalsToDelete: string[],
+		deploymentId: string
+	): Promise<void> {
+		if (evals.length === 0 && evalsToDelete.length === 0) {
+			return;
+		}
+
+		const payload = {
+			deploymentId,
+			create: evals,
+			delete: evalsToDelete,
+		};
+		this.logger.trace(
+			'[CLI EVAL SYNC] Sending payload to POST /cli/devmode/eval: %s',
+			JSON.stringify(payload, null, 2)
+		);
+
+		await this.apiClient.request(
+			'POST',
+			'/cli/devmode/eval',
+			z.object({ success: z.boolean() }),
+			payload
+		);
+	}
+}
+
+class MockDevmodeSyncService implements IDevmodeSyncService {
+	constructor(private logger: Logger) {}
+
+	async sync(
+		currentMetadata: BuildMetadata,
+		previousMetadata: BuildMetadata | undefined,
+		projectId: string,
+		deploymentId: string
+	): Promise<void> {
+		// Build previous agent IDs set
+		this.logger.debug('Mock syncing agents and evals for deploymentId: %s', deploymentId);
+		const previousAgentIds = new Set<string>();
+		if (previousMetadata) {
+			for (const agent of previousMetadata.agents || []) {
+				previousAgentIds.add(agent.id);
+			}
+		}
+
+		// Build previous eval IDs set
+		const previousEvalIds = new Set<string>();
+		if (previousMetadata) {
+			for (const agent of previousMetadata.agents || []) {
+				if (agent.evals) {
+					for (const evalItem of agent.evals) {
+						previousEvalIds.add(evalItem.id);
+					}
+				}
+			}
+		}
+
+		// Get agents and evals to sync using shared diff logic
+		const { create: agentsToCreate, delete: agentsToDelete } = getAgentsToSync(
+			currentMetadata.agents,
+			previousAgentIds,
+			projectId,
+			this.logger
+		);
+		const { create: evalsToCreate, delete: evalsToDelete } = getEvalsToSync(
+			currentMetadata,
+			previousEvalIds,
+			projectId,
+			this.logger
+		);
+
+		// Log the requests that would be made
+		if (agentsToCreate.length > 0 || agentsToDelete.length > 0) {
+			this.logger.info(
+				'[MOCK] Would make request: POST /cli/devmode/agent with %d agent(s) to create, %d agent(s) to delete',
+				agentsToCreate.length,
+				agentsToDelete.length
+			);
+			this.logger.info(
+				'[MOCK] Request payload: %s',
+				JSON.stringify({ create: agentsToCreate, delete: agentsToDelete }, null, 2)
+			);
+		}
+
+		if (evalsToCreate.length > 0 || evalsToDelete.length > 0) {
+			this.logger.info(
+				'[MOCK] Would make request: POST /cli/devmode/eval with %d eval(s) to create, %d eval(s) to delete',
+				evalsToCreate.length,
+				evalsToDelete.length
+			);
+			this.logger.info(
+				'[MOCK] Request payload: %s',
+				JSON.stringify({ create: evalsToCreate, delete: evalsToDelete }, null, 2)
+			);
+		}
+
+		if (
+			agentsToCreate.length === 0 &&
+			agentsToDelete.length === 0 &&
+			evalsToCreate.length === 0 &&
+			evalsToDelete.length === 0
+		) {
+			this.logger.info('[MOCK] No requests would be made (no changes detected)');
+		}
+	}
+}
+
+export function createDevmodeSyncService({
+	logger,
+	apiClient,
+	mock = false,
+}: {
+	logger: Logger;
+	apiClient: APIClient;
+	mock?: boolean;
+}): IDevmodeSyncService {
+	if (mock) {
+		return new MockDevmodeSyncService(logger);
+	}
+
+	return new DevmodeSyncService(logger, apiClient);
+}

package/src/cmd/dev/templates.ts

@@ -8,7 +8,7 @@ import { z } from 'zod';
 const agent = createAgent({
 	metadata: {
         name: '${name}',
-        description: 'Add my agent description here',
+        description: 'Add your agent description here',
     },
 	schema: {
 		input: z.string(),

package/src/cmd/env/delete.ts

@@ -8,16 +8,27 @@ import {
 	writeEnvFile,
 	filterAgentuitySdkKeys,
 } from '../../env-util';
+import { getCommand } from '../../command-prefix';
+
+const EnvDeleteResponseSchema = z.object({
+	success: z.boolean().describe('Whether the operation succeeded'),
+	key: z.string().describe('Environment variable key that was deleted'),
+	path: z.string().describe('Local file path where env var was removed'),
+});
 
 export const deleteSubcommand = createSubcommand({
 	name: 'delete',
 	aliases: ['del', 'remove', 'rm'],
 	description: 'Delete an environment variable',
+	tags: ['destructive', 'deletes-resource', 'slow', 'requires-auth', 'requires-project'],
+	idempotent: true,
+	examples: [getCommand('env delete OLD_FEATURE_FLAG'), getCommand('env rm PORT')],
 	requires: { auth: true, project: true, apiClient: true },
 	schema: {
 		args: z.object({
 			key: z.string().describe('the environment variable key to delete'),
 		}),
+		response: EnvDeleteResponseSchema,
 	},
 
 	async handler(ctx) {
@@ -43,5 +54,11 @@ export const deleteSubcommand = createSubcommand({
 		tui.success(
 			`Environment variable '${args.key}' deleted successfully (cloud + ${envFilePath})`
 		);
+
+		return {
+			success: true,
+			key: args.key,
+			path: envFilePath,
+		};
 	},
 });

package/src/cmd/env/get.ts

@@ -3,10 +3,19 @@ import { createSubcommand } from '../../types';
 import * as tui from '../../tui';
 import { projectGet } from '@agentuity/server';
 import { maskSecret } from '../../env-util';
+import { getCommand } from '../../command-prefix';
+import { ErrorCode } from '../../errors';
+
+const EnvGetResponseSchema = z.object({
+	key: z.string().describe('Environment variable key name'),
+	value: z.string().describe('Environment variable value'),
+});
 
 export const getSubcommand = createSubcommand({
 	name: 'get',
 	description: 'Get an environment variable value',
+	tags: ['read-only', 'fast', 'requires-auth', 'requires-project'],
+	examples: [getCommand('env get NODE_ENV'), getCommand('env get LOG_LEVEL')],
 	requires: { auth: true, project: true, apiClient: true },
 	schema: {
 		args: z.object({
@@ -18,7 +27,9 @@ export const getSubcommand = createSubcommand({
 				.default(false)
 				.describe('mask the value in output (default: true in TTY, false otherwise)'),
 		}),
+		response: EnvGetResponseSchema,
 	},
+	idempotent: true,
 
 	async handler(ctx) {
 		const { args, opts, apiClient, project } = ctx;
@@ -32,7 +43,7 @@ export const getSubcommand = createSubcommand({
 		const value = projectData.env?.[args.key];
 
 		if (value === undefined) {
-			tui.fatal(`Environment variable '${args.key}' not found`);
+			tui.fatal(`Environment variable '${args.key}' not found`, ErrorCode.RESOURCE_NOT_FOUND);
 		}
 
 		// Display the value, masked if requested
@@ -49,5 +60,10 @@ export const getSubcommand = createSubcommand({
 				console.log(`${args.key}=${value}`);
 			}
 		}
+
+		return {
+			key: args.key,
+			value,
+		};
 	},
 });