@ottocode/sdk 0.1.287 → 0.1.288

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ottocode/sdk",
-	"version": "0.1.287",
+	"version": "0.1.288",
 	"description": "AI agent SDK for building intelligent assistants - tree-shakable and comprehensive",
 	"author": "nitishxyz",
 	"license": "MIT",

package/src/core/src/index.ts

@@ -52,6 +52,11 @@ export type {
 export { buildFsTools } from './tools/builtin/fs/index';
 export { buildGitTools } from './tools/builtin/git';
 export { buildTerminalTool } from './tools/builtin/terminal';
+export {
+	buildLazyToolsRecord,
+	buildLoadFirstPartyToolsTool,
+	buildSimulatorTool,
+} from './tools/lazy/index';
 
 // =======================
 // Terminals

package/src/core/src/tools/builtin/fs/index.ts

@@ -1,11 +1,9 @@
 import type { Tool } from 'ai';
 import { buildEditTool } from './edit.ts';
 import { buildReadTool } from './read.ts';
-import { buildReadImageTool } from './read-image.ts';
 import { buildMultiEditTool } from './multiedit.ts';
 import { buildWriteTool } from './write.ts';
 import { buildCopyIntoTool } from './copy-into.ts';
-import { buildCopyAttachmentTool } from './copy-attachment.ts';
 import { buildLsTool } from './ls.ts';
 import { buildTreeTool } from './tree.ts';
 import { buildPwdTool } from './pwd.ts';
@@ -16,12 +14,10 @@ export function buildFsTools(
 ): Array<{ name: string; tool: Tool }> {
 	const out: Array<{ name: string; tool: Tool }> = [];
 	out.push(buildReadTool(projectRoot));
-	out.push(buildReadImageTool(projectRoot));
 	out.push(buildEditTool(projectRoot));
 	out.push(buildMultiEditTool(projectRoot));
 	out.push(buildWriteTool(projectRoot));
 	out.push(buildCopyIntoTool(projectRoot));
-	out.push(buildCopyAttachmentTool(projectRoot));
 	out.push(buildLsTool(projectRoot));
 	out.push(buildTreeTool(projectRoot));
 	out.push(buildPwdTool());

package/src/core/src/tools/builtin/patch.txt

@@ -58,6 +58,17 @@ new block after line 200
 Line-number mode is concise but fragile if the file changes after you read it.
 Prefer text/context patches for small edits.
 
+When deleting a contiguous block larger than 10 lines, prefer line-number deletion:
+
+```text
+*** Begin Patch
+*** Delete Lines in: path/to/file.ts
+*** Lines: start-end
+*** End Patch
+```
+
+Do not reproduce the whole deleted block unless line numbers are unavailable or uncertain.
+
 ---
 
 ## Standard mode: Add / Update / Delete

package/src/core/src/tools/lazy/index.ts

@@ -0,0 +1,12 @@
+export {
+	buildLazyToolCatalogDescription,
+	buildLoadToolsTool,
+	type LazyToolBrief,
+} from './load-tools.ts';
+export {
+	buildLazyToolsRecord,
+	buildLoadFirstPartyToolsTool,
+	getLazyToolDefinitions,
+	type LazyToolDefinition,
+} from './registry.ts';
+export { buildSimulatorTool } from './simulator.ts';

package/src/core/src/tools/lazy/load-tools.ts

@@ -0,0 +1,56 @@
+import { tool, type Tool } from 'ai';
+import { z } from 'zod/v3';
+
+export type LazyToolBrief = {
+	name: string;
+	description: string;
+};
+
+export function buildLazyToolCatalogDescription(
+	briefs: LazyToolBrief[],
+): string {
+	if (briefs.length === 0) return 'No lazy tools available.';
+	return briefs
+		.map((brief) => `- ${brief.name}: ${brief.description.slice(0, 180)}`)
+		.join('\n');
+}
+
+export function buildLoadToolsTool(briefs: LazyToolBrief[]): {
+	name: string;
+	tool: Tool;
+} {
+	const catalog = buildLazyToolCatalogDescription(briefs);
+	const validNames = new Set(briefs.map((brief) => brief.name));
+
+	return {
+		name: 'load_tools',
+		tool: tool({
+			description: `Load first-party tools by name so they become available for use in the next step. Call this before using a listed tool.\n\nAvailable tools:\n${catalog}`,
+			inputSchema: z.object({
+				tools: z
+					.array(z.string())
+					.describe('Array of first-party tool names to load.'),
+			}),
+			execute: async ({ tools: requested }) => {
+				const loaded: string[] = [];
+				const notFound: string[] = [];
+				for (const name of requested) {
+					if (validNames.has(name)) {
+						loaded.push(name);
+					} else {
+						notFound.push(name);
+					}
+				}
+				return {
+					ok: true,
+					loaded,
+					...(notFound.length > 0 ? { notFound } : {}),
+					message:
+						loaded.length > 0
+							? `Loaded ${loaded.length} tool(s). They are now available for use.`
+							: 'No valid tools to load.',
+				};
+			},
+		}),
+	};
+}

package/src/core/src/tools/lazy/registry.ts

@@ -0,0 +1,51 @@
+import type { Tool } from 'ai';
+import { buildCopyAttachmentTool } from '../builtin/fs/copy-attachment.ts';
+import { buildReadImageTool } from '../builtin/fs/read-image.ts';
+import { buildSimulatorTool } from './simulator.ts';
+import { buildLoadToolsTool, type LazyToolBrief } from './load-tools.ts';
+
+export type LazyToolDefinition = LazyToolBrief & {
+	build: (projectRoot: string) => { name: string; tool: Tool };
+};
+
+export function getLazyToolDefinitions(): LazyToolDefinition[] {
+	return [
+		{
+			name: 'simulator',
+			description:
+				'Control Apple Simulator via serve-sim: start, status, click, type, button, rotate, screenshot, accessibility tree, foreground app, logs, stop.',
+			build: buildSimulatorTool,
+		},
+		{
+			name: 'read_image',
+			description:
+				'Read and inspect a local image file such as a screenshot, icon, or diagram.',
+			build: buildReadImageTool,
+		},
+		{
+			name: 'copy_attachment_to_project',
+			description:
+				'Copy an original uploaded chat attachment into the project without recompression.',
+			build: buildCopyAttachmentTool,
+		},
+	];
+}
+
+export function buildLazyToolsRecord(
+	projectRoot: string,
+): Record<string, Tool> {
+	const record: Record<string, Tool> = {};
+	for (const definition of getLazyToolDefinitions()) {
+		const built = definition.build(projectRoot);
+		record[built.name] = built.tool;
+	}
+	return record;
+}
+
+export function buildLoadFirstPartyToolsTool(): { name: string; tool: Tool } {
+	const briefs = getLazyToolDefinitions().map(({ name, description }) => ({
+		name,
+		description,
+	}));
+	return buildLoadToolsTool(briefs);
+}

package/src/core/src/tools/lazy/simulator.ts

@@ -0,0 +1,1039 @@
+import { spawn, type ChildProcess } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { basename, extname, join } from 'node:path';
+import { tool, type Tool } from 'ai';
+import { z } from 'zod/v3';
+import { createToolError } from '../error.ts';
+
+const DEFAULT_TIMEOUT_MS = 30_000;
+const FETCH_TIMEOUT_MS = 5_000;
+const LOG_TIMEOUT_MS = 2_000;
+const DEFAULT_PREVIEW_PORT = 3200;
+const DEFAULT_PREVIEW_URL = `http://localhost:${DEFAULT_PREVIEW_PORT}`;
+const SCREENSHOT_ARTIFACTS_DIR = '.otto/artifacts/simulator';
+const SCREENSHOT_MODEL_MAX_EDGE = 1024;
+const SCREENSHOT_MODEL_JPEG_QUALITY = 70;
+const HID_KEYBOARD_LEFT_GUI = 227;
+const HID_KEYBOARD_V = 25;
+
+let previewProcess: ChildProcess | null = null;
+let previewStdout = '';
+let previewStderr = '';
+
+const buttonNames = [
+	'home',
+	'swipe_home',
+	'app_switcher',
+	'lock',
+	'siri',
+	'side_button',
+] as const;
+
+const orientations = [
+	'portrait',
+	'portrait_upside_down',
+	'landscape_left',
+	'landscape_right',
+] as const;
+
+const simulatorActions = [
+	'start',
+	'status',
+	'stop',
+	'click',
+	'type',
+	'paste',
+	'button',
+	'rotate',
+	'config',
+	'accessibility_tree',
+	'foreground',
+	'take_screenshot',
+	'launch',
+	'terminate',
+	'open_url',
+	'list_apps',
+	'logs',
+] as const;
+
+type ServeSimEntry = {
+	device?: string;
+	pid?: number;
+	port?: number;
+	url?: string;
+	streamUrl?: string;
+	wsUrl?: string;
+};
+
+type ExecResult = {
+	exitCode: number;
+	stdout: string;
+	stderr: string;
+};
+
+type BunImageMetadata = {
+	width?: number;
+	height?: number;
+	format?: string;
+};
+
+type BunImagePipeline = {
+	metadata(): Promise<BunImageMetadata>;
+	resize(
+		width: number,
+		height?: number,
+		options?: {
+			fit?: 'inside';
+			withoutEnlargement?: boolean;
+		},
+	): BunImagePipeline;
+	jpeg(options?: { quality?: number }): BunImagePipeline;
+	bytes(): Promise<Uint8Array>;
+};
+
+type BunImageConstructor = new (
+	input: string | ArrayBuffer | Uint8Array | Blob,
+) => BunImagePipeline;
+
+type JsonValue =
+	| null
+	| boolean
+	| number
+	| string
+	| JsonValue[]
+	| { [key: string]: JsonValue };
+
+const simulatorInputSchema = z.object({
+	action: z.enum(simulatorActions),
+	device: z.string().optional(),
+	x: z.number().min(0).max(1).optional(),
+	y: z.number().min(0).max(1).optional(),
+	text: z.string().optional(),
+	name: z.enum(buttonNames).optional(),
+	orientation: z.enum(orientations).optional(),
+	bundleId: z.string().optional(),
+	url: z.string().optional(),
+	args: z.array(z.string()).optional(),
+	outputPath: z.string().optional(),
+	timeoutMs: z.number().min(250).max(10_000).optional(),
+});
+
+type SimulatorInput =
+	| { action: 'start'; device?: string }
+	| { action: 'status'; device?: string }
+	| { action: 'stop'; device?: string }
+	| { action: 'click'; x: number; y: number; device?: string }
+	| { action: 'type'; text: string; device?: string }
+	| { action: 'paste'; text: string; device?: string }
+	| { action: 'button'; name: (typeof buttonNames)[number]; device?: string }
+	| {
+			action: 'rotate';
+			orientation: (typeof orientations)[number];
+			device?: string;
+	  }
+	| { action: 'config'; device?: string }
+	| { action: 'accessibility_tree'; device?: string }
+	| { action: 'foreground'; device?: string }
+	| { action: 'take_screenshot'; device?: string; outputPath?: string }
+	| { action: 'launch'; device?: string; bundleId: string; args?: string[] }
+	| { action: 'terminate'; device?: string; bundleId: string }
+	| { action: 'open_url'; device?: string; url: string }
+	| { action: 'list_apps'; device?: string; text?: string }
+	| { action: 'logs'; device?: string; timeoutMs?: number };
+
+function withDevice(args: string[], device?: string): string[] {
+	return device ? [...args, '-d', device] : args;
+}
+
+function getBunImageConstructor(): BunImageConstructor | undefined {
+	return (Bun as typeof Bun & { Image?: BunImageConstructor }).Image;
+}
+
+function toJsonValue(value: unknown): JsonValue {
+	if (value === undefined) return null;
+	try {
+		return JSON.parse(JSON.stringify(value)) as JsonValue;
+	} catch {
+		return String(value);
+	}
+}
+
+function requireNumber(value: number | undefined, name: string): number {
+	if (typeof value === 'number') return value;
+	throw new Error(`Missing required number field: ${name}`);
+}
+
+function requireString(value: string | undefined, name: string): string {
+	if (typeof value === 'string') return value;
+	throw new Error(`Missing required string field: ${name}`);
+}
+
+function parseSimulatorInput(
+	input: z.infer<typeof simulatorInputSchema>,
+): SimulatorInput {
+	switch (input.action) {
+		case 'click':
+			return {
+				action: 'click',
+				x: requireNumber(input.x, 'x'),
+				y: requireNumber(input.y, 'y'),
+				device: input.device,
+			};
+		case 'type':
+		case 'paste':
+			return {
+				action: input.action,
+				text: requireString(input.text, 'text'),
+				device: input.device,
+			};
+		case 'button':
+			return {
+				action: 'button',
+				name: input.name ?? 'home',
+				device: input.device,
+			};
+		case 'rotate':
+			return {
+				action: 'rotate',
+				orientation: input.orientation ?? 'portrait',
+				device: input.device,
+			};
+		case 'take_screenshot':
+			return {
+				action: 'take_screenshot',
+				device: input.device,
+				outputPath: input.outputPath,
+			};
+		case 'launch':
+			return {
+				action: 'launch',
+				device: input.device,
+				bundleId: requireString(input.bundleId, 'bundleId'),
+				args: input.args,
+			};
+		case 'terminate':
+			return {
+				action: 'terminate',
+				device: input.device,
+				bundleId: requireString(input.bundleId, 'bundleId'),
+			};
+		case 'open_url':
+			return {
+				action: 'open_url',
+				device: input.device,
+				url: requireString(input.url, 'url'),
+			};
+		case 'list_apps':
+			return {
+				action: 'list_apps',
+				device: input.device,
+				text: input.text,
+			};
+		case 'logs':
+			return {
+				action: 'logs',
+				device: input.device,
+				timeoutMs: input.timeoutMs,
+			};
+		default:
+			return { action: input.action, device: input.device } as SimulatorInput;
+	}
+}
+
+async function execServeSim(args: string[]): Promise<ExecResult> {
+	return new Promise((resolve, reject) => {
+		const child = spawn('npx', ['--yes', 'serve-sim', ...args], {
+			stdio: ['ignore', 'pipe', 'pipe'],
+		});
+		let stdout = '';
+		let stderr = '';
+		const timeout = setTimeout(() => {
+			child.kill('SIGTERM');
+			reject(new Error(`serve-sim timed out after ${DEFAULT_TIMEOUT_MS}ms`));
+		}, DEFAULT_TIMEOUT_MS);
+		child.stdout.setEncoding('utf8');
+		child.stderr.setEncoding('utf8');
+		child.stdout.on('data', (chunk) => {
+			stdout += chunk;
+		});
+		child.stderr.on('data', (chunk) => {
+			stderr += chunk;
+		});
+		child.on('error', (error) => {
+			clearTimeout(timeout);
+			reject(error);
+		});
+		child.on('close', (exitCode) => {
+			clearTimeout(timeout);
+			resolve({ exitCode: exitCode ?? 0, stdout, stderr });
+		});
+	});
+}
+
+async function runCommand(
+	command: string,
+	args: string[],
+): Promise<ExecResult> {
+	return new Promise((resolve, reject) => {
+		const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+		let stdout = '';
+		let stderr = '';
+		const timeout = setTimeout(() => {
+			child.kill('SIGTERM');
+			reject(new Error(`${command} timed out after ${DEFAULT_TIMEOUT_MS}ms`));
+		}, DEFAULT_TIMEOUT_MS);
+		child.stdout.setEncoding('utf8');
+		child.stderr.setEncoding('utf8');
+		child.stdout.on('data', (chunk) => {
+			stdout += chunk;
+		});
+		child.stderr.on('data', (chunk) => {
+			stderr += chunk;
+		});
+		child.on('error', (error) => {
+			clearTimeout(timeout);
+			reject(error);
+		});
+		child.on('close', (exitCode) => {
+			clearTimeout(timeout);
+			resolve({ exitCode: exitCode ?? 0, stdout, stderr });
+		});
+	});
+}
+
+async function runCommandWithInput(
+	command: string,
+	args: string[],
+	input: string,
+): Promise<ExecResult> {
+	return new Promise((resolve, reject) => {
+		const child = spawn(command, args, { stdio: ['pipe', 'pipe', 'pipe'] });
+		let stdout = '';
+		let stderr = '';
+		const timeout = setTimeout(() => {
+			child.kill('SIGTERM');
+			reject(new Error(`${command} timed out after ${DEFAULT_TIMEOUT_MS}ms`));
+		}, DEFAULT_TIMEOUT_MS);
+		child.stdout.setEncoding('utf8');
+		child.stderr.setEncoding('utf8');
+		child.stdout.on('data', (chunk) => {
+			stdout += chunk;
+		});
+		child.stderr.on('data', (chunk) => {
+			stderr += chunk;
+		});
+		child.on('error', (error) => {
+			clearTimeout(timeout);
+			reject(error);
+		});
+		child.on('close', (exitCode) => {
+			clearTimeout(timeout);
+			resolve({ exitCode: exitCode ?? 0, stdout, stderr });
+		});
+		child.stdin.end(input);
+	});
+}
+
+type KeyboardEventPayload = {
+	type: 'down' | 'up';
+	usage: number;
+};
+
+async function sendKeyboardEvents(
+	wsUrl: string,
+	events: KeyboardEventPayload[],
+	delayMs = 12,
+): Promise<void> {
+	await new Promise<void>((resolve, reject) => {
+		const socket = new WebSocket(wsUrl);
+		socket.binaryType = 'arraybuffer';
+		socket.onopen = async () => {
+			try {
+				for (const event of events) {
+					const encoded = new TextEncoder().encode(JSON.stringify(event));
+					const payload = new Uint8Array(1 + encoded.length);
+					payload[0] = 6;
+					payload.set(encoded, 1);
+					socket.send(payload);
+					await new Promise((done) => setTimeout(done, delayMs));
+				}
+				setTimeout(() => {
+					socket.close();
+					resolve();
+				}, 50);
+			} catch (error) {
+				socket.close();
+				reject(error);
+			}
+		};
+		socket.onerror = () =>
+			reject(new Error(`WebSocket connection failed: ${wsUrl}`));
+	});
+}
+
+function parseJson<T>(raw: string): T | null {
+	try {
+		return JSON.parse(raw) as T;
+	} catch {
+		return null;
+	}
+}
+
+async function listStreams(device?: string): Promise<ServeSimEntry[]> {
+	const result = await execServeSim(
+		device ? ['--list', '-q', device] : ['--list', '-q'],
+	);
+	if (result.exitCode !== 0) {
+		throw new Error(
+			result.stderr.trim() || result.stdout.trim() || 'serve-sim --list failed',
+		);
+	}
+	const parsed = parseJson<ServeSimEntry[] | ServeSimEntry>(
+		result.stdout.trim(),
+	);
+	if (!parsed) return [];
+	return Array.isArray(parsed) ? parsed : [parsed];
+}
+
+async function resolveDeviceTarget(
+	device?: string,
+): Promise<string | undefined> {
+	const requested = device?.trim();
+	if (!requested) return undefined;
+	try {
+		const streams = await listStreams(requested);
+		const resolved = streams.find((stream) => stream.device)?.device;
+		return resolved ?? requested;
+	} catch {
+		return requested;
+	}
+}
+
+async function getSimctlTarget(device?: string): Promise<string> {
+	return (await resolveDeviceTarget(device)) ?? 'booted';
+}
+
+async function ensureStream(device?: string): Promise<ServeSimEntry> {
+	const existing = await listStreams(device);
+	const first = existing[0];
+	if (first?.streamUrl || first?.url) return first;
+	const result = await execServeSim(
+		device ? ['--detach', '-q', device] : ['--detach', '-q'],
+	);
+	if (result.exitCode !== 0) {
+		throw new Error(
+			result.stderr.trim() || result.stdout.trim() || 'serve-sim start failed',
+		);
+	}
+	const parsed = parseJson<ServeSimEntry>(result.stdout.trim());
+	if (!parsed) throw new Error('serve-sim returned invalid JSON');
+	return parsed;
+}
+
+async function canReach(url: string): Promise<boolean> {
+	const controller = new AbortController();
+	const timeout = setTimeout(() => controller.abort(), 500);
+	try {
+		const response = await fetch(url, { signal: controller.signal });
+		return response.ok || response.status < 500;
+	} catch {
+		return false;
+	} finally {
+		clearTimeout(timeout);
+	}
+}
+
+async function waitForPreviewUrl(timeoutMs = 4000): Promise<string | null> {
+	const startedAt = Date.now();
+	while (Date.now() - startedAt < timeoutMs) {
+		if (await canReach(DEFAULT_PREVIEW_URL)) return DEFAULT_PREVIEW_URL;
+		if (
+			previewProcess?.exitCode !== null &&
+			previewProcess?.exitCode !== undefined
+		)
+			return null;
+		await new Promise((resolve) => setTimeout(resolve, 150));
+	}
+	return (await canReach(DEFAULT_PREVIEW_URL)) ? DEFAULT_PREVIEW_URL : null;
+}
+
+async function ensurePreviewServer(): Promise<string> {
+	if (await canReach(DEFAULT_PREVIEW_URL)) return DEFAULT_PREVIEW_URL;
+	if (!previewProcess || previewProcess.exitCode !== null) {
+		previewStdout = '';
+		previewStderr = '';
+		const args = ['serve-sim', '--port', String(DEFAULT_PREVIEW_PORT)];
+		previewProcess = spawn('bunx', args, {
+			stdio: ['ignore', 'pipe', 'pipe'],
+		});
+		previewProcess.stdout?.setEncoding('utf8');
+		previewProcess.stderr?.setEncoding('utf8');
+		previewProcess.stdout?.on('data', (chunk) => {
+			previewStdout += chunk;
+		});
+		previewProcess.stderr?.on('data', (chunk) => {
+			previewStderr += chunk;
+		});
+		previewProcess.on('close', () => {
+			previewProcess = null;
+		});
+	}
+
+	const url = await waitForPreviewUrl();
+	if (!url) {
+		throw new Error(
+			previewStderr || previewStdout || 'serve-sim preview failed to start',
+		);
+	}
+	return url;
+}
+
+async function fetchJson(url: string): Promise<unknown> {
+	const controller = new AbortController();
+	const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+	try {
+		const response = await fetch(url, { signal: controller.signal });
+		if (!response.ok)
+			throw new Error(`${response.status} ${response.statusText}`);
+		return await response.json();
+	} finally {
+		clearTimeout(timeout);
+	}
+}
+
+async function fetchTextFor(url: string, timeoutMs: number): Promise<string> {
+	const controller = new AbortController();
+	const timeout = setTimeout(() => controller.abort(), timeoutMs);
+	try {
+		const response = await fetch(url, { signal: controller.signal });
+		return await response.text();
+	} catch (error) {
+		if (error instanceof Error && error.name === 'AbortError') return '';
+		throw error;
+	} finally {
+		clearTimeout(timeout);
+	}
+}
+
+function concatBytes(chunks: Uint8Array[], totalLength: number): Uint8Array {
+	const output = new Uint8Array(totalLength);
+	let offset = 0;
+	for (const chunk of chunks) {
+		output.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return output;
+}
+
+function indexOfBytes(haystack: Uint8Array, needle: Uint8Array): number {
+	for (
+		let index = 0;
+		index <= haystack.byteLength - needle.byteLength;
+		index++
+	) {
+		let matched = true;
+		for (let needleIndex = 0; needleIndex < needle.byteLength; needleIndex++) {
+			if (haystack[index + needleIndex] !== needle[needleIndex]) {
+				matched = false;
+				break;
+			}
+		}
+		if (matched) return index;
+	}
+	return -1;
+}
+
+function findJpegEnd(bytes: Uint8Array, startIndex: number): number {
+	for (let index = startIndex + 2; index < bytes.byteLength - 1; index++) {
+		if (bytes[index] === 0xff && bytes[index + 1] === 0xd9) {
+			return index + 2;
+		}
+	}
+	return -1;
+}
+
+async function fetchFirstMjpegFrame(url: string): Promise<Uint8Array> {
+	const controller = new AbortController();
+	const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+	try {
+		const response = await fetch(url, { signal: controller.signal });
+		if (!response.ok)
+			throw new Error(`${response.status} ${response.statusText}`);
+		const reader = response.body?.getReader();
+		if (!reader) throw new Error('serve-sim response did not include a body');
+
+		const chunks: Uint8Array[] = [];
+		let totalLength = 0;
+		const headerDelimiter = new TextEncoder().encode('\r\n\r\n');
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			chunks.push(value);
+			totalLength += value.byteLength;
+			const bytes = concatBytes(chunks, totalLength);
+			const headerEnd = indexOfBytes(bytes, headerDelimiter);
+			if (headerEnd >= 0) {
+				const headerText = new TextDecoder().decode(bytes.slice(0, headerEnd));
+				const lengthMatch = headerText.match(/content-length:\s*(\d+)/i);
+				if (lengthMatch?.[1]) {
+					const frameStart = headerEnd + headerDelimiter.byteLength;
+					const frameLength = Number(lengthMatch[1]);
+					const frameEnd = frameStart + frameLength;
+					if (bytes.byteLength >= frameEnd) {
+						return bytes.slice(frameStart, frameEnd);
+					}
+				}
+
+				const frameStart = indexOfBytes(bytes, new Uint8Array([0xff, 0xd8]));
+				if (frameStart >= 0) {
+					const frameEnd = findJpegEnd(bytes, frameStart);
+					if (frameEnd >= 0) return bytes.slice(frameStart, frameEnd);
+				}
+			}
+		}
+		throw new Error('serve-sim stream ended before a complete frame was read');
+	} finally {
+		clearTimeout(timeout);
+	}
+}
+
+async function prepareScreenshotForModel(bytes: Uint8Array): Promise<{
+	data: Uint8Array;
+	mediaType: string;
+	compressed: boolean;
+	width?: number;
+	height?: number;
+}> {
+	const ImageCtor = getBunImageConstructor();
+	if (!ImageCtor) {
+		return { data: bytes, mediaType: 'image/jpeg', compressed: false };
+	}
+
+	try {
+		const image = new ImageCtor(bytes);
+		const metadata = await image.metadata();
+		const width = metadata.width;
+		const height = metadata.height;
+		if (!width || !height) {
+			return { data: bytes, mediaType: 'image/jpeg', compressed: false };
+		}
+
+		const longestEdge = Math.max(width, height);
+		if (longestEdge <= SCREENSHOT_MODEL_MAX_EDGE) {
+			return {
+				data: bytes,
+				mediaType: 'image/jpeg',
+				compressed: false,
+				width,
+				height,
+			};
+		}
+
+		const scale = SCREENSHOT_MODEL_MAX_EDGE / longestEdge;
+		const targetWidth = Math.max(1, Math.round(width * scale));
+		const targetHeight = Math.max(1, Math.round(height * scale));
+		const compressed = await image
+			.resize(targetWidth, targetHeight, {
+				fit: 'inside',
+				withoutEnlargement: true,
+			})
+			.jpeg({ quality: SCREENSHOT_MODEL_JPEG_QUALITY })
+			.bytes();
+
+		return {
+			data: compressed,
+			mediaType: 'image/jpeg',
+			compressed: true,
+			width: targetWidth,
+			height: targetHeight,
+		};
+	} catch {
+		return { data: bytes, mediaType: 'image/jpeg', compressed: false };
+	}
+}
+
+function buildScreenshotArtifactPath(
+	projectRoot: string,
+	outputPath?: string,
+): {
+	relativePath: string;
+	absPath: string;
+} {
+	const requestedName = outputPath?.trim()
+		? basename(outputPath.trim())
+		: `screenshot-${Date.now()}-${randomUUID()}.jpg`;
+	const extension = extname(requestedName).toLowerCase();
+	const fileName = extension
+		? requestedName
+		: `${requestedName || `screenshot-${randomUUID()}`}.jpg`;
+	const safeName = fileName.replace(/[^a-zA-Z0-9._-]/g, '-');
+	const relativePath = `${SCREENSHOT_ARTIFACTS_DIR}/${safeName}`;
+	return {
+		relativePath,
+		absPath: join(projectRoot, relativePath),
+	};
+}
+
+function getStreamUrl(entry: ServeSimEntry): string {
+	if (entry.streamUrl) {
+		const url = new URL(entry.streamUrl);
+		return url.origin;
+	}
+	throw new Error('No serve-sim streamUrl found. Start the simulator first.');
+}
+
+function getPreviewUrl(entry: ServeSimEntry): string {
+	if (entry.url) return entry.url.replace(/\/$/, '');
+	throw new Error('No serve-sim preview url found. Start the simulator first.');
+}
+
+async function runCliAction(args: string[]) {
+	const result = await execServeSim(args);
+	if (result.exitCode !== 0) {
+		return createToolError(
+			result.stderr.trim() ||
+				result.stdout.trim() ||
+				'serve-sim command failed',
+			'execution',
+			{ args },
+		);
+	}
+	return {
+		ok: true,
+		stdout: result.stdout.trim(),
+		stderr: result.stderr.trim(),
+	};
+}
+
+function shouldPasteForText(text: string): boolean {
+	return text.length > 8 || /[:/.?#&=%]/.test(text);
+}
+
+async function pasteTextIntoFocusedField(
+	text: string,
+	device?: string,
+): Promise<ExecResult & { method: 'paste' }> {
+	const target = await getSimctlTarget(device);
+	const pbcopy = await runCommandWithInput(
+		'xcrun',
+		['simctl', 'pbcopy', target],
+		text,
+	);
+	if (pbcopy.exitCode !== 0) return { ...pbcopy, method: 'paste' };
+	const stream = await ensureStream(device);
+	if (!stream.wsUrl) {
+		return {
+			exitCode: 1,
+			stdout: pbcopy.stdout,
+			stderr: 'No serve-sim WebSocket URL found for paste action',
+			method: 'paste',
+		};
+	}
+	await sendKeyboardEvents(stream.wsUrl, [
+		{ type: 'down', usage: HID_KEYBOARD_LEFT_GUI },
+		{ type: 'down', usage: HID_KEYBOARD_V },
+		{ type: 'up', usage: HID_KEYBOARD_V },
+		{ type: 'up', usage: HID_KEYBOARD_LEFT_GUI },
+	]);
+	return {
+		exitCode: 0,
+		stdout: pbcopy.stdout,
+		stderr: pbcopy.stderr,
+		method: 'paste',
+	};
+}
+
+async function runTextInputAction(text: string, device?: string) {
+	if (shouldPasteForText(text)) {
+		const result = await pasteTextIntoFocusedField(text, device);
+		if (result.exitCode !== 0) {
+			return createToolError(
+				result.stderr.trim() ||
+					result.stdout.trim() ||
+					'simulator paste failed',
+				'execution',
+				{ method: 'paste' },
+			);
+		}
+		return {
+			ok: true,
+			method: 'paste',
+			stdout: result.stdout.trim(),
+			stderr: result.stderr.trim(),
+		};
+	}
+
+	const typeDevice = await resolveDeviceTarget(device);
+	const result = await execServeSim(withDevice(['type', text], typeDevice));
+	if (result.exitCode !== 0) {
+		return createToolError(
+			result.stderr.trim() || result.stdout.trim() || 'serve-sim type failed',
+			'execution',
+			{ method: 'type' },
+		);
+	}
+	return {
+		ok: true,
+		method: 'type',
+		stdout: result.stdout.trim(),
+		stderr: result.stderr.trim(),
+	};
+}
+
+export function buildSimulatorTool(projectRoot: string): {
+	name: string;
+	tool: Tool;
+} {
+	return {
+		name: 'simulator',
+		tool: tool({
+			description:
+				'Control an Apple Simulator through serve-sim and simctl. Coordinates are normalized 0..1. Use action=start to open the simulator preview, launch to open an app bundle, take_screenshot for screenshots, click for taps, type for short keyboard input, paste for URLs/long strings in focused fields, and button/rotate/status/config/accessibility_tree/foreground/list_apps/logs/stop as needed. Prefer this tool over shell for simulator operations.',
+			inputSchema: simulatorInputSchema,
+			toModelOutput({ output }) {
+				const result = output as {
+					ok?: boolean;
+					path?: string;
+					artifact?: {
+						data?: string;
+						mediaType?: string;
+						transmittedSize?: number;
+						compressed?: boolean;
+						width?: number;
+						height?: number;
+					};
+				};
+				const image = result.artifact;
+				if (
+					result.ok === true &&
+					typeof image?.data === 'string' &&
+					typeof image.mediaType === 'string'
+				) {
+					const dimensions =
+						typeof image.width === 'number' && typeof image.height === 'number'
+							? `, ${image.width}x${image.height}`
+							: '';
+					const compressed = image.compressed ? ', compressed' : '';
+					return {
+						type: 'content',
+						value: [
+							{
+								type: 'text',
+								text: `Simulator screenshot captured from ${result.path ?? 'current screen'} (${image.mediaType}${dimensions}${compressed}, ${image.transmittedSize ?? image.data.length} bytes sent to the model). Inspect the following image content.`,
+							},
+							{
+								type: 'image-data',
+								data: image.data,
+								mediaType: image.mediaType,
+							},
+						],
+					};
+				}
+				return { type: 'json', value: toJsonValue(output) };
+			},
+			execute: async (rawInput) => {
+				const input = parseSimulatorInput(rawInput);
+				try {
+					switch (input.action) {
+						case 'start': {
+							const previewUrl = await ensurePreviewServer();
+							const stream = await ensureStream(input.device);
+							return {
+								ok: true,
+								stream,
+								previewUrl,
+								message: `Simulator preview available at ${previewUrl}`,
+							};
+						}
+						case 'status': {
+							const streams = await listStreams(input.device);
+							const previewUrl = (await canReach(DEFAULT_PREVIEW_URL))
+								? DEFAULT_PREVIEW_URL
+								: undefined;
+							return { ok: true, streams, count: streams.length, previewUrl };
+						}
+						case 'stop':
+							return runCliAction(
+								input.device ? ['--kill', input.device] : ['--kill'],
+							);
+						case 'click': {
+							const clickDevice = await resolveDeviceTarget(input.device);
+							return runCliAction(
+								withDevice(
+									['tap', String(input.x), String(input.y)],
+									clickDevice,
+								),
+							);
+						}
+						case 'type': {
+							return runTextInputAction(input.text, input.device);
+						}
+						case 'paste': {
+							const result = await pasteTextIntoFocusedField(
+								input.text,
+								input.device,
+							);
+							if (result.exitCode !== 0) {
+								return createToolError(
+									result.stderr.trim() ||
+										result.stdout.trim() ||
+										'simulator paste failed',
+									'execution',
+									{ method: 'paste' },
+								);
+							}
+							return {
+								ok: true,
+								method: 'paste',
+								stdout: result.stdout.trim(),
+								stderr: result.stderr.trim(),
+							};
+						}
+						case 'button': {
+							const buttonDevice = await resolveDeviceTarget(input.device);
+							return runCliAction(
+								withDevice(['button', input.name], buttonDevice),
+							);
+						}
+						case 'rotate': {
+							const rotateDevice = await resolveDeviceTarget(input.device);
+							return runCliAction(
+								withDevice(['rotate', input.orientation], rotateDevice),
+							);
+						}
+						case 'config': {
+							const stream = await ensureStream(input.device);
+							const config = await fetchJson(`${getStreamUrl(stream)}/config`);
+							return { ok: true, config, stream };
+						}
+						case 'accessibility_tree': {
+							const stream = await ensureStream(input.device);
+							const tree = await fetchJson(`${getStreamUrl(stream)}/ax`);
+							return { ok: true, accessibilityTree: tree, stream };
+						}
+						case 'foreground': {
+							const stream = await ensureStream(input.device);
+							const foreground = await fetchJson(
+								`${getStreamUrl(stream)}/foreground`,
+							);
+							return { ok: true, foreground, stream };
+						}
+						case 'take_screenshot': {
+							const stream = await ensureStream(input.device);
+							const bytes = await fetchFirstMjpegFrame(
+								`${getStreamUrl(stream)}/stream.mjpeg?raw=1`,
+							);
+							const screenshot = await prepareScreenshotForModel(bytes);
+							const { relativePath, absPath } = buildScreenshotArtifactPath(
+								projectRoot,
+								input.outputPath,
+							);
+							await mkdir(join(projectRoot, SCREENSHOT_ARTIFACTS_DIR), {
+								recursive: true,
+							});
+							await writeFile(absPath, bytes);
+							return {
+								ok: true,
+								path: relativePath,
+								message: `Simulator screenshot stored in Otto artifacts at ${relativePath}`,
+								artifact: {
+									kind: 'simulator_screenshot',
+									path: relativePath,
+									mediaType: screenshot.mediaType,
+									data: Buffer.from(screenshot.data).toString('base64'),
+									originalSize: bytes.byteLength,
+									transmittedSize: screenshot.data.byteLength,
+									compressed: screenshot.compressed,
+									width: screenshot.width,
+									height: screenshot.height,
+								},
+								stream,
+							};
+						}
+						case 'launch': {
+							const result = await runCommand('xcrun', [
+								'simctl',
+								'launch',
+								await getSimctlTarget(input.device),
+								input.bundleId,
+								...(input.args ?? []),
+							]);
+							return {
+								ok: result.exitCode === 0,
+								exitCode: result.exitCode,
+								bundleId: input.bundleId,
+								stdout: result.stdout.trim(),
+								stderr: result.stderr.trim(),
+							};
+						}
+						case 'terminate': {
+							const result = await runCommand('xcrun', [
+								'simctl',
+								'terminate',
+								await getSimctlTarget(input.device),
+								input.bundleId,
+							]);
+							return {
+								ok: result.exitCode === 0,
+								exitCode: result.exitCode,
+								bundleId: input.bundleId,
+								stdout: result.stdout.trim(),
+								stderr: result.stderr.trim(),
+							};
+						}
+						case 'open_url': {
+							const result = await runCommand('xcrun', [
+								'simctl',
+								'openurl',
+								await getSimctlTarget(input.device),
+								input.url,
+							]);
+							return {
+								ok: result.exitCode === 0,
+								exitCode: result.exitCode,
+								url: input.url,
+								stdout: result.stdout.trim(),
+								stderr: result.stderr.trim(),
+							};
+						}
+						case 'list_apps': {
+							const result = await runCommand('xcrun', [
+								'simctl',
+								'listapps',
+								await getSimctlTarget(input.device),
+							]);
+							const filter = input.text?.trim().toLowerCase();
+							const stdout = filter
+								? result.stdout
+										.split('\n')
+										.filter((line) => line.toLowerCase().includes(filter))
+										.join('\n')
+								: result.stdout;
+							return {
+								ok: result.exitCode === 0,
+								exitCode: result.exitCode,
+								stdout: stdout.trim(),
+								stderr: result.stderr.trim(),
+							};
+						}
+						case 'logs': {
+							const stream = await ensureStream(input.device);
+							const logs = await fetchTextFor(
+								`${getPreviewUrl(stream)}/.sim/logs`,
+								input.timeoutMs ?? LOG_TIMEOUT_MS,
+							);
+							return { ok: true, logs, stream };
+						}
+					}
+				} catch (error) {
+					return createToolError(
+						error instanceof Error ? error.message : String(error),
+						'execution',
+						{ action: input.action },
+					);
+				}
+			},
+		}),
+	};
+}

package/src/core/src/tools/loader.ts

@@ -24,6 +24,10 @@ import {
 	getMCPToolsRecord,
 	type MCPToolBrief,
 } from '../mcp/lazy-tools.ts';
+import {
+	buildLazyToolsRecord,
+	buildLoadFirstPartyToolsTool,
+} from './lazy/index.ts';
 import { dirname, isAbsolute, join } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { promises as fs } from 'node:fs';
@@ -34,6 +38,7 @@ export type DiscoveredTool = { name: string; tool: Tool };
 
 export type DiscoverResult = {
 	tools: DiscoveredTool[];
+	lazyToolsRecord: Record<string, Tool>;
 	mcpToolsRecord: Record<string, Tool>;
 };
 
@@ -211,6 +216,10 @@ export async function discoverProjectTools(
 		tools.set(term.name, term.tool);
 	}
 
+	const lazyToolsRecord = buildLazyToolsRecord(projectRoot);
+	const loadFirstPartyTools = buildLoadFirstPartyToolsTool();
+	tools.set(loadFirstPartyTools.name, loadFirstPartyTools.tool);
+
 	const mcpManager = getMCPManager();
 	let mcpToolsRecord: Record<string, Tool> = {};
 	let mcpBriefs: MCPToolBrief[] = [];
@@ -225,6 +234,7 @@ export async function discoverProjectTools(
 
 	return {
 		tools: Array.from(tools.entries()).map(([name, tool]) => ({ name, tool })),
+		lazyToolsRecord,
 		mcpToolsRecord,
 	};
 }

package/src/index.ts

@@ -295,6 +295,11 @@ export type { ShellExecutor } from './core/src/index.ts';
 export { createToolError } from './core/src/index.ts';
 export { buildFsTools } from './core/src/index.ts';
 export { buildGitTools } from './core/src/index.ts';
+export {
+	buildLazyToolsRecord,
+	buildLoadFirstPartyToolsTool,
+	buildSimulatorTool,
+} from './core/src/index.ts';
 export {
 	appendCoAuthorTrailer,
 	injectCoAuthorIntoGitCommit,