@mishri/ai-term 1.0.0

package/README.md

@@ -0,0 +1,77 @@
+# ai-term
+
+A terminal AI agent with tool use — read/write files, run shell commands, execute code, and search the web. Built with [Ink](https://github.com/vadimdemedes/ink) and the [Vercel AI SDK](https://sdk.vercel.ai/).
+
+## Requirements
+
+- Node.js 20+
+- An [OpenAI API key](https://platform.openai.com/api-keys)
+
+## Quick start (from source)
+
+```bash
+git clone https://github.com/Mishragini/ai-term.git
+cd ai-term
+npm install
+cp .env.example .env
+# Add your OPENAI_API_KEY to .env
+npm start
+```
+
+Type your prompt at the `>` prompt. Type `exit` or `quit` to leave.
+
+## Install globally (after publishing to npm)
+
+Once published:
+
+```bash
+npm install -g ai-term
+export OPENAI_API_KEY=your-key-here
+ai-term
+```
+
+Or run without installing:
+
+```bash
+npx ai-term
+```
+
+Set `OPENAI_API_KEY` in your environment or in a `.env` file in the directory where you run the command.
+
+## Environment variables
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `OPENAI_API_KEY` | Yes | OpenAI API key for the agent |
+| `LMNR_PROJECT_API_KEY` | No | [Laminar](https://lmnr.ai/) project key for tracing |
+
+## Tools
+
+The agent can use these tools (shell and code execution require your approval before running):
+
+- **readFile / writeFile / listFiles / deleteFile** — file operations in the working directory
+- **shellCommand** — run shell commands (approval required)
+- **execCode** — execute JavaScript/TypeScript snippets (approval required)
+- **webSearch** — search the web via OpenAI
+
+## Development
+
+```bash
+npm run build    # compile TypeScript to dist/
+npm start        # build and run
+npm run eval-file       # run file-tool evals
+npm run eval-multiturn  # run multi-turn agent evals
+```
+
+## Publishing to npm
+
+```bash
+npm login
+npm publish
+```
+
+The package ships the compiled `dist/` output and exposes the `ai-term` CLI command.
+
+## License
+
+ISC

package/dist/agent/context/compaction.d.ts

@@ -0,0 +1,3 @@
+import { type ModelMessage } from "ai";
+export declare function compactConversation(messages: ModelMessage[], model?: string): Promise<ModelMessage[]>;
+//# sourceMappingURL=compaction.d.ts.map

package/dist/agent/context/compaction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction.d.ts","sourceRoot":"","sources":["../../../src/agent/context/compaction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,YAAY,EAAE,MAAM,IAAI,CAAC;AAqBrD,wBAAsB,mBAAmB,CACrC,QAAQ,EAAE,YAAY,EAAE,EACxB,KAAK,GAAE,MAAqB,2BA2B/B"}

package/dist/agent/context/compaction.js

@@ -0,0 +1,39 @@
+import { generateText } from "ai";
+import { extractMessageText } from "./tokenEstimator.js";
+import { openai } from "@ai-sdk/openai";
+const SUMMARIZATION_PROMPT = `You are a conversation summarizer. Your task is to create a concise summary of the conversation so far that preserves:
+
+1. Key decisions and conclusions reached
+2. Important context and facts mentioned
+3. Any pending tasks or questions
+4. The overall goal of the conversation
+
+Be concise but complete. The summary should allow the conversation to continue naturally.
+
+Conversation to summarize:
+`;
+function messagesToText(messages) {
+    return messages.map((msg) => `[${msg.role}]: ${extractMessageText(msg)}`).join("\n\n");
+}
+export async function compactConversation(messages, model = "gpt-5-mini") {
+    const conversationMessages = messages.filter((msg) => msg.role !== "system");
+    if (conversationMessages.length === 0)
+        return [];
+    const conversationText = messagesToText(conversationMessages);
+    const { text: summary } = await generateText({
+        model: openai(model),
+        prompt: `${SUMMARIZATION_PROMPT} ${conversationText}`
+    });
+    const compactMessages = [
+        {
+            role: "user",
+            content: `[CONVERSATION SUMMARY]\nThe following is a summary of our conversation so far:\n\n${summary}\n\nPlease continue from where we left off.`
+        },
+        {
+            role: "assistant",
+            content: "I understand. I've reviewed the summary of our conversation and I'm ready to continue. How can I help you next?",
+        }
+    ];
+    return compactMessages;
+}
+//# sourceMappingURL=compaction.js.map

package/dist/agent/context/compaction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction.js","sourceRoot":"","sources":["../../../src/agent/context/compaction.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAqB,MAAM,IAAI,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,MAAM,oBAAoB,GAAG;;;;;;;;;;CAU5B,CAAC;AAGF,SAAS,cAAc,CAAC,QAAwB;IAC5C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,MAAM,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;AAC1F,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,QAAwB,EACxB,QAAgB,YAAY;IAG5B,MAAM,oBAAoB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAA;IAE5E,IAAI,oBAAoB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAEhD,MAAM,gBAAgB,GAAG,cAAc,CAAC,oBAAoB,CAAC,CAAA;IAE7D,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC;QACzC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;QACpB,MAAM,EAAE,GAAG,oBAAoB,IAAI,gBAAgB,EAAE;KACxD,CAAC,CAAA;IAGF,MAAM,eAAe,GAAmB;QACpC;YACI,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,qFAAqF,OAAO,6CAA6C;SACrJ;QACD;YACI,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,iHAAiH;SAC7H;KACJ,CAAA;IAED,OAAO,eAAe,CAAA;AAC1B,CAAC"}

package/dist/agent/context/index.d.ts

@@ -0,0 +1,7 @@
+import type { ModelMessage } from "ai";
+export declare function estimateMessagesTokens(messages: ModelMessage[]): {
+    input: number;
+    output: number;
+    total: number;
+};
+//# sourceMappingURL=index.d.ts.map

package/dist/agent/context/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/agent/context/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AASvC,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,EAAE;;;;EAoB9D"}

package/dist/agent/context/index.js

@@ -0,0 +1,24 @@
+import { extractMessageText } from "./tokenEstimator.js";
+//usually use and llm to estimate token but sine i dont have money use vague ass assumption that 3.75chars = 1 token
+function estimateTokens(text) {
+    return Math.ceil(text.length / 3.75);
+}
+export function estimateMessagesTokens(messages) {
+    let input = 0, output = 0;
+    for (const msg of messages) {
+        const text = extractMessageText(msg);
+        const tokens = estimateTokens(text);
+        if (msg.role === "assistant") {
+            output += tokens;
+        }
+        else {
+            input += tokens;
+        }
+    }
+    return {
+        input,
+        output,
+        total: input + output
+    };
+}
+//# sourceMappingURL=index.js.map

package/dist/agent/context/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/agent/context/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAGzD,oHAAoH;AACpH,SAAS,cAAc,CAAC,IAAY;IAChC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,CAAA;AACxC,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,QAAwB;IAE3D,IAAI,KAAK,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAA;IACzB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAA;QACpC,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;QAEnC,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,MAAM,IAAI,MAAM,CAAA;QACpB,CAAC;aAAM,CAAC;YACJ,KAAK,IAAI,MAAM,CAAA;QACnB,CAAC;IACL,CAAC;IAED,OAAO;QACH,KAAK;QACL,MAAM;QACN,KAAK,EAAE,KAAK,GAAG,MAAM;KACxB,CAAA;AAEL,CAAC"}

package/dist/agent/context/modelLimits.d.ts

@@ -0,0 +1,7 @@
+import type { ModelLimits } from "../type.js";
+export declare const DEFAULT_THRESHOLD = 0.8;
+export declare const MODEL_LIMITS: Record<string, ModelLimits>;
+export declare function getModelLimits(model: string): ModelLimits;
+export declare function isOverThreshold(totalTokens: number, contextWindow: number, threshold?: number): boolean;
+export declare function calculateUsagePercentage(totalTokens: number, contextWindow: number): number;
+//# sourceMappingURL=modelLimits.d.ts.map

package/dist/agent/context/modelLimits.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"modelLimits.d.ts","sourceRoot":"","sources":["../../../src/agent/context/modelLimits.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAE7C,eAAO,MAAM,iBAAiB,MAAM,CAAA;AAEpC,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAYpD,CAAA;AASD,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,eAO3C;AAED,wBAAgB,eAAe,CAC3B,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,MAAM,EACrB,SAAS,GAAE,MAA0B,WAGxC;AAGD,wBAAgB,wBAAwB,CACpC,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,MAAM,UAGxB"}

package/dist/agent/context/modelLimits.js

@@ -0,0 +1,31 @@
+export const DEFAULT_THRESHOLD = 0.8;
+export const MODEL_LIMITS = {
+    "gpt-5": {
+        inputLimit: 272000,
+        outputLimit: 128000,
+        contextWindow: 400000
+    },
+    "gpt-5-mini": {
+        inputLimit: 272000,
+        outputLimit: 128000,
+        contextWindow: 400000,
+    },
+};
+const DEFAULT_LIMITS = {
+    inputLimit: 128000,
+    outputLimit: 16000,
+    contextWindow: 128000,
+};
+export function getModelLimits(model) {
+    if (MODEL_LIMITS[model]) {
+        return MODEL_LIMITS[model];
+    }
+    return DEFAULT_LIMITS;
+}
+export function isOverThreshold(totalTokens, contextWindow, threshold = DEFAULT_THRESHOLD) {
+    return totalTokens > contextWindow * threshold;
+}
+export function calculateUsagePercentage(totalTokens, contextWindow) {
+    return (totalTokens / contextWindow) * 100;
+}
+//# sourceMappingURL=modelLimits.js.map

package/dist/agent/context/modelLimits.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"modelLimits.js","sourceRoot":"","sources":["../../../src/agent/context/modelLimits.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAA;AAEpC,MAAM,CAAC,MAAM,YAAY,GAAgC;IACrD,OAAO,EAAE;QACL,UAAU,EAAE,MAAM;QAClB,WAAW,EAAE,MAAM;QACnB,aAAa,EAAE,MAAM;KACxB;IAED,YAAY,EAAE;QACV,UAAU,EAAE,MAAM;QAClB,WAAW,EAAE,MAAM;QACnB,aAAa,EAAE,MAAM;KACxB;CACJ,CAAA;AAED,MAAM,cAAc,GAAgB;IAChC,UAAU,EAAE,MAAM;IAClB,WAAW,EAAE,KAAK;IAClB,aAAa,EAAE,MAAM;CACxB,CAAC;AAGF,MAAM,UAAU,cAAc,CAAC,KAAa;IACxC,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,YAAY,CAAC,KAAK,CAAC,CAAA;IAC9B,CAAC;IAED,OAAO,cAAc,CAAA;AAEzB,CAAC;AAED,MAAM,UAAU,eAAe,CAC3B,WAAmB,EACnB,aAAqB,EACrB,YAAoB,iBAAiB;IAErC,OAAO,WAAW,GAAG,aAAa,GAAG,SAAS,CAAA;AAClD,CAAC;AAGD,MAAM,UAAU,wBAAwB,CACpC,WAAmB,EACnB,aAAqB;IAErB,OAAO,CAAC,WAAW,GAAG,aAAa,CAAC,GAAG,GAAG,CAAA;AAC9C,CAAC"}

package/dist/agent/context/tokenEstimator.d.ts

@@ -0,0 +1,3 @@
+import type { ModelMessage } from "ai";
+export declare function extractMessageText(msg: ModelMessage): string;
+//# sourceMappingURL=tokenEstimator.d.ts.map

package/dist/agent/context/tokenEstimator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenEstimator.d.ts","sourceRoot":"","sources":["../../../src/agent/context/tokenEstimator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAEvC,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,YAAY,UA2BnD"}

package/dist/agent/context/tokenEstimator.js

@@ -0,0 +1,28 @@
+export function extractMessageText(msg) {
+    if (typeof msg.content === "string") {
+        return msg.content;
+    }
+    if (Array.isArray(msg.content)) {
+        return msg.content.map((part) => {
+            if (typeof part === "string") {
+                return part;
+            }
+            //assistant/user text content
+            if ("text" in part)
+                return part.text;
+            //tool-call-content
+            if ("value" in part && typeof part.value === "string")
+                return part.value;
+            //tool-result-content
+            if ("output" in part && typeof part.output === "object" && part.output) {
+                const output = part.output;
+                if ("value" in output && typeof output.value === "string") {
+                    return output.value;
+                }
+            }
+            return JSON.stringify(part);
+        }).join(" ");
+    }
+    return JSON.stringify(msg.content);
+}
+//# sourceMappingURL=tokenEstimator.js.map

package/dist/agent/context/tokenEstimator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenEstimator.js","sourceRoot":"","sources":["../../../src/agent/context/tokenEstimator.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,kBAAkB,CAAC,GAAiB;IAChD,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,GAAG,CAAC,OAAO,CAAA;IACtB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAC5B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC3B,OAAO,IAAI,CAAA;YACf,CAAC;YACD,6BAA6B;YAC7B,IAAI,MAAM,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC,IAAI,CAAA;YACpC,mBAAmB;YACnB,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC,KAAK,CAAA;YACxE,qBAAqB;YACrB,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACrE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;gBAC1B,IAAI,OAAO,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;oBACxD,OAAO,MAAM,CAAC,KAAK,CAAA;gBACvB,CAAC;YACL,CAAC;YAED,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAChB,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;AACtC,CAAC"}

package/dist/agent/evals/agent-multiturn.eval.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=agent-multiturn.eval.d.ts.map

package/dist/agent/evals/agent-multiturn.eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-multiturn.eval.d.ts","sourceRoot":"","sources":["../../../src/agent/evals/agent-multiturn.eval.ts"],"names":[],"mappings":""}

package/dist/agent/evals/agent-multiturn.eval.js

@@ -0,0 +1,32 @@
+import { evaluate } from "@lmnr-ai/lmnr";
+import dataset from "./data/agent-multiturn.json" with { type: "json" };
+import { LMNR_PROJECT_API_KEY } from "../../config.js";
+import { multiturnWithMocks } from "./executors.js";
+import { llmJudge, toolCallOrder, toolsAvoided } from "./evaluators.js";
+const executor = (data) => {
+    return multiturnWithMocks(data);
+};
+evaluate({
+    data: dataset,
+    executor,
+    evaluators: {
+        toolsAvoided: (output, target) => {
+            if (!target?.forbiddenTools?.length)
+                return 1;
+            return toolsAvoided(output, target);
+        },
+        toolOrder: (output, target) => {
+            if (!target?.expectedToolOrder?.length)
+                return 1;
+            return toolCallOrder(output, target);
+        },
+        outputQuality: (output, target) => {
+            if (!target)
+                return 1;
+            return llmJudge(output, target);
+        }
+    },
+    config: { projectApiKey: LMNR_PROJECT_API_KEY },
+    groupName: "multiturn-eval"
+});
+//# sourceMappingURL=agent-multiturn.eval.js.map

package/dist/agent/evals/agent-multiturn.eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-multiturn.eval.js","sourceRoot":"","sources":["../../../src/agent/evals/agent-multiturn.eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,OAAO,MAAM,6BAA6B,CAAE,OAAM,IAAI,EAAE,MAAM,EAAC,CAAA;AACtE,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAExE,MAAM,QAAQ,GAAG,CAAC,IAAuB,EAAE,EAAE;IACzC,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAA;AACnC,CAAC,CAAA;AACD,QAAQ,CAAC;IACL,IAAI,EAAE,OAA0E;IAChF,QAAQ;IACR,UAAU,EAAE;QACR,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE;YAC7B,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,MAAM;gBAAE,OAAO,CAAC,CAAA;YAC7C,OAAO,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACvC,CAAC;QACD,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE;YAC1B,IAAI,CAAC,MAAM,EAAE,iBAAiB,EAAE,MAAM;gBAAE,OAAO,CAAC,CAAA;YAChD,OAAO,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACxC,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE;YAC9B,IAAI,CAAC,MAAM;gBAAE,OAAO,CAAC,CAAA;YACrB,OAAO,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACnC,CAAC;KACJ;IACD,MAAM,EAAE,EAAE,aAAa,EAAE,oBAAoB,EAAE;IAC/C,SAAS,EAAE,gBAAgB;CAC9B,CAAC,CAAA"}

package/dist/agent/evals/data/agent-multiturn.json

@@ -0,0 +1,111 @@
+[
+    {
+        "data": {
+            "prompt": "Read package.json and tell me the project name",
+            "mockTools": {
+                "readFile": {
+                    "description": "Read the contents of a file at the specified path.",
+                    "parameters": { "path": "string" },
+                    "mockReturn": "{ \"name\": \"ai-agent-course\", \"version\": \"1.0.0\", \"description\": \"Build an AI agent from scratch\" }"
+                },
+                "shell": {
+                    "description": "Execute the shell command and return it's output",
+                    "parameters": { "command": "string" },
+                    "mockReturn": "command output here"
+                }
+            },
+            "config": {
+                "maxSteps": 5
+            }
+        },
+        "target": {
+            "originalTask": "Read package.json and tell me the project name.",
+            "expectedToolOrder": ["readFile"],
+            "forbiddenTools": ["shell"],
+            "mockToolResults": {
+                "readFile": "{ \"name\": \"ai-agent-course\", \"version\": \"1.0.0\", \"description\": \"Build an AI agent from scratch\" }"
+            },
+            "category": "task-completion"
+        },
+        "metadata": {
+            "description": "Fresh task: User asks to read file and extract info Should use readFile and not shell."
+        }
+    },
+    {
+        "data": {
+            "messages": [
+                {
+                    "role": "user",
+                    "content": "I'm working on a Node.js project and need help understanding it"
+                },
+                {
+                    "role": "assistant",
+                    "content": "I'd be happy to help you understand your Node.js project! What would you like to know about it?"
+                },
+                {
+                    "role": "user",
+                    "content": "List the files in the src directory and then read the main entry point"
+                }
+            ],
+            "mockTools": {
+                "listFiles": {
+                    "description": "List all files and directories in the specified directory path",
+                    "parameters": { "directory": "string" },
+                    "mockReturn": "index.ts\nApp.tsx\ncomponents/\nutils/"
+                },
+                "readFile": {
+                    "description": "Read the contents of a file at the specified path",
+                    "parameters": { "path": "string" },
+                    "mockReturn": "import { App } from './App';\n\nconsole.log('Starting application...');\nApp.run();"
+                }
+            },
+            "config": {
+                "maxSteps": 10
+            }
+        },
+        "target": {
+            "originalTask": "List the files in the src directory and then read the main entry point",
+            "expectedToolOrder": ["listFiles", "readFile"],
+            "mockToolResults": {
+                "listFiles": "index.ts\nApp.tsx\ncomponents/\nutils/",
+                "readFile": "import { App } from './App';\n\nconsole.log('Starting application...');\nApp.run();"
+            },
+            "category": "conversation-continuation"
+        },
+        "metadata": {
+            "description": "Mid-conversation: Pre-filled history, user asks multi-step task. Should list then read."
+        }
+    },
+    {
+        "data": {
+            "prompt": "What files are in the src directory?",
+            "mockTools": {
+                "listFiles": {
+                    "description": "List all files and directories in the specified directory path",
+                    "parameters": { "directory": "string" },
+                    "mockReturn": "index.ts\nApp.tsx\ncomponents/"
+                },
+                "shell": {
+                    "description": "Execute a shell command and return its output",
+                    "parameters": { "command": "string" },
+                    "mockReturn": "index.ts  App.tsx  components/"
+                }
+            },
+            "config": {
+                "maxSteps": 5
+            }
+        },
+        "target": {
+            "originalTask": "What files are in the src directory?",
+            "expectedToolOrder": ["listFiles"],
+            "forbiddenTools": ["shell"],
+            "mockToolResults": {
+                "listFiles": "index.ts\nApp.tsx\ncomponents/"
+            },
+            "category": "negative"
+        },
+        "metadata": {
+            "description": "Negative: File listing task should use listFiles tool, NOT shell (ls command). Tests tool category preference."
+        }
+    }
+]

package/dist/agent/evals/data/tool-calls.json

@@ -0,0 +1,28 @@
+[
+    {
+        "data": {
+            "prompt": "Remove the old backup.txt file",
+            "tools": ["readFile", "writeFile", "listFiles", "deleteFile"]
+        },
+        "target": {
+            "expectedTools": ["deleteFile"],
+            "category": "golden"
+        },
+        "metadata": {
+            "description": "File deletion request - should use deleteFile"
+        }
+    },
+    {
+        "data": {
+            "prompt": "What's in this project? Show me around.",
+            "tools": ["readFile", "writeFile", "listFiles", "deleteFile"]
+        },
+        "target": {
+            "expectedTools": ["listFiles"],
+            "category": "secondary"
+        },
+        "metadata": {
+            "description": "Ambiguous exploration request - likely uses listFiles"
+        }
+    }
+]

package/dist/agent/evals/evaluators.d.ts

@@ -0,0 +1,6 @@
+import type { EvalTarget, MultiturnEvalTarget, MultiturnRes, SingleTurnRes } from "./types.js";
+export declare function toolsSelected(output: SingleTurnRes, expected: EvalTarget): 1 | 0;
+export declare function toolsAvoided(output: SingleTurnRes | MultiturnRes, expected: EvalTarget | MultiturnEvalTarget): 1 | 0;
+export declare const toolCallOrder: (output: MultiturnRes, expected: MultiturnEvalTarget) => number;
+export declare const llmJudge: (output: MultiturnRes, target: MultiturnEvalTarget) => Promise<number>;
+//# sourceMappingURL=evaluators.d.ts.map

package/dist/agent/evals/evaluators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluators.d.ts","sourceRoot":"","sources":["../../../src/agent/evals/evaluators.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAI/F,wBAAgB,aAAa,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,SAOxE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,YAAY,EAAE,QAAQ,EAAE,UAAU,GAAG,mBAAmB,SAM5G;AAED,eAAO,MAAM,aAAa,GAAI,QAAQ,YAAY,EAAE,UAAU,mBAAmB,WAgBhF,CAAA;AAGD,eAAO,MAAM,QAAQ,GAAU,QAAQ,YAAY,EAAE,QAAQ,mBAAmB,oBA2B/E,CAAA"}

package/dist/agent/evals/evaluators.js

@@ -0,0 +1,59 @@
+import { generateText, Output } from "ai";
+import { openai } from "@ai-sdk/openai";
+import z from "zod";
+export function toolsSelected(output, expected) {
+    const expectedTools = expected.expectedTools;
+    if (!expectedTools?.length)
+        return 1;
+    const selected = new Set(output.toolNames);
+    return expectedTools.every((t) => selected.has(t)) ? 1 : 0;
+}
+export function toolsAvoided(output, expected) {
+    if (!expected.forbiddenTools?.length)
+        return 1;
+    const selected = new Set("toolNames" in output ? output.toolNames : output.toolsUsed);
+    return expected.forbiddenTools.some((t) => selected.has(t)) ? 0 : 1;
+}
+export const toolCallOrder = (output, expected) => {
+    const expectedOrder = expected.expectedToolOrder;
+    if (!expectedOrder?.length)
+        return 1;
+    const resultOrder = output.toolCallOrder;
+    let expectedIndex = 0;
+    for (const toolName of resultOrder) {
+        if (toolName === expectedOrder[expectedIndex]) {
+            expectedIndex++;
+            if (expectedIndex === expectedOrder.length)
+                break;
+        }
+    }
+    return expectedIndex / expectedOrder.length;
+};
+export const llmJudge = async (output, target) => {
+    const result = await generateText({
+        model: openai("gpt-5-mini"),
+        system: `You are an eval agent whose jon is return scores from 1-10 for the task performed by an ai agent.
+            Details provided by the user includes what the task was, the tools called by the agent and the results pd the tool calls along with the final text response generated.
+
+            Scoring criteria:
+            - 10: Response fully addresses the task using tool results correctly
+            - 7-9: Response is mostly correct with minor issues
+            - 4-6: Response partially addresses the task
+            - 1-3: Response is mostly incorrect or irrelevant
+            `,
+        prompt: `Task: ${target.originalTask}
+            Tools Called : ${JSON.stringify(output.toolsUsed)}
+            Tool Results Provided : ${JSON.stringify(target.mockToolResults)}
+
+            AT's final response: ${output.text}
+            `,
+        output: Output.object({
+            schema: z.object({
+                score: z.number().min(1).max(10).describe("Score from 1-10 where 10 is perfect"),
+                reason: z.string().describe("Brief explanation for the score"),
+            }),
+        }),
+    });
+    return result.output.score / 10;
+};
+//# sourceMappingURL=evaluators.js.map

package/dist/agent/evals/evaluators.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluators.js","sourceRoot":"","sources":["../../../src/agent/evals/evaluators.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,EAAqB,MAAM,IAAI,CAAC;AAE7D,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,CAAC,MAAM,KAAK,CAAC;AAEpB,MAAM,UAAU,aAAa,CAAC,MAAqB,EAAE,QAAoB;IACrE,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAA;IAC5C,IAAI,CAAC,aAAa,EAAE,MAAM;QAAE,OAAO,CAAC,CAAA;IAEpC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IAE1C,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AAC9D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAoC,EAAE,QAA0C;IACzG,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM;QAAE,OAAO,CAAC,CAAA;IAE9C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IAErF,OAAO,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AACvE,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,MAAoB,EAAE,QAA6B,EAAE,EAAE;IACjF,MAAM,aAAa,GAAG,QAAQ,CAAC,iBAAiB,CAAA;IAEhD,IAAI,CAAC,aAAa,EAAE,MAAM;QAAE,OAAO,CAAC,CAAC;IAErC,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAA;IAExC,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,KAAK,MAAM,QAAQ,IAAI,WAAW,EAAE,CAAC;QACjC,IAAI,QAAQ,KAAK,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;YAC5C,aAAa,EAAE,CAAC;YAChB,IAAI,aAAa,KAAK,aAAa,CAAC,MAAM;gBAAE,MAAK;QACrD,CAAC;IACL,CAAC;IAED,OAAO,aAAa,GAAG,aAAa,CAAC,MAAM,CAAA;AAC/C,CAAC,CAAA;AAGD,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,MAAoB,EAAE,MAA2B,EAAE,EAAE;IAChF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAC9B,KAAK,EAAE,MAAM,CAAC,YAAY,CAAC;QAC3B,MAAM,EAAE;;;;;;;;aAQH;QACL,MAAM,EAAE,SAAS,MAAM,CAAC,YAAY;6BACf,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC;sCACvB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC;;mCAEzC,MAAM,CAAC,IAAI;aACjC;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;YAClB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;gBACb,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,qCAAqC,CAAC;gBAChF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;aACjE,CAAC;SACL,CAAC;KACL,CAAC,CAAA;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,EAAE,CAAA;AACnC,CAAC,CAAA"}

package/dist/agent/evals/executors.d.ts

@@ -0,0 +1,31 @@
+import type { EvalData, MultiturnEvalData } from "./types.js";
+import z from "zod";
+export declare const TOOL_DEFINITIONS: Record<string, {
+    description: string;
+    inputSchema: z.ZodObject<z.ZodRawShape>;
+}>;
+export declare const singleTurnWithMocks: (data: EvalData) => Promise<{
+    toolCalls: {
+        toolName: string;
+        args: unknown;
+    }[];
+    toolNames: string[];
+    selectedAny: boolean;
+}>;
+export declare const multiturnWithMocks: (data: MultiturnEvalData) => Promise<{
+    text: string;
+    steps: {
+        toolCalls: {
+            toolName: string;
+            args: unknown;
+        }[] | undefined;
+        toolResults: {
+            toolName: string;
+            result: unknown;
+        }[] | undefined;
+        text: string | undefined;
+    }[];
+    toolsUsed: string[];
+    toolCallOrder: string[];
+}>;
+//# sourceMappingURL=executors.d.ts.map

package/dist/agent/evals/executors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executors.d.ts","sourceRoot":"","sources":["../../../src/agent/evals/executors.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE9D,OAAO,CAAC,MAAM,KAAK,CAAC;AAWpB,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAA;CAAE,CA2B7G,CAAA;AAGD,eAAO,MAAM,mBAAmB,GAAU,MAAM,QAAQ;;;;;;;EAkCvD,CAAA;AAED,eAAO,MAAM,kBAAkB,GAAU,MAAM,iBAAiB;;;;;;;;;;;;;;;EAmD/D,CAAA"}