automatey 0.1.0

package/.env.defaults

@@ -0,0 +1,100 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# automatey — default configuration
+#
+# Copy relevant lines to .env.secret (or set as real env vars) to override.
+# These values are also readable by the ConfigManager as JSON at:
+#   ~/.automatey/config.json
+#
+# See .env.secret.example for provider API key examples.
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+# ── LLM Provider ─────────────────────────────────────────────────────────────
+# Which provider to use: nemotron | openai | anthropic | perplexity
+LLM_PROVIDER=nemotron
+
+# The model identifier passed to the API
+LLM_MODEL=nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4
+
+# Comma-separated list of selectable models (shown in /model command)
+# LLM_MODELS=nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4,gpt-4o,claude-opus-4-5
+
+# API key — required for openai / anthropic / perplexity; optional for nemotron (local)
+# LLM_API_KEY=sk-...
+#
+# Provider-specific API keys (read from environment or .env.secret):
+# OPENAI_API_KEY=sk-...
+# ANTHROPIC_API_KEY=sk-ant-...
+# PERPLEXITY_API_KEY=pplx-...
+
+
+# ── nemotron / OpenAI-compatible endpoint ────────────────────────────────────
+# Base URL for the vLLM / OpenAI-compatible server (nemotron provider only)
+LLM_BASE_URL=http://localhost:8002
+
+
+# ── Token budgets ─────────────────────────────────────────────────────────────
+# Context window size (characters are estimated at 4 chars/token)
+# The context manager trims oldest messages when this is exceeded.
+LLM_MAX_TOKENS=200000
+
+# Per-call output token limit when thinking is OFF
+LLM_MAX_OUTPUT_TOKENS=4096
+
+# Per-call output token limit when thinking is ON (must be > LLM_THINKING_BUDGET)
+# Effective limit = max(LLM_MAX_OUTPUT_TOKENS_THINK, LLM_THINKING_BUDGET + 4096)
+LLM_MAX_OUTPUT_TOKENS_THINK=8192
+
+# Token budget for chain-of-thought reasoning (thinking=true)
+LLM_THINKING_BUDGET=4096
+
+
+# ── Sampling ──────────────────────────────────────────────────────────────────
+TEMPERATURE=0.1
+
+
+# ── Agentic loop ──────────────────────────────────────────────────────────────
+# Maximum number of tool-call rounds before aborting with a warning
+AGENT_MAX_TOOL_ROUNDS=20
+
+# Maximum empty-response retries before giving up
+AGENT_MAX_EMPTY_RETRIES=2
+
+# Context usage ratio (0.0–1.0) that triggers auto-compact summarization.
+# Set to 1.0 to disable auto-compact.
+AGENT_COMPACT_THRESHOLD=0.8
+
+
+# ── Sessions ──────────────────────────────────────────────────────────────────
+# Directory for saved sessions (default: ~/.automatey/sessions)
+# SESSION_DIR=/path/to/sessions
+
+
+# ── MCP ───────────────────────────────────────────────────────────────────────
+# Full MCP server config used by the activate_tools mechanism.
+# Default: ~/.automatey/mcp.json
+# MCP_CONFIG=/path/to/mcp.json
+
+# Startup-subset config (servers to connect at launch; rest deferred).
+# Search order: {cwd}/.automatey/mcp.chat.json → ~/.automatey/mcp.chat.json
+# MCP_CHAT_CONFIG=/path/to/mcp.chat.json
+
+
+# ── UI ────────────────────────────────────────────────────────────────────────
+# Show token count in prompt: true | false
+UI_SHOW_TOKENS=true
+
+# Syntax-highlight code blocks in responses: true | false
+UI_SYNTAX_HIGHLIGHT=true
+
+
+# ── Eval / LLM-as-judge ───────────────────────────────────────────────────────
+# Provider for the LLM judge used in /eval runs.
+# Supported: openai | anthropic | perplexity
+# EVAL_JUDGE_PROVIDER=perplexity
+
+# Model identifier for the judge (must belong to EVAL_JUDGE_PROVIDER)
+# EVAL_JUDGE_MODEL=sonar-pro
+
+# When to invoke the judge: always | on-failure | never
+# EVAL_JUDGE_MODE=on-failure

package/.env.secret.example

@@ -0,0 +1,42 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# automatey — secret keys example
+#
+# Copy this file to .env.secret and fill in your actual API keys.
+# .env.secret is gitignored and will never be committed.
+#
+# Usage: the agent loads .env.defaults first, then .env.secret overrides.
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+# ── OpenAI ───────────────────────────────────────────────────────────────────
+# Required when LLM_PROVIDER=openai
+# Get yours at: https://platform.openai.com/api-keys
+OPENAI_API_KEY=sk-...
+
+
+# ── Anthropic ────────────────────────────────────────────────────────────────
+# Required when LLM_PROVIDER=anthropic
+# Get yours at: https://console.anthropic.com/settings/keys
+ANTHROPIC_API_KEY=sk-ant-...
+
+
+# ── Perplexity ───────────────────────────────────────────────────────────────
+# Required when LLM_PROVIDER=perplexity OR EVAL_JUDGE_PROVIDER=perplexity
+# Get yours at: https://www.perplexity.ai/settings/api
+PERPLEXITY_API_KEY=pplx-...
+
+
+# ── Brave Search ─────────────────────────────────────────────────────────────
+# Required for the brave-search MCP server tool
+# Get yours at: https://api.search.brave.com/app/keys
+BRAVE_API_KEY=BSA...
+
+
+# ── LLM-as-judge (eval) ───────────────────────────────────────────────────────
+# Optional — enables automatic verdict generation in /eval runs.
+# EVAL_JUDGE_PROVIDER can be: openai | anthropic | perplexity
+# EVAL_JUDGE_MODE can be: always | on-failure | never
+#
+# EVAL_JUDGE_PROVIDER=perplexity
+# EVAL_JUDGE_MODEL=sonar-pro
+# EVAL_JUDGE_MODE=on-failure

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Max Golov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,258 @@
+<p align="center">
+  <img src="extra/logo/automatey-term+scanlines.svg" alt="Automatey" width="220" />
+</p>
+
+<h1 align="center">⚓ Automatey</h1>
+<p align="center"><em>Yer lean &amp; mean agentic helper — no fluff, all action, a touch o' pirate.</em></p>
+
+<p align="center">
+  <img alt="tests" src="https://img.shields.io/badge/tests-124%20passing-brightgreen?style=flat-square">
+  <img alt="node" src="https://img.shields.io/badge/node-%3E%3D20-blue?style=flat-square">
+  <img alt="license" src="https://img.shields.io/badge/license-MIT-blue?style=flat-square">
+</p>
+
+---
+
+**Automatey** is a minimal, MCP-powered CLI agent that lets any LLM wield real tools — file ops, shell commands, web search, memory, planning — through a clean ReAct loop. No bloat. No framework lock-in. Just a sharp hook and a fast ship.
+
+```
+══════════════════════════════════════════════════════════════════════
+  🤖  automatey — lean & mean agent
+══════════════════════════════════════════════════════════════════════
+  Provider:  nemotron
+  Model:     nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4
+  Session:   session-2026-03-12
+  Sandbox:   /workspace/my-project/sandbox
+
+  Type /help for commands  |  Ctrl+C to exit
+  Logs: tail -f ~/.automatey/logs/agent.log
+══════════════════════════════════════════════════════════════════════
+```
+
+
+## Features
+
+- **Providers**: Nemotron (vLLM/OpenAI-compatible), OpenAI, Anthropic
+- **MCP tools**: Any stdio or HTTP MCP server; auto-loaded from `./mcp.json` or `~/.automatey/mcp.json`
+- **ReAct loop**: Up to 20 tool-call rounds per message (configurable)
+- **Chain-of-thought**: `/think` toggle for Nemotron / Anthropic reasoning tokens
+- **Sandbox**: Isolated directory for agent file I/O and code execution (`./sandbox` default, gitignored)
+- **Sessions**: Save/load conversation sessions in `~/.automatey/sessions/`
+- **Checkpoints**: `/checkpoint` — full conversation snapshots with BM25 keyword search
+- **Auto-compact**: LLM summarises older context when usage ≥ 80%
+- **Skills**: Progressive SKILL.md loading from `.agents/skills/`
+- **Planner MCP**: Bundled `mcp/planner` — todos + plans
+- **Coder MCP**: Bundled `mcp/coder` — read/write/edit files, run commands, glob files
+
+## Quick Start
+
+```bash
+git clone https://github.com/maxgolov/automatey.git
+cd automatey
+npm install
+cp mcp.example.json mcp.json   # edit to add your API keys
+npm run build
+node dist/index.js chat
+```
+
+Override provider, model, and sandbox:
+
+```bash
+node dist/index.js chat --provider openai --model gpt-4o --sandbox ./my-sandbox
+```
+
+## CLI Installation
+
+### Option A — `npm link` (recommended)
+
+```bash
+npm run build
+npm run link:cli   # registers "automatey" globally via symlink
+```
+
+After linking:
+```bash
+automatey chat
+automatey --help
+```
+
+Unlink: `npm run unlink:cli`
+
+### Option B — Shell alias
+
+```bash
+alias automatey="node /workspace/simple-agent/dist/index.js"
+```
+
+> JSON / JSONC in `mcp.json`: Standard JSON does not allow `//` comments.
+> The agent uses a JSONC parser — `//` line comments and `/* */` block comments
+> are fully supported in all `mcp.json` files.
+
+## MCP Config — `mcp.json`
+
+The agent looks for MCP config in this order:
+1. `./mcp.json` (current working directory / project root) 
+2. `~/.automatey/mcp.json` (global fallback)
+
+Copy `mcp.example.json` from this repo as your starting point:
+
+```bash
+cp mcp.example.json mcp.json          # project-local
+# OR
+cp mcp.example.json ~/.automatey/mcp.json  # global
+```
+
+Example `mcp.json`:
+
+```jsonc
+{
+  // JSONC is supported — '//' and '/* */' comments are stripped before parsing
+  "mcpServers": {
+    "planner": {
+      "type": "stdio",
+      "command": "node",
+      "args": ["./mcp/planner/dist/server.js"]
+    },
+    "brave-search": {
+      "type": "stdio",
+      "command": "npx",
+      "args": ["-y", "@brave/brave-search-mcp-server", "--transport", "stdio"],
+      "env": { "BRAVE_API_KEY": "${env:BRAVE_API_KEY}" },
+      "requiresEnv": "BRAVE_API_KEY"
+    },
+    "memento": {
+      "type": "http",
+      "url": "http://localhost:3500/mcp",
+      "portCheck": true
+    }
+  }
+}
+```
+
+Conditional loading:
+- `requiresEnv` — skip server if env var is missing (no key = no server, no error)
+- `portCheck: true` — skip HTTP/SSE server if the URL is unreachable at startup (1 s probe)
+
+## Config
+
+Config lives in `~/.automatey/config.json` (auto-created on first run):
+
+```json
+{
+  "provider": "nemotron",
+  "llm": {
+    "baseUrl": "http://localhost:8002",
+    "model": "nvidia/Llama-3.1-Nemotron-Nano-8B-v1"
+  }
+}
+```
+
+Environment variables (copy `.env.defaults` → `.env` to override):
+
+| Variable | Default | Description |
+|---|---|---|
+| `LLM_PROVIDER` | `nemotron` | `nemotron` \| `openai` \| `anthropic` |
+| `LLM_MODEL` | Nemotron NVFP4 | Model ID |
+| `LLM_BASE_URL` | `http://localhost:8002` | vLLM endpoint |
+| `TEMPERATURE` | `0.1` | Sampling temperature |
+| `AGENT_MAX_TOOL_ROUNDS` | `20` | Max ReAct rounds |
+| `AGENT_COMPACT_THRESHOLD` | `0.8` | Auto-compact at 80% context fill |
+
+## Commands
+
+| Command | Description |
+|---------|-------------|
+| `/help` | Show all commands |
+| `/model` | List / switch model |
+| `/think [on\|off\|budget N]` | Toggle CoT reasoning |
+| `/save [name]` | Save session |
+| `/load [name]` | Load session |
+| `/servers` | Manage MCP connections |
+| `/config` | Show config |
+| `/cost` | Show estimated token usage |
+| `/compact` | Manually compact context via LLM summarization |
+| `/checkpoint [save\|list\|restore N\|search q\|delete N]` | Manage checkpoints |
+| `/clear` | Clear context |
+| `/exit` | Quit |
+
+## Context Management
+
+### Auto-compact
+When estimated context usage reaches `AGENT_COMPACT_THRESHOLD` (default 80%), the older portion of the conversation is automatically summarized by the LLM and replaced with a concise summary message. This keeps the token count manageable without discarding knowledge.
+
+Disable per-session: the `/compact` command can be used to trigger compaction manually at any time.
+
+### Checkpoints
+
+Checkpoints are full conversation snapshots saved to `~/.automatey/checkpoints/` as JSON:
+
+```
+/checkpoint save              # save current conversation
+/checkpoint list              # list checkpoints (newest first)
+/checkpoint restore 2         # restore checkpoint #2 into context
+/checkpoint search "bm25"     # BM25 keyword search across all checkpoints
+/checkpoint delete 3          # delete checkpoint #3
+```
+
+The BM25 search indexes the full message history of every checkpoint and ranks them by keyword relevance.
+
+## Built-in MCP Servers
+
+### 🗂 Coder (`mcp/coder`)
+| Tool | What it does |
+|------|-------------|
+| `read_file` | Read file contents with optional line range |
+| `write_file` | Write / create a file |
+| `edit_file` | Replace an exact string in a file |
+| `execute_command` | Run a shell command (default cwd: sandbox) |
+| `search_text` | Grep-style text search |
+| `list_dir` | List directory contents |
+| `glob_files` | Find files by glob pattern (`**/*.ts`, `src/**`) |
+
+### 📋 Planner (`mcp/planner`)
+Todos and multi-step plans persisted to `~/.automatey/planner/`.
+
+## Development
+
+```bash
+npm run dev          # tsx watch (no build needed)
+npm run build        # tsc + build all MCP servers
+npm test             # 124 tests (Vitest)
+npm run test:watch   # watch mode
+```
+
+**VS Code tasks** (Ctrl+Shift+B / Ctrl+Shift+P → Run Task):
+- `Build: All`
+- `Run: CLI (automatey)` — builds first, then launches
+- `Run: CLI (dev, no build)` — tsx, faster iteration
+- `Test: All`
+- `Test: Hello World (verbose)`
+
+## Tests
+
+```
+Tests  124 passed
+  ├── unit/           command-parser, context-manager, config, session,
+  │                   mcp-config, llm-client, skills, coder-server
+  └── integration/    openai, anthropic, nemotron, planner,
+                      coder-hello-world (all 3 providers)
+```
+
+The `coder-hello-world` integration tests drive a full ReAct loop per provider —
+LLM writes `index.js`, executes it, output is verified. Results live in `sandbox/<provider>/`:
+
+```
+[openai]    execute_command output: Hello, World!
+[anthropic] execute_command output: Hello, World!
+[nemotron]  execute_command output: Hello, World!
+```
+
+## Artwork
+
+Logos in `extra/logo/` are from the [Automatey](https://github.com/top-5/automatey) terminal project,  
+licensed **CC BY 4.0** — Copyright © 2024–2025 Top-5 And Contributors.  
+Used here with attribution as permitted by the license.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

package/dist/adapters/index.d.ts

@@ -0,0 +1,3 @@
+export { LLMClient } from './llm-client.js';
+export type { LLMClientConfig } from './llm-client.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/adapters/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/adapters/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/adapters/index.js

@@ -0,0 +1,3 @@
+// Adapters barrel
+export { LLMClient } from './llm-client.js';
+//# sourceMappingURL=index.js.map

package/dist/adapters/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/adapters/index.ts"],"names":[],"mappings":"AAAA,kBAAkB;AAClB,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/adapters/judge-client.d.ts

@@ -0,0 +1,59 @@
+/**
+ * LLM-as-Judge — uses generateObject() with a structured Zod schema to score
+ * eval responses with a second LLM.
+ *
+ * Env vars:
+ *   EVAL_JUDGE_PROVIDER  — perplexity | openai | anthropic | nemotron
+ *   EVAL_JUDGE_MODEL     — model name (defaults per provider below)
+ *   EVAL_JUDGE_MODE      — on-failure (default) | always
+ *
+ * Default models per provider:
+ *   perplexity → sonar-pro   (web-grounded, includes citations)
+ *   openai     → gpt-4o-mini
+ *   anthropic  → claude-haiku-4-5
+ *   nemotron   → (same as primary)
+ */
+import { z } from 'zod';
+import type { LLMProvider } from '#core/interfaces';
+export declare const JudgeVerdictSchema: z.ZodObject<{
+    verdict: z.ZodEnum<["correct", "incorrect", "uncertain"]>;
+    reasoning: z.ZodString;
+    confidence: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    reasoning: string;
+    verdict: "correct" | "incorrect" | "uncertain";
+    confidence: number;
+}, {
+    reasoning: string;
+    verdict: "correct" | "incorrect" | "uncertain";
+    confidence: number;
+}>;
+export type JudgeVerdict = z.infer<typeof JudgeVerdictSchema> & {
+    /** Cited URLs from Perplexity responses (when EVAL_JUDGE_PROVIDER=perplexity). */
+    sources?: string[];
+};
+export type JudgeMode = 'never' | 'on-failure' | 'always';
+export interface JudgeConfig {
+    provider: LLMProvider;
+    model: string;
+    mode: JudgeMode;
+    /** Base URL (needed for nemotron/openai-compatible). */
+    baseUrl?: string;
+    /** API key override — falls back to env-var per provider. */
+    apiKey?: string;
+}
+/**
+ * Build a JudgeConfig from environment variables.
+ * Returns null when EVAL_JUDGE_PROVIDER is not set (judge disabled).
+ */
+export declare function judgeConfigFromEnv(overrideBaseUrl?: string): JudgeConfig | null;
+/**
+ * Judge a single eval result.
+ * @param config  Judge configuration (provider, model, mode).
+ * @param question  The original question asked.
+ * @param expectedAnswer  The expected/reference answer.
+ * @param actualAnswer  The actual response from the primary LLM.
+ * @returns JudgeVerdict (includes `sources[]` for Perplexity judges).
+ */
+export declare function judgeResult(config: JudgeConfig, question: string, expectedAnswer: string, actualAnswer: string): Promise<JudgeVerdict>;
+//# sourceMappingURL=judge-client.d.ts.map

package/dist/adapters/judge-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-client.d.ts","sourceRoot":"","sources":["../../src/adapters/judge-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAOH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAIpD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;EAI7B,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,GAAG;IAC9D,kFAAkF;IAClF,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,YAAY,GAAG,QAAQ,CAAC;AAE1D,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,WAAW,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,CAAC;IAChB,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAaD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CA0B/E;AAyCD;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,YAAY,CAAC,CA4BvB"}

package/dist/adapters/judge-client.js

@@ -0,0 +1,123 @@
+/**
+ * LLM-as-Judge — uses generateObject() with a structured Zod schema to score
+ * eval responses with a second LLM.
+ *
+ * Env vars:
+ *   EVAL_JUDGE_PROVIDER  — perplexity | openai | anthropic | nemotron
+ *   EVAL_JUDGE_MODEL     — model name (defaults per provider below)
+ *   EVAL_JUDGE_MODE      — on-failure (default) | always
+ *
+ * Default models per provider:
+ *   perplexity → sonar-pro   (web-grounded, includes citations)
+ *   openai     → gpt-4o-mini
+ *   anthropic  → claude-haiku-4-5
+ *   nemotron   → (same as primary)
+ */
+import { createAnthropic } from '@ai-sdk/anthropic';
+import { createOpenAI } from '@ai-sdk/openai';
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+import { createPerplexity } from '@ai-sdk/perplexity';
+import { generateObject } from 'ai';
+import { z } from 'zod';
+// ── Types ─────────────────────────────────────────────────────────────────────
+export const JudgeVerdictSchema = z.object({
+    verdict: z.enum(['correct', 'incorrect', 'uncertain']),
+    reasoning: z.string(),
+    confidence: z.number().min(0).max(1),
+});
+// ── Defaults ──────────────────────────────────────────────────────────────────
+const DEFAULT_MODELS = {
+    perplexity: 'sonar-pro',
+    openai: 'gpt-4o-mini',
+    anthropic: 'claude-haiku-4-5',
+    nemotron: 'nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16',
+};
+// ── Config factory ────────────────────────────────────────────────────────────
+/**
+ * Build a JudgeConfig from environment variables.
+ * Returns null when EVAL_JUDGE_PROVIDER is not set (judge disabled).
+ */
+export function judgeConfigFromEnv(overrideBaseUrl) {
+    const provider = process.env.EVAL_JUDGE_PROVIDER;
+    if (!provider)
+        return null;
+    const validProviders = ['perplexity', 'openai', 'anthropic', 'nemotron'];
+    if (!validProviders.includes(provider)) {
+        throw new Error(`EVAL_JUDGE_PROVIDER="${provider}" is not valid. Choose from: ${validProviders.join(', ')}`);
+    }
+    const model = process.env.EVAL_JUDGE_MODEL ?? DEFAULT_MODELS[provider];
+    const rawMode = process.env.EVAL_JUDGE_MODE ?? 'on-failure';
+    const validModes = ['never', 'on-failure', 'always'];
+    if (!validModes.includes(rawMode)) {
+        throw new Error(`EVAL_JUDGE_MODE="${rawMode}" is not valid. Choose from: ${validModes.join(', ')}`);
+    }
+    return {
+        provider,
+        model,
+        mode: rawMode,
+        baseUrl: process.env.LLM_BASE_URL ?? overrideBaseUrl,
+    };
+}
+// ── Model builder ─────────────────────────────────────────────────────────────
+function buildJudgeModel(config) {
+    switch (config.provider) {
+        case 'perplexity':
+            return createPerplexity({
+                apiKey: config.apiKey ?? process.env.PERPLEXITY_API_KEY ?? '',
+            })(config.model);
+        case 'openai':
+            return createOpenAI({
+                apiKey: config.apiKey ?? process.env.OPENAI_API_KEY ?? '',
+            })(config.model);
+        case 'anthropic':
+            return createAnthropic({
+                apiKey: config.apiKey ?? process.env.ANTHROPIC_API_KEY ?? '',
+            })(config.model);
+        case 'nemotron': {
+            const baseUrl = (config.baseUrl ?? 'http://localhost:8002').replace(/\/$/, '');
+            const provider = createOpenAICompatible({
+                name: 'vllm-judge',
+                baseURL: `${baseUrl}/v1`,
+                ...(config.apiKey ? { headers: { Authorization: `Bearer ${config.apiKey}` } } : {}),
+            });
+            return provider(config.model);
+        }
+    }
+}
+// ── Judge function ────────────────────────────────────────────────────────────
+const JUDGE_SYSTEM_PROMPT = 'You are an impartial evaluator judging whether an AI assistant answered a question correctly. ' +
+    'Analyze the expected answer and the actual response. ' +
+    'Return a JSON object with: verdict ("correct" / "incorrect" / "uncertain"), ' +
+    'a brief reasoning string, and a confidence score between 0 and 1.';
+/**
+ * Judge a single eval result.
+ * @param config  Judge configuration (provider, model, mode).
+ * @param question  The original question asked.
+ * @param expectedAnswer  The expected/reference answer.
+ * @param actualAnswer  The actual response from the primary LLM.
+ * @returns JudgeVerdict (includes `sources[]` for Perplexity judges).
+ */
+export async function judgeResult(config, question, expectedAnswer, actualAnswer) {
+    const model = buildJudgeModel(config);
+    const prompt = `Question: ${question}\n\n` +
+        `Expected answer: ${expectedAnswer}\n\n` +
+        `Actual response: ${actualAnswer}`;
+    const result = await generateObject({
+        model,
+        schema: JudgeVerdictSchema,
+        system: JUDGE_SYSTEM_PROMPT,
+        prompt,
+        maxRetries: 0,
+    });
+    const verdict = { ...result.object };
+    // Perplexity includes web citations — surface them in the verdict.
+    const meta = result.experimental_providerMetadata;
+    const sources = meta?.perplexity?.sources
+        ?.map((s) => s?.url)
+        .filter((u) => typeof u === 'string');
+    if (sources && sources.length > 0) {
+        verdict.sources = sources;
+    }
+    return verdict;
+}
+//# sourceMappingURL=judge-client.js.map

package/dist/adapters/judge-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-client.js","sourceRoot":"","sources":["../../src/adapters/judge-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,iFAAiF;AAEjF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;IACtD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACrC,CAAC,CAAC;AAmBH,iFAAiF;AAEjF,MAAM,cAAc,GAAgC;IAClD,UAAU,EAAE,WAAW;IACvB,MAAM,EAAE,aAAa;IACrB,SAAS,EAAE,kBAAkB;IAC7B,QAAQ,EAAE,4CAA4C;CACvD,CAAC;AAEF,iFAAiF;AAEjF;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,eAAwB;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,mBAA8C,CAAC;IAC5E,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,MAAM,cAAc,GAAkB,CAAC,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;IACxF,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,wBAAwB,QAAQ,gCAAgC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5F,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,YAAY,CAAC;IAC5D,MAAM,UAAU,GAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;IAClE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAoB,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,oBAAoB,OAAO,gCAAgC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ;QACR,KAAK;QACL,IAAI,EAAE,OAAoB;QAC1B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,eAAe;KACrD,CAAC;AACJ,CAAC;AAED,iFAAiF;AAEjF,SAAS,eAAe,CAAC,MAAmB;IAC1C,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,YAAY;YACf,OAAO,gBAAgB,CAAC;gBACtB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE;aAC9D,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEnB,KAAK,QAAQ;YACX,OAAO,YAAY,CAAC;gBAClB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;aAC1D,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEnB,KAAK,WAAW;YACd,OAAO,eAAe,CAAC;gBACrB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;aAC7D,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEnB,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,uBAAuB,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC/E,MAAM,QAAQ,GAAG,sBAAsB,CAAC;gBACtC,IAAI,EAAE,YAAY;gBAClB,OAAO,EAAE,GAAG,OAAO,KAAK;gBACxB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACpF,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;AACH,CAAC;AAED,iFAAiF;AAEjF,MAAM,mBAAmB,GACvB,gGAAgG;IAChG,uDAAuD;IACvD,8EAA8E;IAC9E,mEAAmE,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAmB,EACnB,QAAgB,EAChB,cAAsB,EACtB,YAAoB;IAEpB,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAEtC,MAAM,MAAM,GACV,aAAa,QAAQ,MAAM;QAC3B,oBAAoB,cAAc,MAAM;QACxC,oBAAoB,YAAY,EAAE,CAAC;IAErC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,KAAK;QACL,MAAM,EAAE,kBAAkB;QAC1B,MAAM,EAAE,mBAAmB;QAC3B,MAAM;QACN,UAAU,EAAE,CAAC;KACd,CAAC,CAAC;IAEH,MAAM,OAAO,GAAiB,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IAEnD,mEAAmE;IACnE,MAAM,IAAI,GAAI,MAAgH,CAAC,6BAA6B,CAAC;IAC7J,MAAM,OAAO,GAAG,IAAI,EAAE,UAAU,EAAE,OAAO;QACvC,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC;SACnB,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;IACrD,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC;IAC5B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}