@letta-ai/letta-code 0.27.18 → 0.27.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@letta-ai/letta-code",
-  "version": "0.27.18",
+  "version": "0.27.20",
   "description": "Letta Code is a CLI tool for interacting with stateful Letta agents from the terminal.",
   "type": "module",
   "packageManager": "bun@1.3.0",
@@ -47,6 +47,7 @@
     "@earendil-works/pi-ai": "^0.79.6",
     "@letta-ai/letta-client": "^1.10.2",
     "@pierre/diffs": "1.2.2",
+    "@scarf/scarf": "^1.4.0",
     "glob": "^13.0.0",
     "ink-link": "^5.0.0",
     "node-pty": "^1.1.0",
@@ -97,7 +98,8 @@
     "test:update-chain:manual": "bun run src/test-utils/update-chain-smoke.ts --mode manual",
     "test:update-chain:startup": "bun run src/test-utils/update-chain-smoke.ts --mode startup",
     "prepublishOnly": "bun run build",
-    "postinstall": "node scripts/postinstall-patches.js || echo letta: vendor patches skipped && node -e \"try{require('fs').chmodSync(require('path').join(require.resolve('node-pty/package.json'),'../prebuilds/darwin-arm64/spawn-helper'),0o755)}catch(e){}\" || true"
+    "postinstall": "node scripts/postinstall-patches.js || echo letta: vendor patches skipped && node -e \"try{require('fs').chmodSync(require('path').join(require.resolve('node-pty/package.json'),'../prebuilds/darwin-arm64/spawn-helper'),0o755)}catch(e){}\" || true",
+    "mod-learning:memory-citations": "bun scripts/mod-learning/learn-mod.ts --env docs/examples/mods/learning/memory-citations.env.json"
   },
   "lint-staged": {
     "*.{ts,tsx,js,jsx,json}": [

package/scripts/mod-learning/learn-mod.ts

@@ -0,0 +1,306 @@
+#!/usr/bin/env bun
+import { spawn } from "node:child_process";
+import { closeSync, openSync } from "node:fs";
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  defaultModLearningRunDirectory,
+  type ModLearningProgress,
+  type ModLearningReport,
+  readModLearningEnv,
+  runModLearning,
+} from "../../src/mods/learning-harness.ts";
+
+const DEFAULT_OPTIMIZATION_STEPS = 5;
+
+interface Args {
+  backend?: string;
+  candidate?: string;
+  candidateCount?: number;
+  candidateFileName?: string;
+  evalModel?: string;
+  generationModel?: string;
+  help: boolean;
+  env: string;
+  foreground: boolean;
+  out?: string;
+  promoteTo?: string;
+  repoRoot: string;
+  scenarioLimit?: number;
+  skipGeneration: boolean;
+}
+
+function readOptionValue(
+  argv: string[],
+  index: number,
+  optionName: string,
+): { nextIndex: number; value: string } {
+  const arg = argv[index] ?? "";
+  const equalsPrefix = `${optionName}=`;
+  if (arg.startsWith(equalsPrefix)) {
+    const value = arg.slice(equalsPrefix.length);
+    if (!value) throw new Error(`${optionName} requires a value`);
+    return { nextIndex: index, value };
+  }
+
+  const value = argv[index + 1];
+  if (!value || value.startsWith("--")) {
+    throw new Error(`${optionName} requires a value`);
+  }
+  return { nextIndex: index + 1, value };
+}
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = {
+    env: "docs/examples/mods/learning/memory-citations.env.json",
+    foreground: false,
+    help: false,
+    repoRoot: process.cwd(),
+    skipGeneration: false,
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--help" || arg === "-h") {
+      args.help = true;
+    } else if (arg === "--background") {
+      args.foreground = false;
+    } else if (arg === "--foreground") {
+      args.foreground = true;
+    } else if (arg === "--skip-generation") {
+      args.skipGeneration = true;
+    } else if (arg?.startsWith("--")) {
+      const optionName = arg.includes("=")
+        ? arg.slice(0, arg.indexOf("="))
+        : arg;
+      const { nextIndex, value } = readOptionValue(argv, index, optionName);
+      index = nextIndex;
+      switch (optionName) {
+        case "--backend":
+          args.backend = value;
+          break;
+        case "--candidate":
+          args.candidate = value;
+          break;
+        case "--candidates":
+          args.candidateCount = Number(value);
+          break;
+        case "--candidate-file-name":
+          args.candidateFileName = value;
+          break;
+        case "--eval-model":
+          args.evalModel = value;
+          break;
+        case "--env":
+          args.env = value;
+          break;
+        case "--generation-model":
+          args.generationModel = value;
+          break;
+        case "--model":
+          args.generationModel = value;
+          args.evalModel = value;
+          break;
+        case "--out":
+          args.out = value;
+          break;
+        case "--promote-to":
+          args.promoteTo = value;
+          break;
+        case "--repo-root":
+          args.repoRoot = value;
+          break;
+        case "--scenario-limit":
+          args.scenarioLimit = Number(value);
+          break;
+        default:
+          throw new Error(`Unknown argument: ${arg}`);
+      }
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+
+  return args;
+}
+
+function printHelp(): void {
+  console.log(`Usage: bun scripts/mod-learning/learn-mod.ts [options]
+
+Runs the mod learning dogfood loop:
+  env/demo -> generate candidate mod -> headless eval with LETTA_MODS_DIR (and legacy LETTA_EXTENSIONS_DIR for pre-rename branches) -> artifacts/report
+
+Options:
+  --env <path>                  Learning env JSON (default: memory-citations env)
+  --out <dir>                   Run artifact directory (default: .letta/mod-learning-runs/<slug>-<timestamp>)
+  --candidate <path>            Use an existing candidate mod instead of generation
+  --candidates <n>              Run N optimization iterations for one learned mod (default: 5 for generated runs)
+  --candidate-file-name <name>  Candidate filename inside the eval mod directory
+  --model <handle>              Model for generation and eval
+  --generation-model <handle>   Model for candidate generation
+  --eval-model <handle>         Model for headless eval
+  --backend <mode>              Backend flag forwarded to letta (api or local)
+  --scenario-limit <n>          Evaluate only the first N scenarios (fast smoke testing)
+  --repo-root <path>            Repo root (default: cwd)
+  --foreground                  Run learning in this process and return a pass/fail exit code
+  --background                  Explicitly use the default detached mode
+  --skip-generation             Expect the candidate file to already exist in the run dir
+  --promote-to <path>           Copy passing candidate to this repo-relative path
+  -h, --help                    Show this help
+`);
+}
+
+async function launchBackground(params: {
+  argv: string[];
+  repoRoot: string;
+  runDir: string;
+  outWasProvided: boolean;
+}): Promise<void> {
+  await mkdir(params.runDir, { recursive: true });
+  const stdoutPath = path.join(params.runDir, "background.stdout");
+  const stderrPath = path.join(params.runDir, "background.stderr");
+  const metadataPath = path.join(params.runDir, "background.json");
+  const childArgv = params.argv.filter(
+    (arg) => arg !== "--background" && arg !== "--foreground",
+  );
+  childArgv.push("--foreground");
+  if (!params.outWasProvided) childArgv.push("--out", params.runDir);
+
+  const stdoutFd = openSync(stdoutPath, "a");
+  const stderrFd = openSync(stderrPath, "a");
+  let childPid: number | undefined;
+  try {
+    const child = spawn(
+      process.execPath,
+      [fileURLToPath(import.meta.url), ...childArgv],
+      {
+        cwd: params.repoRoot,
+        detached: true,
+        env: process.env,
+        stdio: ["ignore", stdoutFd, stderrFd],
+      },
+    );
+    child.unref();
+    childPid = child.pid;
+    await writeFile(
+      metadataPath,
+      `${JSON.stringify(
+        {
+          args: childArgv,
+          command: process.execPath,
+          pid: childPid,
+          runDir: params.runDir,
+          startedAt: new Date().toISOString(),
+          stderrPath,
+          stdoutPath,
+        },
+        null,
+        2,
+      )}\n`,
+      "utf8",
+    );
+  } finally {
+    closeSync(stdoutFd);
+    closeSync(stderrFd);
+  }
+
+  console.log(`BACKGROUND ${params.runDir}`);
+  console.log(`PID ${childPid ?? "unknown"}`);
+  console.log(`stdout ${stdoutPath}`);
+  console.log(`stderr ${stderrPath}`);
+  console.log(`progress tail -f ${stdoutPath}`);
+  console.log(`report ${path.join(params.runDir, "report.md")}`);
+}
+
+function formatScore(
+  score: number | undefined,
+  maxScore: number | undefined,
+): string {
+  if (score === undefined) return "";
+  if (maxScore === undefined) return ` score ${score}`;
+  const percentage = maxScore > 0 ? Math.round((score / maxScore) * 100) : 0;
+  return ` score ${score}/${maxScore} (${percentage}%)`;
+}
+
+function progressPrefix(progress: ModLearningProgress): string {
+  if (progress.candidateIndex && progress.candidateCount) {
+    return `[${progress.candidateIndex}/${progress.candidateCount}]`;
+  }
+  if (progress.candidateIndex) return `[${progress.candidateIndex}]`;
+  return `[${progress.phase}]`;
+}
+
+function formatProgressLine(progress: ModLearningProgress): string {
+  return `${progressPrefix(progress)} ${progress.message}${formatScore(progress.score, progress.maxScore)}`;
+}
+
+function candidateForPromote(report: ModLearningReport): string {
+  return report.candidatePath;
+}
+
+async function main(): Promise<void> {
+  const argv = process.argv.slice(2);
+  const args = parseArgs(argv);
+  if (args.help) {
+    printHelp();
+    return;
+  }
+
+  const repoRoot = path.resolve(args.repoRoot);
+  const learningEnv = await readModLearningEnv(
+    path.resolve(repoRoot, args.env),
+  );
+  const runDir = args.out
+    ? path.resolve(repoRoot, args.out)
+    : path.resolve(repoRoot, defaultModLearningRunDirectory(learningEnv));
+
+  if (!args.foreground) {
+    await launchBackground({
+      argv,
+      outWasProvided: args.out !== undefined,
+      repoRoot,
+      runDir,
+    });
+    return;
+  }
+
+  let lastProgressLine = "";
+  const report = await runModLearning({
+    backend: args.backend,
+    candidateCount:
+      args.candidateCount ??
+      (args.candidate || args.skipGeneration
+        ? undefined
+        : DEFAULT_OPTIMIZATION_STEPS),
+    candidateFileName: args.candidateFileName,
+    candidateSourcePath: args.candidate,
+    evalModel: args.evalModel,
+    generationModel: args.generationModel,
+    promoteToPath: args.promoteTo,
+    repoRoot,
+    runDir,
+    scenarioLimit: args.scenarioLimit,
+    skipGeneration: args.skipGeneration,
+    spec: learningEnv,
+    onProgress: (progress) => {
+      const line = formatProgressLine(progress);
+      if (line === lastProgressLine) return;
+      lastProgressLine = line;
+      console.log(line);
+    },
+  });
+
+  const status = report.passed ? "PASS" : "FAIL";
+  console.log(`${status} ${report.reportPath}`);
+  console.log(`candidate ${candidateForPromote(report)}`);
+  if (report.passed) {
+    console.log(`promote letta mods promote ${candidateForPromote(report)}`);
+  }
+  if (!report.passed) process.exit(1);
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.stack : String(error));
+  process.exit(1);
+});

package/skills/{context_doctor → context-doctor}/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: Context Doctor
-id: context_doctor
+id: context-doctor
 description: Identify and repair degradation in system prompt, external memory, and skills preventing you from following instructions or remembering information as well as you should.
 ---
 

package/skills/creating-mods/SKILL.md

@@ -7,7 +7,7 @@ description: Creates and edits trusted local Letta Code mods, including tools, s
 
 Use this skill to create or update trusted Letta Code mod files. Mods are trusted local code that add small composable capabilities through mod APIs, not by importing app internals. Dynamic agent/conversation/workspace/model state is passed as `ctx` to tool, command, event, and permission callbacks (panels receive live `agent`/`model` in their render context); do not read mutable global context for model-callable behavior. Prefer scoped handles (`ctx.conversation`, `ctx.cwd`, `ctx.agent`) and guard optional UI with `letta.capabilities`.
 
-Capabilities vary by surface. TUI/headless may load tools, commands, events, UI, and providers; the desktop listener loads provider-only mods for local provider discovery. Always guard optional capabilities.
+Capabilities vary by surface — not every surface loads every capability. The TUI/headless host can load tools, commands, events, UI, and providers; the desktop listener loads tools, commands, providers, and tool/turn events, but not panel UI. Always guard each registration on the capabilities its behavior needs.
 
 ## Choose where the mod file lives
 
@@ -93,6 +93,8 @@ letta.capabilities.providers
 letta.capabilities.ui.panels
 ```
 
+Guard each registration on every capability its behavior depends on — not just the one that registers it. Surfaces load different capability subsets, so a registration that relies on another capability (a command that opens UI, emits an event, or calls a provider) must guard on that capability too. Otherwise it is advertised or activated on a host that cannot fulfill it and silently does nothing. Register where the host can actually do the work.
+
 ## Scoped API model
 
 - In commands and events, use `ctx.conversation` for conversation operations:
@@ -136,6 +138,7 @@ Before finishing, verify:
 - Tool descriptions explain when the model should call them.
 - JSON schemas are object schemas with useful descriptions.
 - Optional UI/event APIs are capability-guarded.
+- Each registration is guarded by every capability its behavior depends on, not just the one that registers it, so it isn't advertised or activated on a surface that can't fulfill it.
 - Provider mods are capability-guarded and clearly documented as local-agent only.
 - Timers, intervals, event registrations, and panels are cleaned up in a disposer.
 - Busy commands return `{ type: "handled" }` quickly and avoid main-conversation sends.

package/skills/creating-mods/references/events.md

@@ -59,7 +59,7 @@ letta.events.on("tool_start", (event, ctx) => {
 
 Lifecycle, turn, tool, compaction, and llm events are wired today.
 
-Lifecycle handlers are notification-only and should not return values. `turn_start` handlers can transform the outbound input for the next model turn. `tool_start` handlers can transform the tool arguments before execution. Compaction and llm handlers are notification-only.
+Lifecycle handlers are notification-only and should not return values. `turn_start` handlers can transform or cancel outbound user-message turns. `tool_start` handlers can transform the tool arguments before execution. Compaction and llm handlers are notification-only.
 
 `compact_start`/`compact_end` and `llm_start`/`llm_end` only fire on the **local backend**, where compaction and provider requests run client-side. On the constellation backend that work happens server-side and these events do not fire, so guard with `letta.capabilities.events.compact` / `letta.capabilities.events.llm` for portable mods.
 
@@ -155,16 +155,18 @@ Handlers run in registration order. Later handlers see the current args after ea
   conversationId: string | null;
   toolCallId: string | null;
   toolName: string;
+  args: Record<string, unknown>;
   status: "success" | "error";
   output: string;
 }
 ```
 
-`tool_end` fires immediately after a tool produces a result, before the agent sees it. Handlers can inspect the result, or return `{ result: { status, output } }` to replace it:
+`tool_end` fires immediately after a tool produces a result, before the agent sees it. `event.args` contains the effective tool invocation arguments after `tool_start` transforms, so handlers can react to the specific file, command, query, etc. Handlers can inspect the result, or return `{ result: { status, output } }` to replace it:
 
 ```ts
 letta.events.on("tool_end", (event) => {
   if (event.toolName !== "Bash" || event.status !== "success") return;
+  if (typeof event.args.command !== "string") return;
   return { result: { status: "success", output: redactSecrets(event.output) } };
 });
 ```
@@ -184,7 +186,7 @@ letta.events.on("tool_end", async (event, ctx) => {
 });
 ```
 
-`turn_start` fires before outbound turns that include a user message. In the TUI this includes normal submits and prompt-style command turns. In headless it includes one-shot prompts and bidirectional user turns.
+`turn_start` fires before outbound turns that include a user message. In the TUI this includes normal submits and prompt-style command turns. In headless it includes one-shot prompts and bidirectional user turns. Listener/Desktop skips approval-only continuations so mods do not rewrite approval payloads; do not rely on `turn_start` to block approval-only continuations on every surface.
 
 Handlers can mutate `event.input` directly or return replacement input:
 
@@ -212,6 +214,18 @@ letta.events.on("turn_start", (event) => {
 });
 ```
 
+Handlers can also cancel a user-message turn before it reaches the backend/model:
+
+```ts
+letta.events.on("turn_start", (event) => {
+  if (!isPlanModeActive(event.conversationId)) {
+    return { cancel: { reason: "Run /plan first." } };
+  }
+});
+```
+
+If multiple handlers cancel, the first valid cancel reason wins. A valid reason is a non-empty string after trimming. Cancellation does not synthesize an assistant response or tool result; it only tells the host not to submit this turn.
+
 Handlers run in registration order. Later handlers see the current input after earlier mutations/returns. If a handler throws, its partial `event.input` mutation is rolled back and the error is recorded as a mod diagnostic.
 
 `turn_start` is intentionally a trusted local mod point: it can rewrite user messages, approval results, and ordering. Keep transforms focused and unsurprising.
@@ -270,12 +284,18 @@ Handlers run in registration order. Later handlers see the current input after e
     promptTokens: number;
     completionTokens: number;
     totalTokens: number;
-  };
+  } | null;
   durationMs: number;
+  error?: {
+    message: string;
+    detail: string;
+    errorType: "llm_error" | "local_backend_error";
+    retryable: boolean;
+  };
 }
 ```
 
-`llm_end` fires once a provider request produces a final message, carrying the stop reason, token usage, and wall-clock duration. A request that fails before producing a final message (e.g. a transport error that triggers a retry) emits no `llm_end`. Both events are notification-only; return values are ignored. A throwing handler is isolated and never breaks the provider request.
+`llm_end` fires when a provider request ends, success or failure. Successful requests include token usage. Requests that fail before usage is available set `usage: null` and include `error`. Retry/failover effects are not supported yet; both events are notification-only and return values are ignored. A throwing handler is isolated and never breaks the provider request.
 
 Handlers also receive:
 

package/skills/creating-mods/references/ui.md

@@ -60,6 +60,29 @@ render(ctx: {
 
 Close panels when they are transient, and close/replace long-lived panels from the activation disposer if reload should remove them.
 
+### Commands that open panels
+
+If a command's `run()` opens a panel, guard the command **registration** on `letta.capabilities.ui.panels` — not just the `openPanel` call:
+
+```ts
+export default function activate(letta) {
+  if (!letta.capabilities.commands) return;
+  if (!letta.capabilities.ui.panels) return; // panel-only command: skip where panels are unsupported
+
+  return letta.commands.register({
+    id: "mycommand",
+    description: "…",
+    run() {
+      const panel = letta.ui.openPanel({ /* … */ });
+      // …
+      return { type: "handled" };
+    },
+  });
+}
+```
+
+Guarding only the `openPanel` call is not enough. The desktop listener has `commands: true` but `ui.panels: false`, so a command registered there is advertised in the command picker while its panel work no-ops — the user sees a command that runs and does nothing. Gating the registration keeps panel-only commands out of hosts that can't render them.
+
 ## Timers and cleanup
 
 ```ts

package/skills/generating-mod-envs/SKILL.md

@@ -0,0 +1,130 @@
+---
+name: generating-mod-envs
+description: Generates and reviews mod learning env JSON files for Letta Code local mods. Use when asked to teach, learn, or optimize a mod behavior; create, draft, validate, improve, or explain envs for `/mods learn --env`; or design evaluation scenarios, memory fixtures, requiredResultMarkers, requiredTraceMarkers, negative controls, and candidate diversity hints.
+disable-model-invocation: true
+user-invocable: true
+---
+
+# Generating mod learning envs
+
+Use this skill to create JSON envs consumed by `/mods learn --env=<path>` or `bun scripts/mod-learning/learn-mod.ts --env <path>`. An env describes the mod behavior to learn and the scenario-suite eval used to score candidates.
+
+## Workflow
+
+1. Define the behavior and eval before writing JSON.
+   - What should the mod do? Tool, turn event, tool event, provider, command, status, etc.
+   - What would a placebo/no-op mod fail?
+   - What unique sentinel strings make success unambiguous?
+2. Choose a path:
+   - Repo example: `docs/examples/mods/learning/<slug>.env.json`
+   - Local/private: any user-requested path
+3. Draft strict JSON. Start from `assets/mod-learning-env.template.json` if useful. No comments or trailing commas.
+4. Prefer `evaluation.scenarios` with at least:
+   - happy path
+   - discrimination/exact-target path
+   - negative control
+5. Validate:
+
+```bash
+bun src/skills/builtin/generating-mod-envs/scripts/validate-mod-env.ts path/to/env.json
+```
+
+If this skill is installed outside the source tree, run the same script from this skill directory: `scripts/validate-mod-env.ts`.
+
+6. If asked to run it:
+
+```text
+/mods learn --env=path/to/env.json --model=auto --backend=api --out=/tmp/<slug>-learn
+```
+
+The raw `scripts/mod-learning/learn-mod.ts` dev script detaches by default. Add `--foreground` only when a blocking pass/fail exit code is needed.
+
+Use single-line `--flag=value` commands for TUI instructions.
+
+## Env shape
+
+Required top-level fields:
+
+- `name`: human display name.
+- `slug`: stable kebab-case run/candidate slug.
+- `objective`: one-paragraph target for the generation agent.
+- `requirements`: concrete pass/fail behavior constraints.
+- `evaluation`: either a single prompt eval or a scenario suite.
+
+Common optional fields:
+
+- `targetModName`: display metadata for the intended mod filename. The harness still chooses the candidate filename from `slug` unless `--candidate-file-name` is passed.
+- `candidateDiversityHints`: strategies assigned across multi-candidate runs.
+- `modApiHints`: concise API reminders that prevent bad generated code.
+- `examples`: small input/expected demos for the generation prompt.
+
+Evaluation fields:
+
+- `evaluation.outputFormat`: use `stream-json` when checking trace markers.
+- `evaluation.timeoutMs`, `evaluation.maxTurns`: per-scenario defaults.
+- `evaluation.memoryFiles`: files seeded under eval `$MEMORY_DIR`.
+- `evaluation.scenarios[]`: scenario-specific overrides and fixtures.
+- In scenario-suite envs, do not add a top-level `evaluation.prompt` unless that prompt must run for every scenario. Assertion-only scenarios should have `assertions` and no `prompt`; only scenarios that require model behavior should define `scenario.prompt`.
+- `requiredResultMarkers`: literal strings required in the final answer.
+- `requiredTraceMarkers`: literal strings required in raw stdout/stderr.
+- `forbiddenResultMarkers`: final-answer strings that fail the run.
+- `forbiddenTraceMarkers`: raw trace strings that fail the run.
+
+## Quality rules
+
+- Design the eval first. A useful env distinguishes success from a no-op mod.
+- Use unique sentinels, e.g. `MY-MOD-CANARY-OK`, not common phrases.
+- Seed `memoryFiles` rather than depending on real user memory or repo files.
+- Include negative controls for non-use. If behavior should be conditional, verify it stays silent when not triggered.
+- Include discrimination scenarios when paths, IDs, or sources matter. Put a tempting wrong sentinel in an irrelevant fixture and forbid it in the final answer.
+- Put load failures in `forbiddenTraceMarkers`, usually:
+  - `[mods] failed to load`
+  - `[extensions] failed to load`
+  - `loaded 0 mod(s)`
+  - `loaded 0 extension(s)`
+- For eval-facing tools, require `requiresApproval: false`, `parallelSafe: true`, and a strict no-argument schema when applicable.
+- Avoid over-brittle trace markers. Prefer stable substrings like the tool name plus `"message_type":"tool_return_message"`.
+- Keep requirements behavioral; put fragile implementation details in `modApiHints` only when needed.
+
+## Minimal scenario-suite example
+
+```json
+{
+  "name": "Hello tool mod learner demo",
+  "slug": "hello-tool",
+  "objective": "Learn a trusted local mod that registers a read-only hello_mod_ping tool returning a fixed sentinel.",
+  "requirements": [
+    "Register a tool named hello_mod_ping.",
+    "The tool must accept no parameters, require no approval, be parallelSafe, and return the exact string HELLO-MOD-OK."
+  ],
+  "candidateDiversityHints": [
+    "Use the smallest possible tool-only implementation.",
+    "Add explicit defensive checks around the tool schema."
+  ],
+  "modApiHints": [
+    "Use export function activate(letta) or a default export.",
+    "Use letta.tools.register({ name, description, parameters, requiresApproval, parallelSafe, run }).",
+    "A no-argument tool schema is { \"type\": \"object\", \"properties\": {}, \"additionalProperties\": false }."
+  ],
+  "evaluation": {
+    "outputFormat": "stream-json",
+    "timeoutMs": 900000,
+    "maxTurns": 6,
+    "forbiddenTraceMarkers": ["[mods] failed to load", "loaded 0 mod(s)"],
+    "scenarios": [
+      {
+        "name": "happy-path",
+        "prompt": "Call the hello_mod_ping tool, then answer with the exact text HELLO-MOD-OK.",
+        "requiredResultMarkers": ["HELLO-MOD-OK"],
+        "requiredTraceMarkers": ["hello_mod_ping", "\"message_type\":\"tool_return_message\""]
+      },
+      {
+        "name": "negative-control",
+        "prompt": "Answer without calling tools: what is 2 + 2?",
+        "requiredResultMarkers": ["4"],
+        "forbiddenTraceMarkers": ["hello_mod_ping"]
+      }
+    ]
+  }
+}
+```

package/skills/generating-mod-envs/assets/mod-learning-env.template.json

@@ -0,0 +1,57 @@
+{
+  "name": "<Display name> mod learner demo",
+  "slug": "example-mod-env",
+  "targetModName": "example-mod-env.ts",
+  "objective": "Learn a trusted local mod that <specific behavior>.",
+  "requirements": [
+    "<Concrete behavior requirement>",
+    "<Concrete eval-facing requirement>"
+  ],
+  "candidateDiversityHints": [
+    "<Alternative implementation strategy for candidate 1>",
+    "<Alternative implementation strategy for candidate 2>"
+  ],
+  "modApiHints": [
+    "Use export function activate(letta) or a default export.",
+    "Use only the trusted local mod API exposed through the letta argument; do not import from repo source files."
+  ],
+  "examples": [
+    {
+      "input": "<Short demo input>",
+      "expected": "<Expected behavior>"
+    }
+  ],
+  "evaluation": {
+    "outputFormat": "stream-json",
+    "timeoutMs": 900000,
+    "maxTurns": 8,
+    "forbiddenTraceMarkers": [
+      "[mods] failed to load",
+      "[extensions] failed to load",
+      "loaded 0 mod(s)",
+      "loaded 0 extension(s)"
+    ],
+    "scenarios": [
+      {
+        "name": "happy-path",
+        "memoryFiles": {
+          "reference/<fixture>.md": "# Fixture\n\nThe sentinel is <SENTINEL-OK>.\n"
+        },
+        "prompt": "<Prompt that triggers the mod and requires the sentinel in the final answer>. The memory directory is available in MEMORY_DIR.",
+        "requiredResultMarkers": ["<SENTINEL-OK>"],
+        "requiredTraceMarkers": ["<stable trace marker>"],
+        "forbiddenResultMarkers": ["<known bad final answer marker>"]
+      },
+      {
+        "name": "negative-control",
+        "prompt": "<Prompt where the mod behavior should not trigger>.",
+        "requiredResultMarkers": ["<ordinary answer marker>"],
+        "forbiddenResultMarkers": ["<mod-only marker>"],
+        "forbiddenTraceMarkers": [
+          "<tool or event marker that should not appear>"
+        ]
+      }
+    ],
+    "prompt": "Run all configured evaluation.scenarios."
+  }
+}