@letta-ai/letta-code 0.27.17 → 0.27.19

package/skills/customizing-statusline/references/migration.md

@@ -1,6 +1,6 @@
 # Statusline Migration
 
-Use this reference when migrating legacy command statuslines, standalone `.sh` statusline scripts, or shell PS1 prompts.
+Use this reference when migrating legacy command statuslines, standalone `.sh` statusline scripts, or shell PS1 prompts into `~/.letta/mods/statusline.tsx`.
 
 ## Legacy Letta command statusline
 
@@ -40,10 +40,10 @@ When migrating:
 
 - Preserve old config and referenced files unless the user explicitly asks to delete them.
 - If `command` references a `.sh` file, read it before writing the new mod.
-- Translate polling (`refreshIntervalMs`) to `setInterval`.
-- Translate direct command output into cached status plus synchronous rendering.
-- If the command output used `\x1e` to split left/right output, convert it to internal full-row layout with `Box`; do not create a new left/right API.
-- Treat old prompt customization separately. The new statusline controls the bottom row, not necessarily the input prompt.
+- Translate polling (`refreshIntervalMs`) to `setInterval` + `panel.update()`.
+- Translate direct command output into a cached closure variable plus synchronous rendering.
+- If the command output used `\x1e` to split left/right output, convert it to `row(left, right, width)`; there is no separate left/right API.
+- Treat old prompt customization separately. The statusline controls the primary row, not necessarily the input prompt.
 
 Old model:
 
@@ -59,17 +59,36 @@ import { promisify } from "node:util";
 
 const execFileAsync = promisify(execFile);
 
-const update = async () => {
-  const { stdout } = await execFileAsync("git", ["branch", "--show-current"], {
-    cwd: process.cwd(),
+export default function activate(letta) {
+  if (!letta.capabilities.ui.panels) return;
+
+  let branch = "";
+
+  const panel = letta.ui.openPanel({
+    id: "statusline",
+    order: 0,
+    render: ({ width, row }) => row(branch, "", width),
   });
-  letta.ui.setStatus("branch", stdout.trim());
-};
 
-letta.ui.setStatuslineRenderer((context) => {
-  const { Text } = context.components;
-  return <Text>{context.statuses.branch ?? ""}</Text>;
-});
+  const update = async () => {
+    try {
+      const { stdout } = await execFileAsync("git", ["branch", "--show-current"], {
+        cwd: process.cwd(),
+      });
+      branch = stdout.trim();
+    } catch {
+      branch = "";
+    }
+    panel.update();
+  };
+
+  void update();
+  const timer = setInterval(update, 30_000);
+  return () => {
+    clearInterval(timer);
+    panel.close();
+  };
+}
 ```
 
 ## Standalone `.sh` file migration
@@ -79,11 +98,11 @@ If the user provides a `.sh` path:
 1. Read the script.
 2. Identify commands, expected stdin JSON, environment variables, and output shape.
 3. Port shell commands to async setup/update code.
-4. Store results with `letta.ui.setStatus(key, value)`.
-5. Render cached status synchronously.
-6. Preserve graceful fallbacks for missing tools, not-a-git-repo, no PR, etc.
+4. Store results in closure variables.
+5. Render cached state synchronously and call `panel.update()` after each refresh.
+6. Preserve graceful fallbacks for missing tools, not-a-git-repo, no PR, etc. (return `""` to hide the line).
 
-If a script depends heavily on stdin JSON, use `context.rawPayload` as a temporary migration aid, but prefer semantic context fields for new code.
+Map the script's stdin JSON to the render context's semantic fields (`agent`, `model`, `width`); compute anything else in setup/update code.
 
 ## Shell PS1 import
 
@@ -127,4 +146,4 @@ If no PS1 is found and the user did not provide other instructions, ask for one
 2. a description of what their prompt shows
 3. the current prompt output as it appears in their terminal
 
-Preserve colors where practical using display components. If the PS1 is too dynamic to port exactly, ask whether to approximate it or port specific commands.
+Preserve colors where practical with `chalk`. If the PS1 is too dynamic to port exactly, ask whether to approximate it or port specific commands.

package/skills/generating-mod-envs/SKILL.md

@@ -0,0 +1,130 @@
+---
+name: generating-mod-envs
+description: Generates and reviews mod learning env JSON files for Letta Code local mods. Use when asked to teach, learn, or optimize a mod behavior; create, draft, validate, improve, or explain envs for `/mods learn --env`; or design evaluation scenarios, memory fixtures, requiredResultMarkers, requiredTraceMarkers, negative controls, and candidate diversity hints.
+disable-model-invocation: true
+user-invocable: true
+---
+
+# Generating mod learning envs
+
+Use this skill to create JSON envs consumed by `/mods learn --env=<path>` or `bun scripts/mod-learning/learn-mod.ts --env <path>`. An env describes the mod behavior to learn and the scenario-suite eval used to score candidates.
+
+## Workflow
+
+1. Define the behavior and eval before writing JSON.
+   - What should the mod do? Tool, turn event, tool event, provider, command, status, etc.
+   - What would a placebo/no-op mod fail?
+   - What unique sentinel strings make success unambiguous?
+2. Choose a path:
+   - Repo example: `docs/examples/mods/learning/<slug>.env.json`
+   - Local/private: any user-requested path
+3. Draft strict JSON. Start from `assets/mod-learning-env.template.json` if useful. No comments or trailing commas.
+4. Prefer `evaluation.scenarios` with at least:
+   - happy path
+   - discrimination/exact-target path
+   - negative control
+5. Validate:
+
+```bash
+bun src/skills/builtin/generating-mod-envs/scripts/validate-mod-env.ts path/to/env.json
+```
+
+If this skill is installed outside the source tree, run the same script from this skill directory: `scripts/validate-mod-env.ts`.
+
+6. If asked to run it:
+
+```text
+/mods learn --env=path/to/env.json --model=auto --backend=api --out=/tmp/<slug>-learn
+```
+
+The raw `scripts/mod-learning/learn-mod.ts` dev script detaches by default. Add `--foreground` only when a blocking pass/fail exit code is needed.
+
+Use single-line `--flag=value` commands for TUI instructions.
+
+## Env shape
+
+Required top-level fields:
+
+- `name`: human display name.
+- `slug`: stable kebab-case run/candidate slug.
+- `objective`: one-paragraph target for the generation agent.
+- `requirements`: concrete pass/fail behavior constraints.
+- `evaluation`: either a single prompt eval or a scenario suite.
+
+Common optional fields:
+
+- `targetModName`: display metadata for the intended mod filename. The harness still chooses the candidate filename from `slug` unless `--candidate-file-name` is passed.
+- `candidateDiversityHints`: strategies assigned across multi-candidate runs.
+- `modApiHints`: concise API reminders that prevent bad generated code.
+- `examples`: small input/expected demos for the generation prompt.
+
+Evaluation fields:
+
+- `evaluation.outputFormat`: use `stream-json` when checking trace markers.
+- `evaluation.timeoutMs`, `evaluation.maxTurns`: per-scenario defaults.
+- `evaluation.memoryFiles`: files seeded under eval `$MEMORY_DIR`.
+- `evaluation.scenarios[]`: scenario-specific overrides and fixtures.
+- In scenario-suite envs, do not add a top-level `evaluation.prompt` unless that prompt must run for every scenario. Assertion-only scenarios should have `assertions` and no `prompt`; only scenarios that require model behavior should define `scenario.prompt`.
+- `requiredResultMarkers`: literal strings required in the final answer.
+- `requiredTraceMarkers`: literal strings required in raw stdout/stderr.
+- `forbiddenResultMarkers`: final-answer strings that fail the run.
+- `forbiddenTraceMarkers`: raw trace strings that fail the run.
+
+## Quality rules
+
+- Design the eval first. A useful env distinguishes success from a no-op mod.
+- Use unique sentinels, e.g. `MY-MOD-CANARY-OK`, not common phrases.
+- Seed `memoryFiles` rather than depending on real user memory or repo files.
+- Include negative controls for non-use. If behavior should be conditional, verify it stays silent when not triggered.
+- Include discrimination scenarios when paths, IDs, or sources matter. Put a tempting wrong sentinel in an irrelevant fixture and forbid it in the final answer.
+- Put load failures in `forbiddenTraceMarkers`, usually:
+  - `[mods] failed to load`
+  - `[extensions] failed to load`
+  - `loaded 0 mod(s)`
+  - `loaded 0 extension(s)`
+- For eval-facing tools, require `requiresApproval: false`, `parallelSafe: true`, and a strict no-argument schema when applicable.
+- Avoid over-brittle trace markers. Prefer stable substrings like the tool name plus `"message_type":"tool_return_message"`.
+- Keep requirements behavioral; put fragile implementation details in `modApiHints` only when needed.
+
+## Minimal scenario-suite example
+
+```json
+{
+  "name": "Hello tool mod learner demo",
+  "slug": "hello-tool",
+  "objective": "Learn a trusted local mod that registers a read-only hello_mod_ping tool returning a fixed sentinel.",
+  "requirements": [
+    "Register a tool named hello_mod_ping.",
+    "The tool must accept no parameters, require no approval, be parallelSafe, and return the exact string HELLO-MOD-OK."
+  ],
+  "candidateDiversityHints": [
+    "Use the smallest possible tool-only implementation.",
+    "Add explicit defensive checks around the tool schema."
+  ],
+  "modApiHints": [
+    "Use export function activate(letta) or a default export.",
+    "Use letta.tools.register({ name, description, parameters, requiresApproval, parallelSafe, run }).",
+    "A no-argument tool schema is { \"type\": \"object\", \"properties\": {}, \"additionalProperties\": false }."
+  ],
+  "evaluation": {
+    "outputFormat": "stream-json",
+    "timeoutMs": 900000,
+    "maxTurns": 6,
+    "forbiddenTraceMarkers": ["[mods] failed to load", "loaded 0 mod(s)"],
+    "scenarios": [
+      {
+        "name": "happy-path",
+        "prompt": "Call the hello_mod_ping tool, then answer with the exact text HELLO-MOD-OK.",
+        "requiredResultMarkers": ["HELLO-MOD-OK"],
+        "requiredTraceMarkers": ["hello_mod_ping", "\"message_type\":\"tool_return_message\""]
+      },
+      {
+        "name": "negative-control",
+        "prompt": "Answer without calling tools: what is 2 + 2?",
+        "requiredResultMarkers": ["4"],
+        "forbiddenTraceMarkers": ["hello_mod_ping"]
+      }
+    ]
+  }
+}
+```

package/skills/generating-mod-envs/assets/mod-learning-env.template.json

@@ -0,0 +1,57 @@
+{
+  "name": "<Display name> mod learner demo",
+  "slug": "example-mod-env",
+  "targetModName": "example-mod-env.ts",
+  "objective": "Learn a trusted local mod that <specific behavior>.",
+  "requirements": [
+    "<Concrete behavior requirement>",
+    "<Concrete eval-facing requirement>"
+  ],
+  "candidateDiversityHints": [
+    "<Alternative implementation strategy for candidate 1>",
+    "<Alternative implementation strategy for candidate 2>"
+  ],
+  "modApiHints": [
+    "Use export function activate(letta) or a default export.",
+    "Use only the trusted local mod API exposed through the letta argument; do not import from repo source files."
+  ],
+  "examples": [
+    {
+      "input": "<Short demo input>",
+      "expected": "<Expected behavior>"
+    }
+  ],
+  "evaluation": {
+    "outputFormat": "stream-json",
+    "timeoutMs": 900000,
+    "maxTurns": 8,
+    "forbiddenTraceMarkers": [
+      "[mods] failed to load",
+      "[extensions] failed to load",
+      "loaded 0 mod(s)",
+      "loaded 0 extension(s)"
+    ],
+    "scenarios": [
+      {
+        "name": "happy-path",
+        "memoryFiles": {
+          "reference/<fixture>.md": "# Fixture\n\nThe sentinel is <SENTINEL-OK>.\n"
+        },
+        "prompt": "<Prompt that triggers the mod and requires the sentinel in the final answer>. The memory directory is available in MEMORY_DIR.",
+        "requiredResultMarkers": ["<SENTINEL-OK>"],
+        "requiredTraceMarkers": ["<stable trace marker>"],
+        "forbiddenResultMarkers": ["<known bad final answer marker>"]
+      },
+      {
+        "name": "negative-control",
+        "prompt": "<Prompt where the mod behavior should not trigger>.",
+        "requiredResultMarkers": ["<ordinary answer marker>"],
+        "forbiddenResultMarkers": ["<mod-only marker>"],
+        "forbiddenTraceMarkers": [
+          "<tool or event marker that should not appear>"
+        ]
+      }
+    ],
+    "prompt": "Run all configured evaluation.scenarios."
+  }
+}

package/skills/generating-mod-envs/scripts/validate-mod-env.ts

@@ -0,0 +1,261 @@
+#!/usr/bin/env bun
+import { readFile } from "node:fs/promises";
+
+type JsonRecord = Record<string, unknown>;
+
+function isRecord(value: unknown): value is JsonRecord {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isStringArray(value: unknown): value is string[] {
+  return (
+    Array.isArray(value) && value.every((item) => typeof item === "string")
+  );
+}
+
+function assertValid(condition: unknown, message: string): asserts condition {
+  if (!condition) throw new Error(message);
+}
+
+function optionalString(record: JsonRecord, key: string, path = key): void {
+  const value = record[key];
+  assertValid(
+    value === undefined ||
+      (typeof value === "string" && value.trim().length > 0),
+    `${path} must be a non-empty string when present`,
+  );
+}
+
+function optionalStringArray(
+  record: JsonRecord,
+  key: string,
+  path = key,
+): void {
+  const value = record[key];
+  assertValid(
+    value === undefined || isStringArray(value),
+    `${path} must be an array of strings`,
+  );
+}
+
+function optionalPositiveNumber(
+  record: JsonRecord,
+  key: string,
+  path = key,
+): void {
+  const value = record[key];
+  assertValid(
+    value === undefined ||
+      (typeof value === "number" && Number.isFinite(value) && value > 0),
+    `${path} must be a positive number`,
+  );
+}
+
+function validateMemoryFiles(value: unknown, path: string): void {
+  if (value === undefined) return;
+  assertValid(isRecord(value), `${path} must be an object`);
+  for (const [filePath, content] of Object.entries(value)) {
+    assertValid(
+      filePath.length > 0 && !filePath.startsWith("/"),
+      `${path} paths must be relative`,
+    );
+    assertValid(
+      typeof content === "string",
+      `${path}[${filePath}] must be a string`,
+    );
+  }
+}
+
+function validateMarkerFields(record: JsonRecord, path: string): void {
+  optionalStringArray(
+    record,
+    "requiredResultMarkers",
+    `${path}.requiredResultMarkers`,
+  );
+  optionalStringArray(
+    record,
+    "requiredTraceMarkers",
+    `${path}.requiredTraceMarkers`,
+  );
+  optionalStringArray(
+    record,
+    "forbiddenResultMarkers",
+    `${path}.forbiddenResultMarkers`,
+  );
+  optionalStringArray(
+    record,
+    "forbiddenTraceMarkers",
+    `${path}.forbiddenTraceMarkers`,
+  );
+}
+
+function requiredMarkerCount(record: JsonRecord): number {
+  return ["requiredResultMarkers", "requiredTraceMarkers"].reduce(
+    (count, key) => {
+      const value = record[key];
+      return count + (Array.isArray(value) ? value.length : 0);
+    },
+    0,
+  );
+}
+
+function assertionCount(record: JsonRecord): number {
+  const value = record.assertions;
+  return Array.isArray(value) ? value.length : 0;
+}
+
+function validateScenario(params: {
+  evaluation: JsonRecord;
+  index: number;
+  scenario: unknown;
+  warnings: string[];
+}): void {
+  const path = `evaluation.scenarios[${params.index}]`;
+  assertValid(isRecord(params.scenario), `${path} must be an object`);
+  const scenario = params.scenario;
+
+  optionalString(scenario, "name", `${path}.name`);
+  optionalString(scenario, "prompt", `${path}.prompt`);
+  assertValid(
+    typeof scenario.prompt === "string" ||
+      typeof params.evaluation.prompt === "string" ||
+      assertionCount(scenario) > 0 ||
+      assertionCount(params.evaluation) > 0,
+    `${path}.prompt or ${path}.assertions is required when evaluation.prompt is absent`,
+  );
+  assertValid(
+    scenario.outputFormat === undefined ||
+      scenario.outputFormat === "json" ||
+      scenario.outputFormat === "stream-json",
+    `${path}.outputFormat must be json or stream-json`,
+  );
+  optionalPositiveNumber(scenario, "timeoutMs", `${path}.timeoutMs`);
+  optionalPositiveNumber(scenario, "maxTurns", `${path}.maxTurns`);
+  validateMemoryFiles(scenario.memoryFiles, `${path}.memoryFiles`);
+  validateMarkerFields(scenario, path);
+
+  if (
+    requiredMarkerCount(scenario) === 0 &&
+    requiredMarkerCount(params.evaluation) === 0
+  ) {
+    params.warnings.push(
+      `${path} has no required result/trace markers and no parent required markers`,
+    );
+  }
+}
+
+function validateEnv(env: unknown): string[] {
+  assertValid(isRecord(env), "env must be a JSON object");
+  assertValid(
+    typeof env.name === "string" && env.name.trim(),
+    "name is required",
+  );
+  assertValid(
+    typeof env.objective === "string" && env.objective.trim(),
+    "objective is required",
+  );
+  assertValid(
+    isStringArray(env.requirements) && env.requirements.length > 0,
+    "requirements must be a non-empty array of strings",
+  );
+
+  optionalString(env, "slug");
+  optionalString(env, "targetModName");
+  optionalStringArray(env, "candidateDiversityHints");
+  optionalStringArray(env, "modApiHints");
+
+  if (env.examples !== undefined) {
+    assertValid(Array.isArray(env.examples), "examples must be an array");
+    for (const [index, example] of env.examples.entries()) {
+      assertValid(isRecord(example), `examples[${index}] must be an object`);
+      assertValid(
+        typeof example.input === "string" && example.input.trim(),
+        `examples[${index}].input is required`,
+      );
+      assertValid(
+        example.expected === undefined || typeof example.expected === "string",
+        `examples[${index}].expected must be a string`,
+      );
+      assertValid(
+        example.notes === undefined || typeof example.notes === "string",
+        `examples[${index}].notes must be a string`,
+      );
+    }
+  }
+
+  assertValid(isRecord(env.evaluation), "evaluation is required");
+  const evaluation = env.evaluation;
+  optionalString(evaluation, "prompt", "evaluation.prompt");
+  assertValid(
+    evaluation.outputFormat === undefined ||
+      evaluation.outputFormat === "json" ||
+      evaluation.outputFormat === "stream-json",
+    "evaluation.outputFormat must be json or stream-json",
+  );
+  optionalPositiveNumber(evaluation, "timeoutMs", "evaluation.timeoutMs");
+  optionalPositiveNumber(evaluation, "maxTurns", "evaluation.maxTurns");
+  validateMemoryFiles(evaluation.memoryFiles, "evaluation.memoryFiles");
+  validateMarkerFields(evaluation, "evaluation");
+
+  const warnings: string[] = [];
+  if (evaluation.scenarios !== undefined) {
+    assertValid(
+      Array.isArray(evaluation.scenarios),
+      "evaluation.scenarios must be an array",
+    );
+    assertValid(
+      evaluation.scenarios.length > 0,
+      "evaluation.scenarios must not be empty",
+    );
+    for (const [index, scenario] of evaluation.scenarios.entries()) {
+      validateScenario({ evaluation, index, scenario, warnings });
+    }
+  } else {
+    assertValid(
+      typeof evaluation.prompt === "string" && evaluation.prompt.trim(),
+      "evaluation.prompt is required when no scenarios are configured",
+    );
+    if (requiredMarkerCount(evaluation) === 0) {
+      warnings.push(
+        "add at least one requiredResultMarkers or requiredTraceMarkers entry so the eval can fail a placebo mod",
+      );
+    }
+  }
+
+  if (
+    typeof env.slug === "string" &&
+    !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(env.slug)
+  ) {
+    warnings.push(
+      "slug should be kebab-case for stable run and candidate paths",
+    );
+  }
+  if (
+    evaluation.scenarios !== undefined &&
+    Array.isArray(evaluation.scenarios) &&
+    evaluation.scenarios.length < 2
+  ) {
+    warnings.push(
+      "prefer at least a happy path and a negative-control scenario",
+    );
+  }
+  return warnings;
+}
+
+async function main(): Promise<void> {
+  const envPath = process.argv[2];
+  if (!envPath || envPath === "--help" || envPath === "-h") {
+    console.error("Usage: bun validate-mod-env.ts path/to/env.json");
+    process.exit(envPath ? 0 : 1);
+  }
+
+  const env = JSON.parse(await readFile(envPath, "utf8"));
+  const warnings = validateEnv(env);
+  console.log(`OK ${envPath}`);
+  for (const warning of warnings) console.warn(`warning: ${warning}`);
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});