@letta-ai/letta-code 0.27.18 → 0.27.20

package/docs/examples/mods/learning/memory-citations.env.json

@@ -0,0 +1,165 @@
+{
+  "name": "Memory citation mod with wiki-link [[path]] format",
+  "slug": "memory-citations",
+  "targetModName": "memory-citations.ts",
+  "objective": "Learn a trusted local mod that causes the agent to cite its memory sources inline using the Wikipedia-style format [[path]] whenever the agent's response is clearly influenced by content from a memory file. The mod must inject a turn_start system reminder describing the citation policy, track memory-file reads via tool_start events, and expose a memory_citation_snapshot tool so the agent can see which memory paths it accessed before answering.",
+  "requirements": [
+    "On turn_start, inject a system message reminding the agent to cite each memory block that clearly influenced its response using the exact inline format [[path]], where path is the memory-relative path (e.g. [[reference/notes.md]]). The reminder must state: only cite when memory clearly influenced the answer, never invent citations, and only cite paths returned by the memory_citation_snapshot tool. The reminder must NOT instruct the agent to call memory_citation_snapshot unconditionally — the agent should only call it when it has actually read memory files during the turn.",
+    "On tool_start, observe tool calls whose arguments reference files inside the active memory directory. Treat Read/Glob/Grep/LS-style tools as high-confidence reads. Treat Bash/ShellCommand/exec_command as medium-confidence when the command string references $MEMORY_DIR or the absolute memory path.",
+    "Use ctx.memfs.memoryDir when available, falling back to process.env.MEMORY_DIR.",
+    "Register a read-only tool named memory_citation_snapshot with no parameters, requiresApproval false, parallelSafe true. It returns JSON with citationCount and a citations array containing path, confidence, evidence, toolName, toolCallId, and observedAt.",
+    "Reset per-conversation citation state at the start of each turn.",
+    "Never invent citations. The snapshot tool must only return paths actually observed from tool_start events.",
+    "The turn_start reminder must include a concrete example of the citation format such as [[reference/notes.md]]."
+  ],
+  "candidateDiversityHints": [
+    "Use a conservative reminder and make memory_citation_snapshot the sole source of citeable paths; prefer simple Map-backed per-conversation state.",
+    "Focus on robust path normalization for absolute MEMORY_DIR paths, $MEMORY_DIR, ${MEMORY_DIR}, and memory-relative paths.",
+    "Focus on negative controls: avoid over-eager citations when no memory path was observed or when the answer does not draw on memory.",
+    "Use a minimal implementation with explicit tool-name allowlists and deduplication by memory-relative path.",
+    "Focus on shell-command evidence parsing while keeping read/list/search tools high-confidence and shell references medium-confidence."
+  ],
+  "modApiHints": [
+    "The mod runtime accepts TypeScript files in LETTA_MODS_DIR and calls export function activate(letta) or a default export.",
+    "letta.events.on(\"turn_start\", handler) can mutate event.input or return a replacement input.",
+    "letta.events.on(\"tool_start\", handler) receives event.toolName, event.args, event.toolCallId, and event.conversationId.",
+    "letta.tools.register({ name, description, parameters, requiresApproval, parallelSafe, run }) registers a tool available to the model.",
+    "For eval compatibility, the snapshot tool must be named exactly memory_citation_snapshot.",
+    "For turn_start reminders, append a separate { type: \"message\", role: \"system\", content: reminder } object to event.input. Do not concatenate onto an existing message content field because content may be structured parts.",
+    "For tool run handlers, ctx.conversation.id is the active conversation id; use it to retrieve state keyed by event.conversationId.",
+    "Avoid fragile generated regex strings with embedded quote characters; simple string matching/splitting is fine. The candidate must be syntactically valid TypeScript."
+  ],
+  "examples": [
+    {
+      "input": "The agent reads reference/notes.md with a Read tool, then answers a question drawing on that file's content.",
+      "expected": "The answer includes the inline citation [[reference/notes.md]] somewhere in the response."
+    },
+    {
+      "input": "The agent runs Bash with cat $MEMORY_DIR/reference/mod-learning.md, then answers using information from that file.",
+      "expected": "The answer includes the inline citation [[reference/mod-learning.md]] somewhere in the response."
+    },
+    {
+      "input": "The agent reads a non-memory file like /etc/hosts.",
+      "expected": "No memory citation is included for that file since it is not in the memory directory."
+    },
+    {
+      "input": "The agent reads a memory file but the answer does not draw on its content.",
+      "expected": "No memory citation is included because the memory did not clearly influence the response."
+    }
+  ],
+  "evaluation": {
+    "outputFormat": "stream-json",
+    "timeoutMs": 900000,
+    "maxTurns": 8,
+    "forbiddenTraceMarkers": [
+      "[mods] failed to load",
+      "[extensions] failed to load",
+      "loaded 0 mod(s)",
+      "loaded 0 extension(s)"
+    ],
+    "scenarios": [
+      {
+        "name": "mod-loads",
+        "assertions": [
+          {
+            "type": "mod_loads",
+            "expectedLoadedCount": 1
+          }
+        ]
+      },
+      {
+        "name": "turn-start-reminder",
+        "assertions": [
+          {
+            "type": "turn_start_injects_message",
+            "role": "system",
+            "contains": ["[[", "]]", "cite", "memory"],
+            "notContains": ["unconditionally", "always call"],
+            "input": [
+              {
+                "type": "message",
+                "role": "user",
+                "content": "What is the deployment target for this project?"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "tool-start-tracks-read",
+        "assertions": [
+          {
+            "type": "tool_start_rewrites_args",
+            "toolName": "Read",
+            "args": {
+              "file_path": "/tmp/mem-test/reference/notes.md"
+            },
+            "expectArgs": {
+              "file_path": "/tmp/mem-test/reference/notes.md"
+            }
+          }
+        ]
+      },
+      {
+        "name": "tool-start-preserves-non-memory-read",
+        "assertions": [
+          {
+            "type": "tool_start_preserves_args",
+            "toolName": "Read",
+            "args": {
+              "file_path": "/etc/hosts"
+            }
+          }
+        ]
+      },
+      {
+        "name": "implicit-memory-citation",
+        "memoryFiles": {
+          "reference/mod-learning.md": "# Project configuration\n\nThe deployment target for this project is CITATION-DOGFOOD-OK. All releases must be tagged with this target name before pushing to the registry.\n"
+        },
+        "prompt": "What is the deployment target for this project? Read the file $MEMORY_DIR/reference/mod-learning.md to find the answer.",
+        "requiredResultMarkers": [
+          "CITATION-DOGFOOD-OK",
+          "[[reference/mod-learning.md]]"
+        ],
+        "requiredTraceMarkers": ["memory_citation_snapshot"],
+        "forbiddenResultMarkers": [
+          "No explicit memory file reads were observed",
+          "memory_citation_snapshot tool is not available"
+        ]
+      },
+      {
+        "name": "exact-relevant-path",
+        "memoryFiles": {
+          "reference/mod-learning.md": "# Deployment target\n\nThe deployment target is NOT-THIS-FILE. This file is intentionally irrelevant for the release-channel question.\n",
+          "reference/release-policy.md": "# Release policy\n\nThe release channel for the project is CANARY-WIKI-LINK. Use this channel for staged rollouts.\n"
+        },
+        "prompt": "What is the release channel for this project? Read the files under $MEMORY_DIR/reference/ to find the answer.",
+        "requiredResultMarkers": [
+          "CANARY-WIKI-LINK",
+          "[[reference/release-policy.md]]"
+        ],
+        "requiredTraceMarkers": ["memory_citation_snapshot"],
+        "forbiddenResultMarkers": [
+          "[[reference/mod-learning.md]]",
+          "NOT-THIS-FILE"
+        ]
+      },
+      {
+        "name": "negative-control-no-memory-use",
+        "memoryFiles": {
+          "reference/mod-learning.md": "# Project configuration\n\nThe deployment target for this project is CITATION-DOGFOOD-OK.\n"
+        },
+        "prompt": "Answer without checking memory: what is 2 + 2?",
+        "requiredResultMarkers": ["4"],
+        "forbiddenResultMarkers": [
+          "[[",
+          "reference/mod-learning.md",
+          "Memory references:",
+          "CITATION-DOGFOOD-OK"
+        ],
+        "forbiddenTraceMarkers": ["\"tool_name\":\"memory_citation_snapshot\""]
+      }
+    ]
+  }
+}

package/docs/examples/mods/learning/uv-pip-install.env.json

@@ -0,0 +1,199 @@
+{
+  "name": "Use uv instead of pip for Python package installs",
+  "slug": "uv-pip-install",
+  "targetModName": "uv-pip-install.ts",
+  "objective": "Learn a trusted local mod that ensures the agent uses uv pip install instead of pip install or python -m pip install when installing Python packages. The mod must inject a turn_start reminder and rewrite Bash tool_start command args for pip install forms. It must not rewrite non-install pip subcommands or non-Python package managers.",
+  "requirements": [
+    "On turn_start, inject a system message reminding the agent to use uv pip install instead of pip install or python -m pip install when installing Python packages.",
+    "On tool_start, intercept Bash tool calls whose command argument contains a pip install command and rewrite it to use uv pip install instead.",
+    "Rewrite 'pip install <package>' to 'uv pip install <package>', preserving all flags and arguments after install.",
+    "Rewrite 'python -m pip install <package>' and 'python3 -m pip install <package>' to 'uv pip install <package>', preserving all flags and arguments after install.",
+    "Do NOT rewrite non-install pip subcommands such as pip list, pip show, pip freeze, pip uninstall, pip check, or pip config.",
+    "Do NOT rewrite commands that use other package managers such as npm install, bun install, pnpm install, cargo install, apt install, or brew install.",
+    "The mod must load without errors."
+  ],
+  "candidateDiversityHints": [
+    "Use a conservative command parser or regex that only rewrites pip install and python -m pip install forms.",
+    "Handle pip install at command boundaries and after simple shell operators like &&, ||, and ; if possible.",
+    "Keep the turn_start reminder concise and specific; the deterministic tool_start assertions are the source of truth."
+  ],
+  "modApiHints": [
+    "The mod runtime accepts TypeScript files in LETTA_MODS_DIR and calls export function activate(letta) or a default export.",
+    "letta.events.on(\"turn_start\", handler) receives event.input (an array of messages). Append a separate { type: \"message\", role: \"system\", content: reminder } object; do not concatenate onto existing content because it may be structured parts.",
+    "letta.events.on(\"tool_start\", handler) receives event.toolName, event.args, event.toolCallId, and event.conversationId.",
+    "The Bash tool is named \"Bash\" and its command argument is event.args.command (a string).",
+    "To rewrite args, return { args: { ...event.args, command: rewrittenCommand } } from the tool_start handler.",
+    "Guard event registration behind letta.capabilities.events.turns and letta.capabilities.events.tools checks.",
+    "Avoid fragile generated regex strings with embedded quote characters; simple string matching/splitting is fine. The candidate must be syntactically valid TypeScript."
+  ],
+  "examples": [
+    {
+      "input": "Bash tool with command=\"pip install requests\"",
+      "expected": "Rewritten to command=\"uv pip install requests\""
+    },
+    {
+      "input": "Bash tool with command=\"python -m pip install --upgrade numpy\"",
+      "expected": "Rewritten to command=\"uv pip install --upgrade numpy\""
+    },
+    {
+      "input": "Bash tool with command=\"pip list\"",
+      "expected": "Not rewritten \u2014 pip list is not an install command"
+    },
+    {
+      "input": "Bash tool with command=\"npm install express\"",
+      "expected": "Not rewritten \u2014 npm is not pip"
+    },
+    {
+      "input": "Bash tool with command=\"pip install -r requirements.txt && echo done\"",
+      "expected": "Rewritten to command=\"uv pip install -r requirements.txt && echo done\""
+    }
+  ],
+  "evaluation": {
+    "scenarios": [
+      {
+        "name": "mod-loads",
+        "assertions": [
+          {
+            "type": "mod_loads",
+            "expectedLoadedCount": 1
+          }
+        ]
+      },
+      {
+        "name": "turn-start-reminder",
+        "assertions": [
+          {
+            "type": "turn_start_injects_message",
+            "role": "system",
+            "contains": ["uv pip install", "pip install"],
+            "input": [
+              {
+                "type": "message",
+                "role": "user",
+                "content": "Install the Python requests library."
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "single-package-tool-rewrite",
+        "assertions": [
+          {
+            "type": "tool_start_rewrites_args",
+            "toolName": "Bash",
+            "args": {
+              "command": "pip install --dry-run requests"
+            },
+            "expectArgs": {
+              "command": "uv pip install --dry-run requests"
+            }
+          }
+        ]
+      },
+      {
+        "name": "python-module-tool-rewrite",
+        "assertions": [
+          {
+            "type": "tool_start_rewrites_args",
+            "toolName": "Bash",
+            "args": {
+              "command": "python -m pip install --dry-run --upgrade numpy"
+            },
+            "expectArgs": {
+              "command": "uv pip install --dry-run --upgrade numpy"
+            }
+          },
+          {
+            "type": "tool_start_rewrites_args",
+            "toolName": "Bash",
+            "args": {
+              "command": "python3 -m pip install --dry-run --upgrade numpy"
+            },
+            "expectArgs": {
+              "command": "uv pip install --dry-run --upgrade numpy"
+            }
+          }
+        ]
+      },
+      {
+        "name": "multiple-packages-tool-rewrite",
+        "assertions": [
+          {
+            "type": "tool_start_rewrites_args",
+            "toolName": "Bash",
+            "args": {
+              "command": "pip install --dry-run fastapi uvicorn"
+            },
+            "expectArgs": {
+              "command": "uv pip install --dry-run fastapi uvicorn"
+            }
+          }
+        ]
+      },
+      {
+        "name": "shell-operator-tool-rewrite",
+        "assertions": [
+          {
+            "type": "tool_start_rewrites_args",
+            "toolName": "Bash",
+            "args": {
+              "command": "pip install -r requirements.txt && echo done"
+            },
+            "expectArgs": {
+              "command": "uv pip install -r requirements.txt && echo done"
+            }
+          }
+        ]
+      },
+      {
+        "name": "negative-control-pip-list-unchanged",
+        "assertions": [
+          {
+            "type": "tool_start_preserves_args",
+            "toolName": "Bash",
+            "args": {
+              "command": "pip list"
+            }
+          }
+        ]
+      },
+      {
+        "name": "negative-control-npm-unchanged",
+        "assertions": [
+          {
+            "type": "tool_start_preserves_args",
+            "toolName": "Bash",
+            "args": {
+              "command": "npm install --dry-run --ignore-scripts --package-lock=false lodash"
+            }
+          }
+        ]
+      },
+      {
+        "name": "negative-control-brew-unchanged",
+        "assertions": [
+          {
+            "type": "tool_start_preserves_args",
+            "toolName": "Bash",
+            "args": {
+              "command": "brew install wget"
+            }
+          }
+        ]
+      },
+      {
+        "name": "negative-control-cargo-unchanged",
+        "assertions": [
+          {
+            "type": "tool_start_preserves_args",
+            "toolName": "Bash",
+            "args": {
+              "command": "cargo install ripgrep"
+            }
+          }
+        ]
+      }
+    ]
+  }
+}