@toolu/opencode 6.8.0

package/plugins/exa-search/skills/exa-search/scripts/search.sh

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Exa API CLI — search, crawl, and find similar content
+# Usage: ./search.sh <command> [options]
+#
+# Reads EXA_API_KEY from the environment (never from .env).
+
+command -v jq   >/dev/null 2>&1 || { echo "exa-search: jq required" >&2;   exit 1; }
+command -v curl >/dev/null 2>&1 || { echo "exa-search: curl required" >&2; exit 1; }
+
+EXA_API_KEY="${EXA_API_KEY:-}"
+EXA_URL="https://api.exa.ai"
+
+if [[ -z "$EXA_API_KEY" ]]; then
+  echo "exa-search: EXA_API_KEY unset" >&2
+  exit 1
+fi
+
+exa_post() {
+  local endpoint="$1"
+  local body="$2"
+  curl -sS --fail-with-body \
+    -X POST "$EXA_URL/$endpoint" \
+    -H "x-api-key: $EXA_API_KEY" \
+    -H "Content-Type: application/json" \
+    -H "Accept: application/json" \
+    -d "$body"
+}
+
+# ── search ──────────────────────────────────────────────────
+cmd_search() {
+  local query="" num_results=10 search_type="auto" category=""
+  local include_domains="" exclude_domains=""
+  local start_date="" end_date=""
+  local include_text="" exclude_text=""
+  local highlights_chars=4000 with_text="false" lean="false"
+
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      -q|--query)       query="$2"; shift 2;;
+      -n|--num-results) num_results="$2"; shift 2;;
+      -t|--type)        search_type="$2"; shift 2;;
+      -c|--category)    category="$2"; shift 2;;
+      --include-domains) include_domains="$2"; shift 2;;
+      --exclude-domains) exclude_domains="$2"; shift 2;;
+      --start-date)     start_date="$2"; shift 2;;
+      --end-date)       end_date="$2"; shift 2;;
+      --include-text)   include_text="$2"; shift 2;;
+      --exclude-text)   exclude_text="$2"; shift 2;;
+      --highlights)     highlights_chars="$2"; shift 2;;
+      --with-text)      with_text="true"; shift;;
+      --lean)           lean="true"; shift;;
+      *)
+        # treat bare arg as query
+        if [[ -z "$query" ]]; then query="$1"; shift
+        else echo "Unknown option: $1" >&2; exit 1; fi
+        ;;
+    esac
+  done
+
+  if [[ -z "$query" ]]; then
+    echo "Usage: search.sh search -q <query> [options]" >&2
+    echo "  -n, --num-results  Number of results (default: 10)" >&2
+    echo "  -t, --type         instant|fast|auto|deep-lite|deep|deep-reasoning (default: auto)" >&2
+    echo "  -c, --category     company|research paper|news|personal site|financial report|people" >&2
+    echo "  --include-domains  Comma-separated domains to include" >&2
+    echo "  --exclude-domains  Comma-separated domains to exclude" >&2
+    echo "  --start-date       Start published date (YYYY-MM-DD)" >&2
+    echo "  --end-date         End published date (YYYY-MM-DD)" >&2
+    echo "  --include-text     Text that must appear in results" >&2
+    echo "  --exclude-text     Text to exclude from results" >&2
+    echo "  --highlights N     Max highlight chars (default: 4000)" >&2
+    echo "  --with-text        Include full text in results" >&2
+    echo "  --lean             Strip image/favicon/subpages/entities for AI prompts" >&2
+    exit 1
+  fi
+
+  # Build JSON with jq to handle escaping properly
+  local body
+  body=$(jq -n \
+    --arg q "$query" \
+    --arg t "$search_type" \
+    --argjson n "$num_results" \
+    --argjson hc "$highlights_chars" \
+    --argjson wt "$with_text" \
+    '{query: $q, type: $t, numResults: $n, contents: {highlights: {maxCharacters: $hc}}}
+     | if $wt then .contents.text = true else . end')
+
+  # Add optional fields
+  if [[ -n "$category" ]]; then
+    body=$(echo "$body" | jq --arg c "$category" '.category = $c')
+  fi
+  if [[ -n "$include_domains" ]]; then
+    body=$(echo "$body" | jq --arg d "$include_domains" '.includeDomains = ($d | split(","))')
+  fi
+  if [[ -n "$exclude_domains" ]]; then
+    body=$(echo "$body" | jq --arg d "$exclude_domains" '.excludeDomains = ($d | split(","))')
+  fi
+  if [[ -n "$start_date" ]]; then
+    body=$(echo "$body" | jq --arg d "${start_date}T00:00:00.000Z" '.startPublishedDate = $d')
+  fi
+  if [[ -n "$end_date" ]]; then
+    body=$(echo "$body" | jq --arg d "${end_date}T00:00:00.000Z" '.endPublishedDate = $d')
+  fi
+  if [[ -n "$include_text" ]]; then
+    body=$(echo "$body" | jq --arg t "$include_text" '.includeText = [$t]')
+  fi
+  if [[ -n "$exclude_text" ]]; then
+    body=$(echo "$body" | jq --arg t "$exclude_text" '.excludeText = [$t]')
+  fi
+
+  if [[ "$lean" == "true" ]]; then
+    exa_post "search" "$body" | jq '{
+      requestId,
+      results: [(.results // [])[] | {
+        title,
+        url,
+        publishedDate,
+        author,
+        highlights,
+        text,
+        summary
+      } | with_entries(select(.value != null and .value != ""))]
+    }'
+  else
+    exa_post "search" "$body" | jq '.'
+  fi
+}
+
+# ── crawl ───────────────────────────────────────────────────
+cmd_crawl() {
+  local max_chars=3000
+  local urls=()
+
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      -m|--max-chars) max_chars="$2"; shift 2;;
+      *)              urls+=("$1"); shift;;
+    esac
+  done
+
+  if [[ ${#urls[@]} -eq 0 ]]; then
+    echo "Usage: search.sh crawl <url> [url...] [-m max_chars]" >&2
+    echo "  Extracts content from one or more URLs" >&2
+    echo "  -m, --max-chars  Max characters per page (default: 3000)" >&2
+    exit 1
+  fi
+
+  # Build URL array as JSON
+  local urls_json
+  urls_json=$(printf '%s\n' "${urls[@]}" | jq -R . | jq -s .)
+
+  local body
+  body=$(jq -n \
+    --argjson mc "$max_chars" \
+    --argjson urls "$urls_json" \
+    '{urls: $urls, text: true, highlights: {maxCharacters: $mc}}')
+
+  exa_post "contents" "$body" | jq '.'
+}
+
+# ── similar ─────────────────────────────────────────────────
+cmd_similar() {
+  local url="" num_results=10 highlights_chars=4000
+
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      -n|--num-results) num_results="$2"; shift 2;;
+      --highlights)     highlights_chars="$2"; shift 2;;
+      *)
+        if [[ -z "$url" ]]; then url="$1"; shift
+        else echo "Unknown option: $1" >&2; exit 1; fi
+        ;;
+    esac
+  done
+
+  if [[ -z "$url" ]]; then
+    echo "Usage: search.sh similar <url> [-n num_results]" >&2
+    echo "  Finds pages similar to the given URL" >&2
+    echo "  -n, --num-results  Number of results (default: 10)" >&2
+    echo "  --highlights N     Max highlight chars (default: 4000)" >&2
+    exit 1
+  fi
+
+  local body
+  body=$(jq -n \
+    --arg u "$url" \
+    --argjson n "$num_results" \
+    --argjson hc "$highlights_chars" \
+    '{url: $u, numResults: $n, contents: {highlights: {maxCharacters: $hc}}}')
+
+  exa_post "findSimilar" "$body" | jq '.'
+}
+
+# ── main ────────────────────────────────────────────────────
+usage() {
+  echo "Exa Search CLI" >&2
+  echo "" >&2
+  echo "Usage: search.sh <command> [options]" >&2
+  echo "" >&2
+  echo "Environment:" >&2
+  echo "  EXA_API_KEY  Required. Exa API key." >&2
+  echo "" >&2
+  echo "Commands:" >&2
+  echo "  search   Search the web (default if no command given)" >&2
+  echo "  crawl    Extract content from URLs" >&2
+  echo "  similar  Find pages similar to a URL" >&2
+  echo "" >&2
+  echo "Run 'search.sh <command>' with no args for command-specific help." >&2
+  exit 1
+}
+
+case "${1:-}" in
+  -h|--help|"") usage;;
+  search)  shift; cmd_search "$@";;
+  crawl)   shift; cmd_crawl "$@";;
+  similar) shift; cmd_similar "$@";;
+  # bare query — default to search
+  *)       cmd_search "$@";;
+esac

package/plugins/jev/.claude-plugin/plugin.json

@@ -0,0 +1,19 @@
+{
+  "name": "jev",
+  "version": "6.8.0",
+  "description": "Typed judgments from TypeSafe's Jev model at runtime (skill + REST wrapper) — yes/no probabilities, choices, and scores a script can branch on. Standalone, no dependencies.",
+  "author": {
+    "name": "Falconiere Barbosa",
+    "email": "hello@falconiere.io"
+  },
+  "homepage": "https://github.com/falconiere/toolu",
+  "repository": "https://github.com/falconiere/toolu.git",
+  "license": "MIT",
+  "keywords": [
+    "claude-code",
+    "typesafe",
+    "jev",
+    "judgment",
+    "skills"
+  ]
+}

package/plugins/jev/.codex-plugin/plugin.json

@@ -0,0 +1,7 @@
+{
+  "name": "jev",
+  "version": "6.8.0",
+  "description": "Typed judgments from TypeSafe's Jev model at runtime (skill + REST wrapper) — yes/no probabilities, choices, and scores a script can branch on. Standalone, no dependencies.",
+  "skills": "./skills/",
+  "hooks": "./hooks/hooks.json"
+}

package/plugins/jev/README.md

@@ -0,0 +1,61 @@
+# jev
+
+Typed judgments from TypeSafe's Jev model at runtime (skill + REST wrapper) — yes/no probabilities, choices, and scores a script can branch on.
+
+## Install
+
+Claude Code:
+
+```text
+/plugin marketplace add Falconiere/toolu
+/plugin install jev@toolu
+```
+
+Codex:
+
+```bash
+codex plugin marketplace add Falconiere/toolu
+codex plugin add jev@toolu
+```
+
+Restart the host after installation. In Codex, review and trust the installed
+hook through `/hooks`; installation alone does not trust hooks. Both hosts
+need `curl`, `jq`, and `TYPESAFE_API_KEY` in their launch environment.
+
+Standalone, no plugin dependencies.
+
+## What it provides
+
+- **`jev` skill** — mandatory when semantic decisions exist: gather evidence, call before the decision it informs, and reassess after new evidence, failed hypotheses, or changed requirements. Batch independent questions and reuse unchanged results.
+- **`jev.sh` wrapper** — `noul` (probability of yes), `choice` (pick one, with the full distribution), `score` (rate on your own ordered levels), and `ask` (many questions in one call).
+- **SessionStart hook** — publishes `jev.sh` at `<config-dir>/jev/jev.sh` and injects the full mandate on startup, resume, clear, and compaction. It checks local prerequisites and makes no API call.
+- **UserPromptSubmit hook** — restates the mandate on every prompt so it survives long sessions; silent for trivial confirmations and when prerequisites are missing (SessionStart already reported them). No API call.
+
+## Wiring
+
+The wrapper calls TypeSafe's single evaluation endpoint,
+`POST https://api.typesafe.ai/v1/systemone`, with `curl` and `jq` — no SDK.
+
+Set `TYPESAFE_API_KEY` in your environment (keys: `https://console.typesafe.ai/settings/keys`);
+it is never read from a `.env` file. `JEV_TIMEOUT` overrides the 60-second timeout
+per attempt. Model defaults to `jev-latest`.
+
+Full CLI reference and usage guidance: [`skills/jev/SKILL.md`](skills/jev/SKILL.md).
+Executable [problem-solving examples](skills/jev/references/problem-solving.md) cover search, debugging, planning, and review with uncertainty and no-match handling.
+Plugin page: [`docs/jev/README.md`](../../docs/jev/README.md).
+
+## Wrapper reference
+
+Text input only; preprocess other formats. Context: 64k tokens/request, 32k for
+state + longest question. Slice large inputs. English is the primary training language.
+
+`@FILE`/stdin become structured state only for a JSON object/array; other values
+stay strings. Default output is `.answers`; `--raw` includes model/token usage.
+Missing/invalid answers fail explicitly.
+
+Retries: at most three attempts for timeout, connection failure, HTTP 408/429/5xx;
+1s then 2s backoff. `Retry-After` seconds or `retry-after-ms` up to 60s is honored;
+longer waits surface the error. Other 4xx, including 401/422, are not retried.
+
+Exit codes: `1` usage/config/invalid response; `22` HTTP error (body on stderr);
+`28` timeout. `JEV_TIMEOUT` sets timeout per attempt (default 60s).

package/plugins/jev/hooks/hooks.json

@@ -0,0 +1,25 @@
+{
+  "hooks": {
+    "SessionStart": [
+      {
+        "matcher": "startup|resume|clear|compact",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/session-start.sh\""
+          }
+        ]
+      }
+    ],
+    "UserPromptSubmit": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/user-prompt-submit.sh\""
+          }
+        ]
+      }
+    ]
+  }
+}

package/plugins/jev/hooks/lib/common.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Shared by jev's hooks. Resolves the host-native config root, the stable
+# published wrapper path, and the local prerequisites, and emits hook output.
+# No network calls: everything here must be safe to run on every prompt.
+#
+# Source via:  . "$HOOK_DIR/lib/common.sh"
+
+# jev_config_root -> writable config root for the active host.
+# An explicit TOOLU_CONFIG_DIR wins; Codex is detected by TOOLU_HOST_OVERRIDE
+# or by its native PLUGIN_ROOT variable; otherwise Claude Code.
+jev_config_root() {
+  if [ -n "${TOOLU_CONFIG_DIR:-}" ]; then
+    printf '%s\n' "$TOOLU_CONFIG_DIR"
+  elif [ "${TOOLU_HOST_OVERRIDE:-}" = codex ] ||
+       { [ -z "${TOOLU_HOST_OVERRIDE:-}" ] && [ -n "${PLUGIN_ROOT:-}" ]; }; then
+    printf '%s\n' "${CODEX_HOME:-$HOME/.codex}"
+  else
+    printf '%s\n' "${CLAUDE_CONFIG_DIR:-$HOME/.claude}"
+  fi
+}
+
+# jev_published_path -> the stable wrapper path the agent's shell can reach.
+jev_published_path() {
+  printf '%s/jev/jev.sh\n' "$(jev_config_root)"
+}
+
+# jev_missing_prereqs WRAPPER -> space-prefixed list of what is missing, or "".
+jev_missing_prereqs() {
+  local dst="$1" missing="" tool
+  for tool in jq curl; do
+    command -v "$tool" >/dev/null 2>&1 || missing="$missing $tool"
+  done
+  [ -n "${TYPESAFE_API_KEY:-}" ] || missing="$missing TYPESAFE_API_KEY"
+  [ -x "$dst" ] || missing="$missing executable-wrapper"
+  printf '%s' "$missing"
+}
+
+# jev_emit EVENT CONTEXT -> hookSpecificOutput JSON; plain text without jq.
+# Both hosts accept the JSON shape for SessionStart and UserPromptSubmit.
+jev_emit() {
+  local event="$1" context="$2"
+  [ -n "$context" ] || return 0
+  if command -v jq >/dev/null 2>&1; then
+    jq -nc --arg event "$event" --arg context "$context" \
+      '{hookSpecificOutput: {hookEventName: $event, additionalContext: $context}}'
+  else
+    printf '%s\n' "$context"
+  fi
+}

package/plugins/jev/hooks/session-start.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# SessionStart hook — publish the jev wrapper at a STABLE, env-independent
+# path. See exa-search's session-start.sh for the same fix rationale:
+# ${CLAUDE_PLUGIN_ROOT} is exported to hook subprocesses only — NOT to the
+# agent's Bash tool subshell — so a SKILL.md path of
+# "${CLAUDE_PLUGIN_ROOT}/skills/jev/scripts/jev.sh" expands to
+# "/skills/.../jev.sh: No such file" when an agent pastes it. Symlink the
+# wrapper to ${CLAUDE_CONFIG_DIR:-$HOME/.claude}/jev/jev.sh instead.
+# Refreshed every session; inject the mandate without network calls.
+set -euo pipefail
+
+# Consume stdin so Claude Code's hook IPC never stalls.
+cat >/dev/null 2>&1 || true
+
+HOOK_DIR="$(cd "$(dirname "$0")" 2>/dev/null && pwd)"
+# Fail soft on a corrupted install: a missing lib must never break the session.
+[ -f "$HOOK_DIR/lib/common.sh" ] || exit 0
+# shellcheck source=lib/common.sh
+. "$HOOK_DIR/lib/common.sh"
+
+plugin_dir="$(cd "$HOOK_DIR/.." 2>/dev/null && pwd)"
+src="${plugin_dir:+$plugin_dir/skills/jev/scripts/jev.sh}"
+[ -n "$src" ] && [ -f "$src" ] || exit 0
+
+reg_root="$(jev_config_root)/jev"
+mkdir -p "$reg_root" 2>/dev/null || { echo "jev: cannot create $reg_root — wrapper not published" >&2; exit 0; }
+
+dst="$reg_root/jev.sh"
+if [ -L "$dst" ] || [ ! -e "$dst" ]; then
+  ln -sf "$src" "$dst" 2>/dev/null || { echo "jev: cannot publish $dst" >&2; exit 0; }
+fi
+
+# This plugin owns its mandate so first-session delivery does not depend on
+# the core plugin, hook ordering, or the model choosing to load a skill. The
+# UserPromptSubmit hook repeats a shorter form on every task so the rule
+# survives long sessions; this is the full statement.
+missing="$(jev_missing_prereqs "$dst")"
+if [ -n "$missing" ]; then
+  context="Jev unavailable (missing:$missing). Set TYPESAFE_API_KEY in the agent's launch environment and install curl/jq. Jev is mandatory on every task once available; until then, state the limitation once per task and use an explicit reasoning/evidence fallback; never invent a Jev result. Do not read credentials from .env."
+else
+  context="Jev is mandatory on every task containing semantic decisions. After initial exploration, identify useful judgments over supplied evidence; you MUST call \"$dst\" before the decision it informs. Reassess after new evidence, failed hypotheses, or changed requirements. Batch independent questions in one ask call. Reuse unchanged evidence and questions rather than repeating calls. If a task has no semantic decision, say so in one sentence rather than skipping silently. Syntax and linked examples: $plugin_dir/skills/jev/SKILL.md. Keep exact rules, tests, and code verification deterministic. On service failure, state the limitation and use an explicit evidence fallback. Jev never replaces tests or authorization."
+fi
+jev_emit SessionStart "$context"
+
+exit 0

package/plugins/jev/hooks/user-prompt-submit.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# UserPromptSubmit hook — restate the Jev mandate on every task.
+#
+# SessionStart states the full rule once per startup, resume, clear, and
+# compaction, but a long session drifts away from it. Each user prompt is a
+# task, so each one gets a short reminder naming the published wrapper. No
+# network calls; the hook only checks local prerequisites, exactly like
+# SessionStart.
+#
+# Silent cases, deliberately:
+#   - trivial confirmations (y / ok / lgtm ...) are not tasks;
+#   - missing prerequisites — SessionStart already reported them, repeating
+#     the same failure on every prompt is noise the agent cannot act on.
+set -euo pipefail
+
+HOOK_DIR="$(cd "$(dirname "$0")" 2>/dev/null && pwd)"
+# Fail soft on a corrupted install: a missing lib must never break the session.
+[ -f "$HOOK_DIR/lib/common.sh" ] || exit 0
+# shellcheck source=lib/common.sh
+. "$HOOK_DIR/lib/common.sh"
+
+input=$(cat 2>/dev/null || true)
+prompt=""
+if command -v jq >/dev/null 2>&1; then
+  prompt=$(jq -r '.prompt // ""' <<<"$input" 2>/dev/null || true)
+fi
+[ -n "$prompt" ] || exit 0
+
+prompt_lower=$(printf '%s' "$prompt" | tr '[:upper:]' '[:lower:]')
+if [[ "$prompt_lower" =~ ^[[:space:]]*(y|n|yes|no|ok|okay|sure|thanks|thank\ you|go\ ahead|looks\ good|lgtm|correct|exactly|right|done|nah|nope|yep|yup|continue)[\.\!\?]?[[:space:]]*$ ]]; then
+  exit 0
+fi
+
+plugin_dir="$(cd "$HOOK_DIR/.." 2>/dev/null && pwd)"
+dst="$(jev_published_path)"
+[ -z "$(jev_missing_prereqs "$dst")" ] || exit 0
+
+context="Jev is mandatory for this task when it contains semantic decisions. After initial exploration, identify useful judgments over supplied evidence; you MUST call \"$dst\" before the decision it informs. Reassess after new evidence, failed hypotheses, or changed requirements. Batch independent questions in one ask call. Reuse unchanged evidence and questions rather than repeating calls. If a task has no semantic decision, say so in one sentence rather than skipping silently. Syntax and linked examples: $plugin_dir/skills/jev/SKILL.md. On failure, state the limitation and use an evidence fallback; Jev never replaces tests or authorization."
+jev_emit UserPromptSubmit "$context"
+
+exit 0

package/plugins/jev/skills/jev/SKILL.md

@@ -0,0 +1,81 @@
+---
+name: jev
+description: Mandatory for semantic decisions during search, debugging, planning, execution, and review — classify, rank, rate, route, or assess supplied evidence.
+---
+
+# Jev: typed judgments
+
+## When to call
+
+**Semantic decision + Jev available → MUST call before that decision.**
+Explore first; supply evidence, candidates, and criteria. Reassess after new
+evidence, failed hypotheses, or changed requirements. Batch independent questions
+in one `ask`; wait for evidence/answers needed by dependent questions. Reuse
+results while evidence, questions, and criteria are unchanged. No call quotas.
+
+No semantic decision → say so once. Missing plugin, credentials, or service →
+state limitation once per task; continue with explicit evidence-based reasoning.
+Never invent results. Architecture, exact checks, arithmetic, reproduction,
+tests, correctness, and authorization stay with the agent/tools.
+
+## CLI
+
+Requires `curl`, `jq`, environment `TYPESAFE_API_KEY`; never read `.env`.
+Use the active host's published wrapper; plugin lifecycle variables are unavailable
+in ordinary shells:
+
+```bash
+# Codex
+JEV="${TOOLU_CONFIG_DIR:-${CODEX_HOME:-$HOME/.codex}}/jev/jev.sh"
+# Claude Code: use this assignment instead
+# JEV="${TOOLU_CONFIG_DIR:-${CLAUDE_CONFIG_DIR:-$HOME/.claude}}/jev/jev.sh"
+```
+
+Repository fallback, when not installed: `plugins/jev/skills/jev/scripts/jev.sh`.
+
+```text
+"$JEV" noul   "question" -s STATE [--true DESC] [--false DESC]
+"$JEV" choice "question" -s STATE -o KEY=DESC -o KEY=DESC
+"$JEV" score  "question" -s STATE -l "lowest situation" -l "highest situation"
+"$JEV" ask questions.json -s STATE
+```
+
+- `-s/--state`: literal text, `@FILE`, or `-` (stdin). Required.
+- `ask`: question-map JSON file or `-` (stdin); keys become answer IDs. Each
+  question needs `type` and `instructions`; `criteria` is required for Choice/Score,
+  optional for Noul.
+  Structured instructions/criteria use `ask`; other commands take strings.
+- Options: `-m/--model` (default `jev-latest`), `--raw` (model/usage),
+  `--id` (default `q`; single-question commands only).
+- Choice: `-o/--option`, 2–255 options; bare `-o KEY` omits description.
+  Score: `-l/--level`, 2–10 levels,
+  lowest first. Read returned legend; fractional Scores are not exact quantities.
+- Output: answer map (`noul`, or `choice`/`score` plus probabilities/confidence).
+  Nonzero exit → fallback, never interpret absent output as a judgment.
+
+## Question contract
+
+- Minimal named evidence; reference fields explicitly, e.g. `source.text`.
+- Exact condition + boundary criteria; instructions and criteria must agree.
+  Meaning belongs here, not in IDs (the model never sees IDs).
+- Split independently useful judgments; keep context for coherent judgments
+  such as claim/source support. High Noul = stated condition holds.
+- Score levels: self-contained situations, e.g. “Feature broken; workaround
+  exists”. Avoid bare numbers or “worse than previous”.
+- Include no-match/insufficient-evidence outcomes when candidates may not fit.
+- Choice probabilities are relative alternatives; graded relevance needs
+  comparable per-candidate Scores. Check answer existence separately from rank.
+- Noul near 0.5 = uncertainty. Choice/Score confidence = concentration, not
+  correctness. Inspect uncertain/conflicting results; gather missing evidence.
+- Calibrate thresholds on representative data, separately per question/type;
+  `P(yes)` need not equal `1-P(not yes)`. Pin model when thresholds depend on it.
+- Record useful evidence → question → answer → next action. Use `--raw` for
+  model/usage evaluation; extra questions cost tokens.
+
+## Load on demand
+
+- Need a pattern: read Setup + the matching section in
+  [problem-solving](references/problem-solving.md), not all examples.
+- Need transport limits/errors: [wrapper reference](../../README.md#wrapper-reference).
+- Designing new question formats: check relevant [live docs](https://docs.typesafe.ai/llms.txt).
+- Evaluation only: [record and scenarios](evals/README.md).

package/plugins/jev/skills/jev/evals/README.md

@@ -0,0 +1,98 @@
+# Ongoing problem-solving evaluation
+
+Manual, opt-in evaluation; never part of hermetic CI. Requires an existing
+`TYPESAFE_API_KEY` in the launch environment. Do not read `.env` or record keys.
+
+## Repeat
+
+1. Run setup and the search, debugging (both evidence sets), planning, and review
+   blocks in [problem-solving.md](../references/problem-solving.md), using the
+   existing wrapper. Retain the scratch results until session evaluation ends.
+2. In a fresh agent session, load the updated skill and reference. Ask it to
+   select and record its next action, not just report the model's answers:
+   - Search: replace the query with “What telescope aperture is recommended to
+     resolve the rings of Saturn?” Keep the same two excerpts. Change the Noul
+     criteria to require an aperture recommendation. Score each excerpt on the
+     same levels: “Unrelated to telescopes or observing Saturn”, “Discusses
+     observing Saturn but omits an aperture recommendation”, “Supplies the
+     requested telescope aperture recommendation”. Neither excerpt matches.
+   - Debugging: supply the first observations, record the chosen experiment,
+     then supply the explicitly synthetic new observations. Record whether the
+     agent abandons the first hypothesis and chooses a discriminating experiment.
+   - Reuse: move the unchanged planning evidence and questions to review with
+     the saved answer. Check that the agent reuses it without another call.
+   - Unavailable: invoke the wrapper with `env -u TYPESAFE_API_KEY`, then ask
+     the agent to continue from the supplied evidence and record its fallback.
+3. Inspect the raw model, usage, choices, uncertainty, and decision impact. Do
+   not assert exact probabilities. A plausible classification alone does not
+   pass a case: it must lead to an appropriate next action or explicit fallback.
+
+No application is included in these examples. Actual debugging experiments and
+technical feasibility checks need a real target; do not report them as executed.
+Installed-host hook lifecycle behavior is separately covered by the hook tests.
+
+## Observed run — 2026-09-21
+
+The parent Codex session ran the documented shell blocks through the repository
+wrapper with real API calls. `jev-latest` resolved to `jev-1.13.0`. Five requests
+used 2,641 input and 254 output tokens; results below are observations, not
+thresholds or guarantees.
+
+| Case | Observed answer | Decision impact |
+| --- | --- | --- |
+| Search, retention query | Existence 0.03; access relevance 1, billing 0 on a 0–2 rubric | Read access first but seek retention policy; neither excerpt supplies a duration. |
+| Debug, initial evidence | `cache`, confidence 0.97 | Prioritize disabling cache in the affected profile. |
+| Debug, changed evidence | `insufficient`, confidence 0.92 | Drop the supplied hypotheses as complete explanations; propose an override removal/restoration experiment. |
+| Planning | `worker` for both preferences | Prefer the existing worker under stated assumptions; technical checks still required. |
+| Review | `contradicts` | Correct the claim against the full policy passage; verify provenance separately. |
+
+A separate fresh Codex agent session then read the updated guidance and performed
+three more real calls (1,660 input / 137 output tokens, same model):
+
+| Session case | Observed behavior and next action |
+| --- | --- |
+| No relevant search match | Existence 0.01 and both Scores 0. Rejected both excerpts and ran a targeted local search for telescope/Saturn/astronomy; no matches. Reported the corpus gap rather than inventing an answer. |
+| Debug with new evidence | Selected cache first, then `insufficient` (confidence 0.88). Changed the proposed experiment to isolating the saved label override; did not claim the synthetic bug was reproduced. |
+| Unchanged planning evidence | Read saved state, questions, and answers. Reused both worker judgments with zero new planning API calls; retained capacity/retry/requirements checks as next steps. |
+| Missing credentials | Actual wrapper exit 1, `jev: TYPESAFE_API_KEY unset`, empty stdout. Compared observations manually and chose the override experiment with explicit reasons, without inventing a result. |
+
+One earlier design-comparison call chose preserving the old initial-call demand
+with confidence 0.56, contrary to the user's explicit plan. The implementing
+agent rejected that recommendation after inspecting the conflict. This is also
+evidence for keeping user requirements and technical verification authoritative.
+
+Limits: these were real agent/API sessions over controlled synthetic evidence,
+not installed-host end-to-end sessions or production accuracy measurements. The
+browser experiment and worker capacity checks were proposed, not executed.
+
+## Repository verification
+
+The 25 Jev hook tests passed locally, including host paths, prerequisite failures,
+confirmation suppression, and JSON output. The full `bun run test` passed in an
+Ubuntu 24.04 ARM container with GNU awk: 1,841 passed and 99 skipped
+(including missing Cargo, platform-specific checks, and opt-in live checks). All Jev hook
+and offline wrapper tests ran; live scenarios were evaluated separately above. Shellcheck and context-budget checks passed. A fresh-context review
+reported no blocking findings.
+
+The native macOS full run was stopped after script-launch delays reproduced on
+a trivial temporary script. A minimal Debian container lacked CI tools and
+failed loopback transport checks; an Ubuntu run using mawk hit a regex compiler
+panic in unchanged debug-log tests. Matching the test tools and using GNU awk
+resolved those environment failures without production-code changes.
+
+## Prompt-size audit — 2026-09-21
+
+Compared with commit `7c5915e`, the skill + shared workflow decreased from
+1,908 to 762 whitespace-delimited words (13,063 to 5,841 UTF-8 bytes).
+These are size measurements, not tokenizer counts. Stage prompts link to the
+shared rules; transport details and evaluation records load only on demand.
+Examples explicitly load Setup + the relevant section. All six executable
+Bash blocks remain byte-identical to the previously evaluated examples.
+
+A fresh agent loaded only the compact skill and Setup + Debugging. Live calls
+through the byte-identical wrapper in the sibling checkout selected `cache`,
+then `insufficient` after changed observations; next experiment shifted to the
+saved override. The isolated checkout call did not complete and was terminated;
+no answer was inferred. Missing-key invocation returned exit 1. The agent
+recognized unchanged-input reuse and retained explicit manual fallback. This
+checks prompt usability, not installed-host behavior or production accuracy.