failproofai 1.0.7-beta.1 → 1.0.7

package/.next/standalone/app/settings/settings-client.tsx

@@ -206,24 +206,7 @@ const HOW_IT_WORKS: ReadonlyArray<{ label: string; body: string }> = [
   },
 ];
 
-export default function SettingsClient({
-  initial,
-  jevPanel,
-}: {
-  initial: ScheduledAuditView | null;
-  /**
-   * The Jev panel, rendered by the server page and handed in as an element
-   * rather than imported here.
-   *
-   * It is its own feature with its own file, its own actions and its own
-   * refresh — nothing on this page reads or writes what it does. Composing it
-   * in keeps that true in the code as well as in the design: this component
-   * does not import the Jev actions, so a test of the scheduled-audit panel
-   * neither renders nor touches Jev, and vice versa. Optional so the existing
-   * callers that only care about the audit panel keep working unchanged.
-   */
-  jevPanel?: ReactNode;
-}) {
+export default function SettingsClient({ initial }: { initial: ScheduledAuditView | null }) {
   // Seeded from the server render, so the first paint already tells the truth
   // about whether scheduled audits are on. See the note in `page.tsx`.
   const [view, setView] = useState<ScheduledAuditView | null>(initial);
@@ -524,10 +507,6 @@ export default function SettingsClient({
                 dashboard can reach <code>~/.failproofai/config.json</code>.
               </p>
             </div>
-            {/* A different file. An unreadable `config.json` says nothing about
-                `jev.json`, and hiding the panel that can turn a second
-                evaluator off would be the worst moment to hide it. */}
-            {jevPanel}
           </div>
         ) : (
           <div className="set-console">
@@ -674,11 +653,6 @@ export default function SettingsClient({
                 </dl>
               </div>
             </div>
-
-            {/* Full width, below the two panels: Jev is a second evaluator for
-                the whole machine, not a property of the scheduled scan, and its
-                endpoint would not fit a half-width column anyway. */}
-            {jevPanel}
           </div>
         )}
 

package/.next/standalone/app/settings/settings.css

@@ -151,51 +151,6 @@
   margin: 0;
 }
 
-/* ── jev ──────────────────────────────────────────────────────────────────── */
-
-/* A full-width cell in the same console, below the two panels. The gap above it
- * is the console's own 1px grid — this rule adds no border of its own. */
-.set-jev {
-  display: flex;
-  flex-direction: column;
-  gap: 16px;
-}
-
-/* Two columns at desktop: a provider or a mode needs about half the width an
- * endpoint does, and stacking five controls in one column turns a short form
- * into a scroll. */
-.set-jev-fields {
-  display: grid;
-  grid-template-columns: repeat(2, minmax(0, 1fr));
-  gap: 16px 20px;
-}
-
-.set-field {
-  display: flex;
-  flex-direction: column;
-  gap: 6px;
-  min-width: 0;
-}
-.set-field-label {
-  font-family: var(--font-mono);
-  font-size: 9.5px;
-  font-weight: 500;
-  letter-spacing: 0.18em;
-  text-transform: uppercase;
-  color: var(--dim);
-}
-
-.set-jev-actions {
-  display: flex;
-  align-items: center;
-  gap: 16px;
-  flex-wrap: wrap;
-}
-
-@media (max-width: 640px) {
-  .set-jev-fields { grid-template-columns: 1fr; }
-}
-
 /* ── masthead ─────────────────────────────────────────────────────────────── */
 
 .set-mast {
@@ -273,40 +228,6 @@
   outline-offset: 1px;
 }
 
-/* `.set-num`'s box, stretched. Same background, same hairline, same square
- * corners and the same pink focus ring — a second field style would read as a
- * second form on one page. */
-.set-text,
-.set-select {
-  font-family: var(--font-mono);
-  font-size: 13px;
-  width: 100%;
-  padding: 6px 10px;
-  background: var(--bg);
-  border: 1px solid var(--line-2);
-  color: var(--ink);
-  border-radius: 0;
-}
-.set-text:focus-visible,
-.set-select:focus-visible {
-  outline: 2px solid var(--accent-pink);
-  outline-offset: 1px;
-}
-.set-text:disabled,
-.set-select:disabled {
-  opacity: 0.5;
-  cursor: not-allowed;
-}
-.set-text::placeholder {
-  color: var(--dim);
-}
-/* The native arrow is drawn by the platform and cannot be tokenised; leaving
- * the control's own appearance in place is the one honest option, so the box
- * around it is what carries the page's style. */
-.set-select {
-  cursor: pointer;
-}
-
 .set-link {
   font-family: var(--font-mono);
   font-size: 12px;

package/.next/standalone/package.json

@@ -1,6 +1,6 @@
 {
   "name": "failproofai",
-  "version": "1.0.7-beta.1",
+  "version": "1.0.7",
   "description": "Observability and enforcement for AI agent harnesses. 39 built-in policies hooked into 12 of them — Claude Code, Codex, Cursor, Hermes, OpenClaw and more — blocking the tool call before it runs. Local dashboard included, no account needed.",
   "bin": {
     "failproofai": "./dist/cli.mjs",
@@ -122,15 +122,15 @@
     "browserslist": "4.28.8"
   },
   "optionalDependencies": {
-    "@failproofai/failproofaid-linux-x64": "1.0.7-beta.1",
-    "@failproofai/failproofaid-linux-arm64": "1.0.7-beta.1",
-    "@failproofai/failproofaid-darwin-x64": "1.0.7-beta.1",
-    "@failproofai/failproofaid-darwin-arm64": "1.0.7-beta.1"
+    "@failproofai/failproofaid-linux-x64": "1.0.7",
+    "@failproofai/failproofaid-linux-arm64": "1.0.7",
+    "@failproofai/failproofaid-darwin-x64": "1.0.7",
+    "@failproofai/failproofaid-darwin-arm64": "1.0.7"
   },
   "failproofaidBinaries": {
-    "linux-x64": "8fc6dcf94834a332791a76b24c9906ef63371ee91b5a8dc2c764db202d5608a7",
-    "linux-arm64": "ce48b73245adc6a0d7cef2ba74e37693bd00d7cff75a8d27b6bdbd8ba0251613",
-    "darwin-x64": "27d8f9d740d409741d43e75850eea5f956ac6b578628cbf2f364d711f89621a6",
-    "darwin-arm64": "0ee02f87dd6e3f0761b18fd5d60ebbbfd5328f3e99bc3d2c909d41d540ee68a0"
+    "linux-x64": "65acfcabaa4dda6d37d2958d4100c366a85f17c68d2250de6126e2717ca773e6",
+    "linux-arm64": "cece883f87d53034003be8f0f293aa89bd0e69306d6032e17b3eb91f230b3ce2",
+    "darwin-x64": "565aa3d560a05dd76e2eb843ddbad593d2d8df94c963bdeb1c7e2a8a3816d0c5",
+    "darwin-arm64": "2b41601b53e77a31439e92d8be9be6b9872e3d4ad7e8664ff89cd2be4ed6c44d"
   }
 }

package/.next/standalone/sdk/python/skill/SKILL.md

@@ -1,19 +1,28 @@
 ---
 name: failproofai-sdk
 description: |-
-  The way to make an AI agent report what it did to Failproof AI — planning what to record, writing the instrumentation, and proving the events land. Reach for it on vague phrasing too: "add observability to my agent", "why isn't my agent showing up?"
+  Make a custom AI agent — Python or TypeScript/JavaScript, on a framework or hand-built — report what it did to Failproof AI, and run your own evaluator worker (the "eval pod") that scores those runs. Reach for it on vague phrasing too: "add observability to my agent", "why isn't my agent showing up?", "run an LLM judge on our own infra".
 
   Trigger when the user wants to:
-  • plan an integration — which points in their agent loop to record, and what the platform must see before sessions, errors, and evals work at all;
-  • write or fix instrumentation — add the `failproofai_sdk` Python SDK to an agent codebase, thread session/agent identity through it, emit tool, model, hook, or human events;
-  • verify it — confirm events are being written, or debug an integration that looks correct and produces nothing.
+  • plan an integration — which points in the agent loop to record;
+  • instrument — add `failproofai-sdk` (Python) or `@failproofai/sdk` (Node, Bun, Deno, Next.js): turn on an adapter (LangChain/LangGraph, CrewAI, LlamaIndex, Pydantic AI, Vercel AI SDK, Mastra) or wire a hand-built loop;
+  • verify — confirm events are written, or debug an integration that produces nothing;
+  • evaluate — write, deploy or debug an Evaluator worker in Python or TypeScript.
 
-  Served by the `failproofai_sdk` Python SDK, inside the user's own agent.
-
-  NOT for reading telemetry that already landed or operating a deployment (that's `fp-cloud-cli`), or building the evaluator service that scores runs (that's `agenteye-evaluator`).
+  NOT for reading telemetry or scores that already landed (that's `fp-cloud-cli`), or deciding what is worth evaluating (that's `failproofai-eval-brainstorm`).
 ---
 
-# Failproof AI Python SDK
+# Failproof AI SDK — Python and TypeScript
+
+Two packages, one pipe: `failproofai-sdk` (Python, imported as `failproofai_sdk`)
+and `@failproofai/sdk` (TypeScript/JavaScript). They write the same 15 events in
+the same wire format into the same spool directory. Everything in this file is
+language-neutral unless it says otherwise, and code blocks are Python.
+**For a TypeScript or JavaScript agent, read `references/typescript.md` alongside
+it**: it has the camelCase names, the adapters, bundler and Next.js setup, the
+no-framework wiring, shutdown and verification. Both packages also ship the
+**evaluator worker** that scores finished sessions on your own infrastructure (§7,
+`references/evaluator.md`).
 
 The SDK records what your agent did, from inside your agent. You call it at points
 you choose; it appends structured events to local `.jsonl` files. A separate
@@ -34,10 +43,14 @@ and it is verifiable on a laptop with no server, no API key, and no network.
 The API is small — 15 event methods, all keyword-only. The hard parts are
 **deciding where to call them** and **knowing which silences are bugs**, because
 this SDK does not raise when you get it wrong. Sections 1-3 are the plan, 4 is the
-code, 5-6 are the proof.
+code, 5-6 are the proof, 7 is scoring the runs.
 
 ## 1. Install it
 
+TypeScript/JavaScript: `npm install @failproofai/sdk` — zero dependencies, Node ≥
+20.9, Bun or Deno, ESM and CommonJS. The npm name has no lookalike trap; the rest of
+this section is Python. See `references/typescript.md`.
+
 ```bash
 pip install failproofai-sdk        # or: uv add failproofai-sdk
 ```
@@ -136,6 +149,10 @@ Full field-by-field catalog: `references/events.md`.
 
 Work with these; none of them raise, so none of them show up in testing.
 
+(TypeScript: the same contract with camelCase spellings, the same `"dev"` default,
+the same reserved names and the same `duration_ms` rule, but a different shutdown
+recipe — `references/typescript.md` → *The contract, in TypeScript* and *Shutdown*.)
+
 - **There IS an ambient session, and it is the ergonomic path.** `session()`,
   `agent()` and `tool_call()` bind identity on contextvars, so `session_id` and
   `agent_id` are optional on all 15 event methods — omitted, they resolve from
@@ -277,7 +294,10 @@ Threading `session_id` and `agent_id` through every call site by hand is the thi
 that makes integrations ugly and abandoned. Don't. Bind identity once per run and
 let the call sites read it.
 
-`references/frameworks.md` covers the four adapters. `references/integration.md` has the hand-written wrapper — one small
+`references/frameworks.md` covers the four Python adapters; `references/typescript.md`
+covers the four TypeScript ones (LangChain.js/LangGraph.js, Vercel AI SDK, Mastra,
+LlamaIndex.TS), Next.js, bundlers, and the three-edit-site wiring for a hand-built
+TypeScript loop. `references/integration.md` has the hand-written wrapper — one small
 module, correct under `asyncio` and threads, adaptable to any codebase — plus
 worked shapes for a tool dispatcher, an LLM client wrapper, and framework-specific
 callback layers. Read it before writing your own; the naive version (a module
@@ -289,6 +309,9 @@ has a request context or a trace id, bind to that instead of inventing one.
 
 ## 5. Verify — watch the files
 
+(TypeScript: same directory, same checklist; the commands are in
+`references/typescript.md` → *Verify*.)
+
 **This is the whole point of the file boundary: you can prove the integration
 without a server.** Run the agent and look.
 
@@ -319,7 +342,7 @@ cat ~/.failproofai/custom-agents/events/*.jsonl | python -m json.tool --json-lin
 
 Then check, in this order — the first failure explains everything downstream:
 
-1. **Any files at all — or do they stop mid-run?** Look at stderr for
+1. **Any files at all — or do they stop mid-run?** (Python) Look at stderr for
    `Exception in thread failproofai-sdk-flush`. **This is the first thing to check and
    the worst thing to miss**: one non-JSON-serializable value killed the writer,
    and everything after it — including the at-exit flush — is gone (§3). The tell
@@ -342,8 +365,8 @@ Then check, in this order — the first failure explains everything downstream:
    even when identity is a module global, and mixing only appears once two runs
    overlap — which is production, not your laptop (§4).
 7. **Do `tool_use` and `tool_result` share a `tool_call_id`?** Unpaired means no
-   duration. Also confirm your ids are unique *process-wide* — a collision pairs
-   the wrong two events and reports a confident wrong duration (§3).
+   duration. Also confirm no id repeats *within a session* for the same kind — a
+   collision pairs the wrong two events and reports a confident wrong duration (§3).
 
 A test-mode loop that costs nothing:
 
@@ -388,6 +411,10 @@ Compare the two paths first: print the directory your agent is actually writing
 (`python -c "import failproofai_sdk._resolver as r; print(r.get_base_dir())"` in the
 agent's own environment, with the agent's own env vars) and check the collector is
 running and pointed at the same one. A `.jsonl` count that only grows is the tell.
+The reverse is healthy: with `failproofaid` running, the directory empties seconds
+after each flush, because the daemon ships each file and deletes it — so an empty
+real spool proves nothing either way. Verify content with the throwaway
+`FAILPROOFAI_HOME` loop in §5, and arrival with `fp-cloud-cli`.
 
 Confirming events arrived on the *platform* is deliberately not this skill's job —
 that is the `fp-cloud-cli` skill, from a **separate environment** (§1). Collector
@@ -396,4 +423,23 @@ setup and deployment are your platform's own documentation.
 If the files look right (§5) and the collector is running against the same
 directory, the integration is done.
 
-<!-- ci: no-op touch to exercise the skill-sync trigger (safe to remove) -->
+## 7. Score the runs — your own evaluator worker
+
+Sessions that land can be scored. Hosted evaluations are written in the dashboard
+(**Analyze → eval authoring**) and run on Failproof AI's managed evaluator: that is
+the default. When an evaluation needs your own model keys, packages, secrets,
+private network or heavy compute, run it in **your own worker** — the eval pod.
+
+It ships in the same packages: `failproofai_sdk.evaluator` and
+`@failproofai/sdk/evaluator`. You declare an `Evaluator`, register versioned
+evaluations with an optional `when` condition, and start it with an
+`evaluations:run` key in `FAILPROOFAI_EVALUATOR_TOKEN`. It only calls out over
+HTTPS — claim finished sessions, score them, submit — so a pod needs egress and a
+secret, and no ingress. Its results carry the **customer** tag.
+
+Two things to settle before writing one: the agents must already produce finished
+sessions (§2 — nothing scores a run with no `agent_end`), and bump an evaluation's
+`version` whenever its logic changes. Everything else — the API in both languages,
+env vars, a Dockerfile, SIGTERM drain vs. the pod's grace period, scaling, and a
+debugging order for a worker that scores nothing — is in `references/evaluator.md`.
+

package/.next/standalone/sdk/python/skill/agents/openai.yaml

@@ -5,4 +5,5 @@
 
 # Let Codex auto-select this skill when a task matches the description
 # (set to false to require explicit `$failproofai-sdk` invocation).
-allow_implicit_invocation: true
+policy:
+  allow_implicit_invocation: true

package/.next/standalone/sdk/python/skill/references/evaluator.md

@@ -0,0 +1,255 @@
+# Your own evaluator worker — the eval pod
+
+An evaluation scores a **finished** session. There are two places one can run:
+
+| | Hosted | Your own worker (this page) |
+|---|---|---|
+| Written | in the dashboard, **Analyze → eval authoring** | in Python or TypeScript, with the SDK you already install |
+| Runs on | Failproof AI's managed evaluator, sandboxed | your infrastructure: a container, a pod, a VM |
+| Use it for | deterministic checks and model-backed checks Failproof AI hosts | LLM judges on your own keys, packages, secrets, your private network, models you host, heavy processing |
+
+Default to hosted. Reach for a worker when the evaluation needs one of the
+right-hand column's things. Deciding *what* is worth evaluating is the
+`failproofai-eval-brainstorm` skill. This page is about building and running the
+worker.
+
+**How it works.** The worker only makes outbound HTTPS calls, and nothing connects
+in to it, so there is no port, no Service and no ingress:
+
+1. It registers its catalogue of evaluations.
+2. It claims finished sessions and runs each applicable evaluation.
+3. It submits the results under a heartbeat.
+
+Its results appear on the evaluations page tagged **customer**. Hosted results are
+tagged **managed**.
+
+**Nothing scores a session that has no `agent_start`/`agent_end`.** The worker
+evaluates sessions, and sessions exist only through those two events (`SKILL.md`
+§2). Get instrumentation landing first.
+
+## Ships in the SDK
+
+| | Python | TypeScript |
+|---|---|---|
+| Install | `pip install failproofai-sdk` | `npm install @failproofai/sdk` |
+| Import | `failproofai_sdk.evaluator` | `@failproofai/sdk/evaluator` |
+| Run | `python evaluator.py` (with `app.run_from_env()`) or `python -m failproofai_sdk.evaluator evaluator:app` | `npx failproofai-evaluator ./evals.js` (export named `app`; `./evals.js#other` for another) or `app.runFromEnv()` |
+
+Importing the tracing SDK does not load the evaluator. In TypeScript the reverse
+holds too; in Python, importing `failproofai_sdk.evaluator` also imports the tracing
+package and starts its flush thread (harmless, but it is there).
+
+## Write the evaluations
+
+Python:
+
+```python
+from failproofai_sdk.evaluator import ConditionResult, EvalResult, Evaluator, Metric, Score
+
+app = Evaluator(name="support-evals", version="2026.09.1")
+
+
+@app.eval(
+    "tool_efficiency",
+    version="1.0.0",
+    labels=["tools", "deterministic"],
+    when=lambda s: ConditionResult(s.count("tool_use") > 0, "no_tool_calls"),
+)
+def tool_efficiency(session):
+    calls = session.events_of_type("tool_use")
+    distinct = {e.payload.get("tool_name") for e in calls if e.payload.get("tool_name")}
+    value = len(distinct) / len(calls)
+    return EvalResult(
+        score=Score(value, passed=value >= 0.7),
+        metrics={"tool_call_count": Metric(len(calls), unit="events")},
+        reasoning=f"{len(distinct)} distinct tools across {len(calls)} calls",
+    )
+
+
+@app.eval(
+    "answer_relevance",
+    version="judge-v1",
+    labels=["llm_judge"],
+    when=lambda s: ConditionResult(s.count("model_response") > 0, "no_model_response"),
+    timeout_seconds=30,
+)
+async def answer_relevance(session):
+    answer = session.events_of_type("model_response")[-1].payload.get("content")
+    value, why = await ask_judge(answer)      # your model call: a 0-1 score and why
+    return EvalResult(score=Score(value, passed=value >= 0.7), reasoning=why)
+
+
+if __name__ == "__main__":
+    app.run_from_env()
+```
+
+TypeScript (same API, camelCase, options objects):
+
+```ts
+import { ConditionResult, EvalResult, Evaluator, Metric, Score } from "@failproofai/sdk/evaluator";
+
+export const app = new Evaluator({ name: "support-evals", version: "2026.09.1" });
+
+app.eval(
+  "tool_efficiency",
+  {
+    version: "1.0.0",
+    labels: ["tools", "deterministic"],
+    when: (s) => new ConditionResult(s.count("tool_use") > 0, "no_tool_calls"),
+  },
+  (session) => {
+    const calls = session.eventsOfType("tool_use");
+    const distinct = new Set(calls.map((e) => e.payload.tool_name).filter(Boolean));
+    const value = distinct.size / calls.length;
+    return new EvalResult({
+      score: new Score(value, { passed: value >= 0.7 }),
+      metrics: { tool_call_count: new Metric(calls.length, { unit: "events" }) },
+      reasoning: `${distinct.size} distinct tools across ${calls.length} calls`,
+    });
+  },
+);
+
+app.eval(
+  "answer_relevance",
+  {
+    version: "judge-v1",
+    labels: ["llm_judge"],
+    when: (s) => new ConditionResult(s.count("model_response") > 0, "no_model_response"),
+    timeoutSeconds: 30,
+  },
+  async (session) => {
+    const answer = session.eventsOfType("model_response").at(-1)?.payload.content;
+    const { value, why } = await askJudge(answer); // your model call
+    return new EvalResult({ score: new Score(value, { passed: value >= 0.7 }), reasoning: why });
+  },
+);
+```
+
+The evaluator module can be an ES module or CommonJS (`.js`, `.mjs`, `.cjs`, plain
+`tsc` output). For a `.ts` file, either compile it and point `failproofai-evaluator`
+at the `.js`, or call `await app.runFromEnv()` at the bottom and start it with
+`npx tsx evals.ts`.
+
+### The rules that bite
+
+- **The key is what results chart under.** It must match `^[a-z][a-z0-9_]*$`.
+  **Bump `version` whenever the logic changes.** Each result keeps the version that produced it, so a chart shows
+  exactly when new logic took over. Keys must be unique, and one worker holds at
+  most 100 evaluations.
+- **`when` is how you scope.** Return `ConditionResult(False, "<reason_code>")` to
+  skip a session, and the reason is recorded. A judge with no condition costs a
+  model call on every session.
+- **Result shapes:**
+  - `result_kind` / `resultKind` is `"score"` by default.
+  - For a `"metric"` or `"assertion"` evaluation, name one `metrics` or
+    `assertions` entry after the key: that entry is the result.
+  - Every `EvalResult` carries at least one and at most 25 scores, metrics or
+    assertions.
+  - `Score` values are 0–1, and anything else throws.
+- **Read payload keys off a real session.** Keys like `tool_name`, `content` and
+  `response` are whatever the agents emitted. The adapters add framework fields
+  under `fw_*`.
+- **Evaluations must yield.**
+  - Every evaluation is bounded by `timeout_seconds` / `timeoutSeconds`, and by
+    300 s when you set none.
+  - **Python:** a synchronous evaluation that overruns cannot be interrupted. Its
+    thread runs on, and a permanently blocked one leaks a thread per session. Write
+    judges and network calls as `async def`.
+  - **TypeScript:** a synchronous loop blocks the only thread, so no timeout can
+    fire. Write evaluations `async`.
+- **The session object:**
+  - Python: `session_id`, `agent_id`, `environment`, `started_at`, `ended_at`,
+    `events`, `count(type)`, `events_of_type(type)`.
+  - TypeScript: `sessionId`, `agentId`, `environment`, `startedAt`, `endedAt`,
+    `events`, `count(type)`, `eventsOfType(type)`.
+  - Each event has `id`, `ts`, `event_type` (TS `eventType`) and `payload`.
+
+## Run it
+
+Create a key with the **`evaluations:run`** permission under **Administration →
+Keys**. Inject it from your secret store; never paste it into a command line or an
+image.
+
+| Variable | Default | |
+|---|---|---|
+| `FAILPROOFAI_EVALUATOR_URL` | required | `https://app.befailproof.ai` for Cloud, or your instance. HTTPS unless loopback |
+| `FAILPROOFAI_EVALUATOR_TOKEN` | required | the `evaluations:run` key |
+| `FAILPROOFAI_EVALUATOR_WORKER_ID` | `<hostname>-<pid>` | names this worker |
+| `FAILPROOFAI_EVALUATOR_CONCURRENCY` | `1` | sessions scored at once by this process |
+| `FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS` | `30` | per request to Failproof AI |
+| `FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS` | `60` | how long a stopping worker waits for runs in flight |
+| `FAILPROOFAI_EVALUATOR_ALLOW_INSECURE_HTTP` | `false` | plain HTTP to a non-loopback URL. **Cleartext token and transcripts.** Isolated dev networks only |
+| `FAILPROOFAI_EVALUATOR_MODULE` | — | the module when the CLI is given none: `module:attr` (Python), `path#export` (TS) |
+
+Local smoke run against Cloud:
+
+```bash
+export FAILPROOFAI_EVALUATOR_URL=https://app.befailproof.ai
+export FAILPROOFAI_EVALUATOR_TOKEN="$(your-secret-store read failproofai/evaluator)"
+python evaluator.py              # or: npx failproofai-evaluator ./evals.js
+```
+
+Then finish a session from an instrumented agent and watch for a **customer**
+result on it. Evaluation runs forward: a worker started now scores sessions that
+finish while it runs. The dashboard's "score sessions you already have" backfill
+covers hosted evaluations only; there is no backfill for a worker's own evaluations.
+
+## Deploy it as a pod
+
+Treat it as a long-running, outbound-only worker:
+
+```dockerfile
+# Python
+FROM python:3.12-slim
+RUN pip install --no-cache-dir failproofai-sdk   # plus whatever your judges need
+COPY evaluator.py /app/evaluator.py
+CMD ["python", "/app/evaluator.py"]
+```
+
+```dockerfile
+# TypeScript (compiled to dist/evals.js)
+FROM node:22-slim
+WORKDIR /app
+COPY package*.json ./
+RUN npm ci --omit=dev
+COPY dist/ ./dist/
+CMD ["npx", "failproofai-evaluator", "./dist/evals.js"]
+```
+
+- **Secrets:**
+  - `FAILPROOFAI_EVALUATOR_TOKEN` comes from a Secret (Kubernetes) or your secret
+    manager.
+  - The judge's own model key belongs there too.
+  - Egress must reach the Failproof AI URL and your model endpoint. No ingress is
+    needed.
+- **Shutdown:** both workers stop on `SIGTERM`/`SIGINT` and drain runs in flight
+  for up to `FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS`. Give the orchestrator
+  more grace than that. In Kubernetes, set `terminationGracePeriodSeconds` above
+  the drain timeout, e.g. 90 for the default 60. Otherwise a rolling deploy
+  hard-kills evaluations mid-run.
+- **Scaling:**
+  - Raise `FAILPROOFAI_EVALUATOR_CONCURRENCY` for I/O-bound judges.
+  - Add replicas for more throughput. Each replica claims its own sessions.
+  - Leave `WORKER_ID` unset, or make it unique per replica: the default
+    `<hostname>-<pid>` already is.
+- **Releasing new logic.** Ship the image with a bumped eval `version`. Removing an
+  evaluation from the worker, or stopping the worker, stops it running. There is
+  nothing to disable in the dashboard, and worker-registered evaluations are not
+  listed on the eval authoring page.
+
+## Debugging a worker that scores nothing
+
+Work through these in order:
+
+1. **Does it start?**
+   - A missing URL or token fails at startup with the variable's name.
+   - An HTTP URL to a non-loopback host is refused unless you opt in.
+2. **Is the key right?** It needs `evaluations:run` in the same organisation as the
+   agents.
+3. **Are sessions finishing?** No `agent_end` means no finished session, and
+   nothing to claim. Check the instrumentation (`SKILL.md` §5).
+4. **Does `when` skip everything?** The skip reason is recorded per session. A
+   condition reading a payload key the agents never emit skips every session.
+5. **Are evaluations timing out?** Check the worker's own log. Make judges `async`
+   and give them a `timeout_seconds` / `timeoutSeconds` that covers one model
+   call.

package/.next/standalone/sdk/python/skill/references/events.md

@@ -1,5 +1,11 @@
 # Event catalog
 
+> TypeScript/JavaScript: the same 15 events on `failproofai.event`, camelCase
+> (`toolUse({ toolName, toolCallId })`) taking one options object; the file written
+> is identical. Name map in `typescript.md`. Code below is Python, and where it
+> says `ValueError` or `TypeError`, TypeScript throws `TypeError` or a plain
+> `Error` (listed in `typescript.md` → *The contract, in TypeScript*).
+
 Every method lives on `failproofai_sdk.event`, is **keyword-only**, and returns `None`.
 Nothing here blocks or does I/O — the call queues the event and returns.
 
@@ -93,16 +99,19 @@ its original) oldest-first, and will keep bracketing the wrong pairs. **Always
 pass `duration_ms` on the `model_response` too** — that is what keeps the reported
 duration right even when the bracketing is wrong.
 
-**Correlation is a process-wide map keyed by your ids.** The SDK holds open
-starts there until their matching end arrives. Consequences, in order of how much
+**Correlation is a map keyed by session, kind and your id.** The SDK holds open
+starts there until their matching end arrives. An id only has to be unique *within
+one session, for one kind* (the second bullet). Consequences, in order of how much
 they hurt:
 
-- **Per-run counters are unsafe.** `call_1`, `call_2` — common in home-grown loops
-  — collide across overlapping runs. The failure is not the missing duration the
-  docs might lead you to expect; it is a *plausible wrong number attributed to the
-  wrong run*, which is worse. Reuse your framework's id (Anthropic and OpenAI
-  tool-call ids are globally unique), or a `uuid4`. `failproofai_sdk.tool_call()`
-  generates a `uuid4` for you.
+- **Per-run counters are unsafe inside a shared session.** `call_1`, `call_2` —
+  common in home-grown loops — are fine while every run has its own session. They
+  collide when two runs share one: a retried job that reuses its job id as the
+  session, or two agents in one session each counting from `call_1`. The failure is
+  not the missing duration the docs might lead you to expect; it is a *plausible
+  wrong number attributed to the wrong run*, which is worse. Reuse your framework's
+  id (Anthropic and OpenAI tool-call ids are globally unique), or a `uuid4`.
+  `failproofai_sdk.tool_call()` generates a `uuid4` for you.
 - **`tool_call_id` and `hook_id` no longer collide with each other.** They live in
   separate namespaces, so a `hook_completed(hook_id="x")` cannot pair with a
   pending `tool_use(tool_call_id="x")`. The key is `<kind>:<session_id>:<id>`, so

package/.next/standalone/sdk/python/skill/references/frameworks.md

@@ -1,5 +1,8 @@
 # Framework integrations
 
+> TypeScript/JavaScript (LangChain.js/LangGraph.js, Vercel AI SDK, Mastra,
+> LlamaIndex.TS, Next.js): see `typescript.md`. This page is the Python SDK.
+
 If the agent runs on LangChain/LangGraph, CrewAI, LlamaIndex or Pydantic AI, you
 do not write the instrumentation — you turn it on. The adapters ship inside the
 SDK wheel and are imported only when you ask for them.

package/.next/standalone/sdk/python/skill/references/install.md

@@ -1,5 +1,8 @@
 # Installing the SDK
 
+> TypeScript/JavaScript: `npm install @failproofai/sdk` — see `typescript.md`.
+> This page is the Python distribution.
+
 ```bash
 pip install failproofai-sdk        # or: uv add failproofai-sdk
 ```

package/.next/standalone/sdk/python/skill/references/integration.md

@@ -1,5 +1,9 @@
 # Writing the integration
 
+> TypeScript/JavaScript: the same three scopes exist as `session()`, `agent()` and
+> `toolCall()` on `AsyncLocalStorage`, and a hand-built loop is three edit sites —
+> see `typescript.md` → *An agent with no framework*. This page is the Python SDK.
+
 Identity is ambient. Bind it once per run with a context manager and every
 `event.*` call inside — including calls in functions that have never heard of
 Failproof AI — lands on the right session and agent.
@@ -50,8 +54,8 @@ a session is *defined* as something that emitted `agent_start`.
   one run rather than splitting it in two.
 - `parent_id` defaults to the enclosing agent from the scope stack. Pass
   `parent_id=None` to force a root span, or a string to override.
-- `tool_call_id` defaults to `uuid4().hex` — unique process-wide, which is what
-  the correlation map needs (see `events.md`).
+- `tool_call_id` defaults to `uuid4().hex` — unique everywhere, so it can never
+  collide in the correlation map (see `events.md`).
 
 ## What `agent()` does on the way out