@anvia/cli 1.1.1 → 1.3.0

package/dist/skills/anvia-channels/scripts/check-channels.sh

@@ -0,0 +1,79 @@
+#!/bin/sh
+# Heuristic checks for Anvia channels code in an app directory.
+# Usage: sh scripts/check-channels.sh [--dir <app-root>]
+# Fails with a list of violations; passes silently with "channels OK".
+
+DIR="."
+if [ "$1" = "--dir" ] && [ -n "$2" ]; then
+  DIR="$2"
+fi
+
+fail=0
+violation() {
+  echo "VIOLATION: $1"
+  fail=1
+}
+warning() {
+  echo "WARNING: $1"
+}
+
+ROOTS_FOUND=0
+for root in "$DIR/src" "$DIR/app" "$DIR/lib" "$DIR/server"; do
+  if [ -d "$root" ]; then
+    ROOTS_FOUND=1
+    break
+  fi
+done
+if [ "$ROOTS_FOUND" -eq 0 ]; then
+  echo "ERROR: no src/app/lib/server directory under '$DIR' — nothing was checked."
+  echo "Run from the app root or pass --dir <app-root>."
+  exit 1
+fi
+
+scan() {
+  # $1 = include pattern; prints matching files under the usual roots.
+  grep -rln --include="$1" \
+    -e 'createChannelAgent(' -e 'serveChannelAgent(' -e 'sendChannelMessage(' \
+    -e 'ChannelAgentService' -e 'ChannelEventHandler' -e 'ChannelInteractionStore' \
+    -e 'telegram(' -e 'discord(' -e 'slack(' \
+    "$DIR/src" "$DIR/app" "$DIR/lib" "$DIR/server" 2>/dev/null
+}
+
+FILES=$( { scan '*.ts'; scan '*.tsx'; } | sort -u)
+
+if [ -z "$FILES" ]; then
+  echo "channels OK (no channel code found)"
+  exit 0
+fi
+
+# 1. Platform credentials come from the environment, never literals.
+creds=$(echo "$FILES" | xargs grep -l -E '(appToken|botToken|token): *"' 2>/dev/null)
+if [ -n "$creds" ]; then
+  violation "hardcoded channel credential in $(echo "$creds" | tr '\n' ' ') — read tokens from the environment (see references/adapters.md)."
+fi
+
+# 2. Unbounded text must go through sendChannelMessage (it splits), not channel.send.
+if echo "$FILES" | xargs grep -E '\.send\(\{[^}]*\$\{' 2>/dev/null | grep -q .; then
+  violation "dynamic text through channel.send() can be truncated — use sendChannelMessage() so long text splits (see references/delivery.md)."
+fi
+
+# 3. The bridge swallows errors unless onError reports them (warning only).
+if echo "$FILES" | xargs grep -l 'createChannelAgent(' 2>/dev/null | grep -q .; then
+  echo "$FILES" | xargs grep -l 'onError:' 2>/dev/null | grep -q . || {
+    warning "createChannelAgent without onError anywhere — stage failures will be silent (see references/channel-agent.md)."
+  }
+fi
+
+# 4. A started service needs a visible shutdown path (warning only).
+if echo "$FILES" | xargs grep -q -E '(service|channelAgent|agent)\.start\(\)' 2>/dev/null; then
+  echo "$FILES" | xargs grep -q -E 'stop\(\)|SIGINT|SIGTERM|process\.once|process\.on' 2>/dev/null || {
+    warning "service.start() without a visible stop()/signal handler — add a shutdown path (see references/channel-agent.md)."
+  }
+fi
+
+if [ "$fail" -eq 0 ]; then
+  echo "channels OK"
+  exit 0
+fi
+echo "See skills/anvia-channels/references/ for fixes."
+exit 1

package/dist/skills/anvia-chat/SKILL.md

@@ -0,0 +1,61 @@
+---
+name: anvia-chat
+description: Wire an Anvia chat UI end to end — server stream route, client transport, React hook, and UI primitives.
+---
+
+# Anvia Chat Skill
+
+Use this skill when the user wants a chat or completion UI backed by Anvia:
+new chat routes, transports, `useChat`/`useCompletion` wiring, message rendering,
+resumable streams, agent tool-approval interactions, or `anvia add chat` work.
+
+## Process
+
+1. Build the server route first (`references/server-protocol.md`).
+2. Pick the transport and state conversions (`references/transports-state.md`).
+3. Wire the React controller and render UI primitives (`references/react-ui.md`).
+4. Run `scripts/check-chat-boundary.sh` from the app root before claiming done.
+
+## Minimal slice
+
+```ts
+// Server route: core events -> client protocol -> framed response.
+import { completionToClientStream, parseClientStreamRequest } from "@anvia/client";
+import { streamCompletion } from "@anvia/core";
+import { createClientStreamResponse } from "@anvia/server";
+
+const body = parseClientStreamRequest(await request.json());
+if (body.type !== "messages") throw new Error("This completion endpoint accepts messages only.");
+const events = completionToClientStream({
+  events: streamCompletion({ model, messages: body.messages }),
+});
+return createClientStreamResponse({ events }); // JSONL by default
+```
+
+```tsx
+// App: explicit transport, hook owns state, primitives render it.
+import { createHttpClientTransport } from "@anvia/client";
+import { useChat } from "@anvia/react";
+import { ChatProvider, ComposerPrimitive, ThreadPrimitive } from "@anvia/react-ui";
+
+const transport = createHttpClientTransport({ endpoint: "/api/chat" });
+const chat = useChat({ transport });
+
+<ChatProvider controller={chat}>
+  <ThreadPrimitive.Root>
+    <ThreadPrimitive.Viewport>
+      <ThreadPrimitive.Messages />
+    </ThreadPrimitive.Viewport>
+    <ComposerPrimitive.Root>
+      <ComposerPrimitive.Input placeholder="Send a message..." />
+      <ComposerPrimitive.Submit>Send</ComposerPrimitive.Submit>
+    </ComposerPrimitive.Root>
+  </ThreadPrimitive.Root>
+</ChatProvider>;
+```
+
+## Output
+
+Keep the answer to the smallest working vertical slice: one route, one transport,
+one hook, one render tree. Point to the relevant reference file instead of
+pasting its contents into chat.

package/dist/skills/anvia-chat/references/react-ui.md

@@ -0,0 +1,104 @@
+# React and UI
+
+## Controllers (`@anvia/react`)
+
+`useChat` and `useCompletion` require an explicit transport boundary:
+
+```ts
+useChat({ transport });
+useChat({
+  transport: createDirectClientTransport({
+    handler: ({ request, abortSignal }) => handleChat({ request, abortSignal }),
+  }),
+});
+```
+
+`useChat`:
+
+- Consumes only framed `ClientStreamFrame` values.
+- Status: `ready | submitted | streaming | waiting | error`.
+- Keeps readonly `UIMessage[]` locally; sends core `Message[]` in
+  `ClientStreamRequest`.
+- Exposes canonical events via `onEvent` and the returned `events` array.
+- Exposes aggregate latest-run usage via `runUsage`; each assistant message
+  keeps its own provider-generation usage.
+- Actions: `sendMessage` (accepts attachments and metadata), `regenerate`,
+  `stop`, `reset`, `setMessages`, `resume()`. State also exposes `suggestions`,
+  `contextUsage`, `text`, and `streamId`. Submit through `sendMessage` rather
+  than mutating `messages`.
+- Supports resumable streams (`resume: { key }`, paired with the server's
+  resumable response) and unified Agent interaction state.
+
+When `chat.status === "waiting"`, render `chat.interactions.pending` and resume
+through the same transport boundary:
+
+```ts
+await chat.respondToInteraction({
+  interactionId,
+  response: { type: "tool-approval", approved: true },
+});
+```
+
+Render those prompts with `HumanInputPrimitive`, or drive them headlessly with
+`useApproval` / `useQuestion` — both from `@anvia/react-ui`.
+
+`useCompletion({ transport })` is single-turn: `complete({ prompt })` or manage
+`input` and call `submit()`. Each call replaces the previous completion, events,
+and usage — it never accumulates messages. The UI counterpart is
+`CompletionProvider` + `CompletionPrimitive`.
+
+`useSmoothStreamText` / `useSmoothStreamItems` only smooth presentation. They do
+not change protocol events or message state.
+
+## Headless primitives (`@anvia/react-ui`)
+
+Primitives are headless: no stylesheet, style via `className` or `asChild`.
+The DOM contract is ARIA attributes plus `data-state` / `data-role`.
+
+- Keep `useChat` as the owner of transport and `UIMessage[]` state; wrap it with
+  `ChatProvider` and render `ThreadPrimitive` / `MessagePrimitive` /
+  `ComposerPrimitive`.
+- Control `ComposerPrimitive.Root` with `input` / `attachments` props when
+  needed; use `submitMessage` for custom payloads. `keepMounted` keeps empty
+  wrappers for layout.
+- `ComposerPrimitive.Input` is Tiptap-backed. Configure `Root` with `triggers`
+  for inline `@` / `/` / `$` entity chips; selections submit under
+  `metadata.composer.entities`. Entity `data` must be finite strict JSON (no
+  class instances, cycles, accessors, symbols, `undefined`, non-finite numbers).
+  Use `ComposerPrimitive.TextareaInput` for plain textarea behavior.
+- Streaming reveal is opt-in and display-only. Keep the lifecycle mounted after
+  streaming stops so the buffered tail drains; `MessagePrimitive.Parts` holds
+  later tool parts behind unrevealed text:
+
+```tsx
+<MessagePrimitive.Parts
+  stream={{
+    isStreaming:
+      chat.status === "streaming" &&
+      message.role === "assistant" &&
+      chat.messages.at(-1)?.id === message.id,
+    resetKey: message.id,
+    flushImmediately: chat.status === "error",
+  }}
+>
+  {(part) => (part.type === "text" ? <MessagePrimitive.Markdown /> : <MessagePrimitive.Part />)}
+</MessagePrimitive.Parts>
+```
+
+- For app-owned text state, `StreamMarkdown` from `@anvia/react-ui/stream` is a
+  context-free renderer: pass displayed text as `content`, set `live` only for
+  the growing tail, and style `[data-state="revealing"]` in the app.
+
+## Editable components (`@anvia/cli`)
+
+For styled, app-owned components instead of headless composition:
+
+```sh
+pnpm dlx @anvia/cli add chat
+```
+
+Items: `chat`, `thread`, `message`, `composer`, `attachment`, `markdown`,
+`tool-fallback`. `add` writes below the `components` alias (normally
+`src/components/anvia`) and installs the matching `@anvia/react-ui` release.
+`@anvia/cli` installs the `[data-state="revealing"]` animation with the
+`markdown`, `message`, `thread`, and `chat` items.

package/dist/skills/anvia-chat/references/server-protocol.md

@@ -0,0 +1,53 @@
+# Server Protocol
+
+Server routes adapt native runtime events at the boundary, then frame them.
+`@anvia/core` owns native completion and Agent events; `@anvia/client` owns the
+wire adapters; `@anvia/server` owns the HTTP framing.
+
+## Completion route
+
+```ts
+import { completionToClientStream, parseClientStreamRequest } from "@anvia/client";
+import { streamCompletion } from "@anvia/core";
+import { createClientStreamResponse } from "@anvia/server";
+
+const body = parseClientStreamRequest(await request.json());
+if (body.type !== "messages") throw new Error("This completion endpoint accepts messages only.");
+const events = completionToClientStream({
+  events: streamCompletion({ model, messages: body.messages }),
+});
+return createClientStreamResponse({ events }); // JSONL by default
+```
+
+## Agent route
+
+Use `agentToClientStream` instead of `completionToClientStream` for Agent events
+(it preserves nested-agent scope). Accept both request shapes: `messages` starts
+a run, `interaction_response` resumes one. Validate with `parseClientStreamRequest`.
+
+For application-defined stream data, `customAgentEventsToClientStream` wraps an
+agent event stream and `mapCustomEvent` maps app events into `data` events; the
+client validates them against `dataSchemas` (see
+`references/transports-state.md`) and they surface as data parts.
+
+## Framing rules
+
+- `createClientStreamResponse({ events })` always emits `stream_start`, ordered
+  `stream_event` frames, then `stream_end`, with header
+  `x-anvia-stream-protocol: anvia.client.v3`. This is the client protocol claim.
+- Pass `format: "sse"` for SSE framing; JSONL is the default.
+- For resumable streams, pass `{ resumable: { streamId, store } }` when creating
+  the response and call `resumeClientStreamResponse({ streamId, after, store })`
+  for resume requests. The client side pairs this with
+  `useChat({ transport, resume: { key } })`.
+- `createEventStreamResponse` / `resumeEventStreamResponse` (and the lower-level
+  `createJsonlStream`, `createSseStream`, `createResumableStream`,
+  `resumeStreamEvents`, `createMemoryResumableStreamStore`) are generic helpers.
+  They serialize the application's own event type and do NOT claim the Anvia
+  client protocol. Use them only for endpoints that intentionally expose a
+  different event contract.
+
+## Error exposure
+
+Errors are masked by default. Use `mapError` only at the server adapter boundary
+when the application intentionally exposes a safe error shape.

package/dist/skills/anvia-chat/references/transports-state.md

@@ -0,0 +1,71 @@
+# Transports and State
+
+## The boundary rule
+
+Client-side `UIMessage[]` never crosses the server boundary. The request carries
+core `Message[]`; the response carries framed `ClientStreamEvent` records.
+Convert explicitly at the edges:
+
+- `uiMessagesToMessages()` — client state to request. Rejects partial tool calls
+  instead of replaying incomplete JSON or inventing empty arguments.
+- `messagesToUIMessages()` — persisted core messages to UI state.
+- `applyClientStreamEvent(messages, event)` — apply canonical stream events to
+  `UIMessage[]`.
+- `parseUIMessage` / `parseUIMessages` — validate externally loaded UI state.
+- `parseClientStreamRequest` / `parseClientStreamEvent` / `parseClientStreamFrame`
+  — validate public wire input at runtime.
+
+`UIMessage.metadata` is application-owned and round-trips unchanged. Run details
+(run ID, usage, context usage, status, trace correlation) live separately in
+`UIMessage.generation`; converting persisted core messages hydrates usage into
+that field and restores persisted sources as UI parts.
+
+## Transports
+
+```ts
+// HTTP: framed JSONL or SSE, validates protocol header, frame order,
+// stream identity, and event IDs.
+const transport = createHttpClientTransport({ endpoint: "/api/chat" });
+
+// Direct (same process / tests): same framed contract, no HTTP.
+const transport = createDirectClientTransport({
+  handler: ({ request, abortSignal }) => handleChat({ request, abortSignal }),
+});
+```
+
+- Import transports, protocol types, `UIMessage`, and conversion helpers from
+  `@anvia/client`. `@anvia/react` deliberately does not re-export them.
+- Low-level JSONL/SSE readers live in `@anvia/client/transport`. They do not
+  imply the client protocol — use them only for non-protocol endpoints.
+- `UIToolMessagePart` states are exact: `input-streaming` carries raw partial
+  text, `input-available` carries parsed JSON input, terminal `output-available`
+  or `error` parts retain that input with their result.
+- Tool-call start/delta/end events are automatic when the provider streams
+  arguments.
+
+## Agent interactions
+
+- The browser never receives an `AgentContinuation`. The server retains it and
+  atomically claims it by interaction ID. Agent interaction wire contracts come
+  from the browser-safe `@anvia/core/agent/interactions` subpath — importing the
+  client never loads the Agent runtime.
+- Name interaction types from `@anvia/core/agent/interactions` when the app
+  needs them.
+
+## App-specific data
+
+Application stream data is explicit and schema-validated:
+
+```ts
+type AppData = {
+  citation_preview: { title: string; url: string };
+};
+
+const transport = createHttpClientTransport<ClientStreamRequest, AppData>({
+  endpoint: "/api/chat",
+  dataSchemas: { citation_preview: citationPreviewSchema },
+});
+```
+
+Non-JSON tool outputs require an explicit `mapOutput`: returning `undefined`
+omits the output, returning `null` exposes JSON `null`.

package/dist/skills/anvia-chat/scripts/check-chat-boundary.sh

@@ -0,0 +1,70 @@
+#!/bin/sh
+# Check the Anvia chat boundary rules in the current app directory.
+# Usage: sh scripts/check-chat-boundary.sh [--dir <app-root>]
+# Fails with a list of violations; passes silently with "chat boundary OK".
+
+DIR="."
+if [ "$1" = "--dir" ] && [ -n "$2" ]; then
+  DIR="$2"
+fi
+
+ROOTS_FOUND=0
+for root in "$DIR/src" "$DIR/app" "$DIR/components"; do
+  if [ -d "$root" ]; then
+    ROOTS_FOUND=1
+    break
+  fi
+done
+if [ "$ROOTS_FOUND" -eq 0 ]; then
+  echo "ERROR: no src/app/components directory under '$DIR' — nothing was checked."
+  echo "Run from the app root or pass --dir <app-root>."
+  exit 1
+fi
+
+fail=0
+violation() {
+  echo "VIOLATION: $1"
+  fail=1
+}
+
+# 1. Client bundle must not import the Agent runtime or server framing helpers.
+if grep -rn --include='*.ts' --include='*.tsx' \
+  -e 'from "@anvia/core/agent"' \
+  -e 'from "@anvia/server"' \
+  "$DIR/src" "$DIR/app" "$DIR/components" 2>/dev/null | grep -v '^Binary' | grep -q .; then
+  violation "client code imports @anvia/core/agent runtime or @anvia/server; keep AgentContinuation and framing server-side (see references/transports-state.md)."
+fi
+
+# 2. useChat/useCompletion require an explicit transport.
+if grep -rln --include='*.tsx' --include='*.ts' \
+  -e 'useChat(' -e 'useCompletion(' "$DIR/src" "$DIR/app" "$DIR/components" 2>/dev/null | grep -q .; then
+  if ! grep -rn --include='*.tsx' --include='*.ts' \
+    -e 'createHttpClientTransport' -e 'createDirectClientTransport' \
+    "$DIR/src" "$DIR/app" "$DIR/components" 2>/dev/null | grep -q .; then
+    violation "useChat/useCompletion used without createHttpClientTransport or createDirectClientTransport (see references/react-ui.md)."
+  fi
+fi
+
+# 3. UIMessage must not be sent to the server; requests carry core Message[].
+if grep -rn --include='*.ts' --include='*.tsx' \
+  -e 'UIMessage\[\].*fetch' -e 'body:.*UIMessage' -e 'JSON.stringify(.*uiMessages\|.*messages: chat.messages)' \
+  "$DIR/src" "$DIR/app" 2>/dev/null | grep -q .; then
+  violation "possible UIMessage sent over the wire; convert with uiMessagesToMessages first (see references/transports-state.md)."
+fi
+
+# 4. Server route must validate the request and claim (or not claim) the protocol.
+if grep -rln --include='*.ts' \
+  -e 'createClientStreamResponse' -e 'agentToClientStream' -e 'completionToClientStream' \
+  "$DIR/src" "$DIR/app" 2>/dev/null | grep -q .; then
+  if ! grep -rn --include='*.ts' -e 'parseClientStreamRequest' \
+    "$DIR/src" "$DIR/app" 2>/dev/null | grep -q .; then
+    violation "stream route does not call parseClientStreamRequest (see references/server-protocol.md)."
+  fi
+fi
+
+if [ "$fail" -eq 0 ]; then
+  echo "chat boundary OK"
+  exit 0
+fi
+echo "See skills/anvia-chat/references/ for fixes."
+exit 1

package/dist/skills/anvia-evals/SKILL.md

@@ -0,0 +1,51 @@
+---
+name: anvia-evals
+description: Evaluate Anvia agents and retrieval — deterministic metrics, semantic similarity, LLM judges, RAG quality, and CLI eval runs.
+---
+
+# Anvia Evals Skill
+
+Use this skill when the user wants to measure output quality: scoring a target
+function or agent over cases, picking metrics, adding an LLM judge, checking RAG
+grounding, or wiring evals into CI.
+
+## Process
+
+1. Start deterministic (`references/metrics.md`) — exact match, contains, semantic similarity.
+2. Add judges only for what strings cannot check (`references/judges.md`).
+3. Run it right (`references/running.md`) — `runEvalSuite` vs `runEvalCli`, negative controls, expectations.
+4. Run `scripts/check-evals.sh` from the app root before claiming done.
+
+## Minimal slice
+
+```ts
+import { contains, exactMatch, runEvalCli } from "@anvia/core/evals";
+
+await runEvalCli({
+  name: "support-basic-metrics",
+  cases: [
+    {
+      id: "refund-window",
+      input: "When can I request a refund?",
+      expected: "Refunds are available for 30 days.",
+    },
+    {
+      id: "wrong-refund-window",
+      input: "Negative control: when can I request a refund?",
+      expected: "Refunds are available for 30 days.",
+    },
+  ],
+  target: async (input) => answerSupportQuestion(input),
+  metrics: [exactMatch(), contains({ expected: ({ case: testCase }) => "30 days" })],
+  expectations: {
+    outcomes: { "wrong-refund-window": { exact_match: "fail", contains: "fail" } },
+  },
+  exitCode: true,
+});
+```
+
+## Output
+
+Every behavior claim needs a case. Prefer cheap deterministic metrics; spend
+LLM-judge budget where wording varies. Point to the relevant reference file
+instead of pasting its contents into chat.

package/dist/skills/anvia-evals/references/judges.md

@@ -0,0 +1,61 @@
+# LLM Judges
+
+Use judges when wording varies but meaning is checkable — policy compliance,
+tone, grading scales. Judges need a model and explicit pass criteria.
+
+```ts
+import { llmJudge, llmScore } from "@anvia/core/evals";
+
+llmJudge({
+  model,
+  schema: z.object({ passed: z.boolean(), reason: z.string() }),
+  passes: (judgment) => judgment.passed,
+  instructions:
+    "Decide whether the output satisfies the expected support policy. Return passed and a short reason.",
+});
+llmScore({
+  model,
+  threshold: 0.8,
+  criteria: [
+    "The output answers the user's question directly.",
+    "The output matches the expected support policy.",
+    "The output does not add unsupported policy details.",
+  ],
+});
+```
+
+## RAG quality metrics
+
+Cases carry `context` and `retrievalContext`; the metrics check grounding:
+
+```ts
+import { answerRelevancy, faithfulness, hallucination, gEval } from "@anvia/core/evals";
+
+answerRelevancy({ model: judgeModel, threshold: 0.8 });
+faithfulness({ model: judgeModel, threshold: 0.8 });
+hallucination({ model: judgeModel, threshold: 0.1 }); // lower is better — note the direction
+gEval({
+  name: "correctness",
+  model: judgeModel,
+  evaluationParams: ["actualOutput", "expectedOutput"],
+  evaluationSteps: [
+    "Check whether the answer preserves the expected refund window.",
+    "Allow different wording when the policy meaning is unchanged.",
+  ],
+  threshold: 0.8,
+});
+```
+
+Related: `promptAlignment`, `jsonCorrectness`, `abstention`, `summarization` for
+specialized outputs; `turnRelevancy` and `knowledgeRetention` for conversations
+(see the cookbook's conversation-quality example).
+
+## Rules
+
+- Pin the judge model (`modelId`) — a silently upgraded judge re-grades history.
+- Prefer a different (usually stronger) model for judging than for generating;
+  self-judging inflates scores.
+- `threshold` direction matters: most metrics pass at/above it, `hallucination`
+  passes at/below it.
+- Keep judge `instructions` / `criteria` / `evaluationSteps` in the eval file
+  under version control — they are part of the test, not ambient prompt text.

package/dist/skills/anvia-evals/references/metrics.md

@@ -0,0 +1,45 @@
+# Deterministic Metrics
+
+All metrics come from `@anvia/core/evals`. Metrics compare the target `output`
+against the case `expected` and report `pass | fail | invalid` with scores and
+comments. Start here — they cost nothing and never flake.
+
+```ts
+import { contains, exactMatch, notContains, matches, semanticSimilarity } from "@anvia/core/evals";
+
+metrics: [
+  exactMatch(), // deep-equal to expected — structurally equal objects/arrays pass
+  contains({
+    expected: ({ case: testCase }) =>
+      testCase.id === "billing-owner" ? "Workspace owners" : "30 days",
+  }),
+  notContains({ expected: "90 days" }), // forbid known-bad phrases
+  matches({ expected: /^\d+ days$/ }), // regex shape checks
+  semanticSimilarity({ model: embeddingModel, threshold: 0.8 }), // embedding closeness, no judge LLM needed
+];
+```
+
+- `expected` (and `actual`) accept selector functions over
+  `{ suiteName, case, output, signal }` — reach per-case metadata as
+  `case.metadata`. Use them to vary expectations per case instead of writing one
+  metric per case.
+- `containsAll` / `containsAny` check multi-fragment outputs; `maxLength` and
+  `requiredFields` guard shape for structured outputs.
+- `defineMetric` builds custom metrics when nothing fits — keep the metric pure
+  (input, output, expected in; outcome out) so results stay reproducible.
+
+## Agent outputs
+
+Metrics work over any target output. For agents, project the response first:
+
+```ts
+contains<string, AgentResponse, string>({ actual: ({ output }) => output.output });
+exactMatch<string, AgentResponse, string>({
+  name: "not_blank",
+  actual: ({ output }) => output.output.trim().length > 0,
+  expected: true,
+});
+```
+
+Always include a not-blank style metric for agent targets — an empty output
+passing every content metric is the classic false green.

package/dist/skills/anvia-evals/references/running.md

@@ -0,0 +1,62 @@
+# Running Evals
+
+## Suite vs CLI
+
+```ts
+import { agentEvalTarget, runEvalCli, runEvalSuite } from "@anvia/core/evals";
+
+// Library use: inspect results programmatically.
+const result = await runEvalSuite({ name: "support-agent-target", cases, target, metrics });
+console.log({
+  passed: result.metrics.passed,
+  failed: result.metrics.failed,
+  invalid: result.metrics.invalid,
+});
+
+// CLI/CI use: pretty table, asserted expectations, process exit code.
+await runEvalCli({
+  name: "support-basic-metrics",
+  cases,
+  target,
+  metrics,
+  expectations,
+  exitCode: true,
+});
+```
+
+- `runEvalSuite` returns full per-case, per-metric results — use it in scripts
+  and notebooks. It also takes execution controls for larger suites:
+  `concurrency`, `caseTimeoutMs`, `failFast`, `caseIds`/`caseFilter`, `shard`,
+  `signal`, `onProgress`, plus reporters and `cost`/`targetUsage` capture.
+- `runEvalCli` prints tables (`printEvalResult` / `formatEvalResult` under the
+  hood), asserts `expectations`, and with `exitCode: true` fails CI on
+  regressions. Always set `exitCode: true` in CI. Output is tunable via
+  `format: "pretty" | "json" | "quiet"`, `redact`, output writers, and
+  `maxValueLength`.
+- `expectations.outcomes` pins known outcomes per case — including intentional
+  failures (negative controls), so a "fixed" negative control fails loudly
+  instead of silently flipping green. `expectations.totals` pins aggregate
+  counts per metric or for the whole run; `defineEvalExpectations`,
+  `assertEvalTotals`, `assertEvalOutcomes`, and `evalExitCode` are the
+  programmatic helpers behind the CLI.
+
+## Targets
+
+- Plain functions: `target: async (input) => answer(input)`.
+- Agents: wrap with `agentEvalTarget({ agent, request: ({ input }) => ({ prompt: input }) })`
+  and project `output.output` in metrics (see `references/metrics.md`).
+- `defineEvalCases` / `defineEvalSuite` / `createEvalTypes` add type safety to
+  large suites — adopt them when cases grow past a handful.
+
+## Case design
+
+- Every case needs `id`, `input`, `expected`. Keep ids stable — `expectations`
+  and history key off them.
+- Include negative controls: inputs the target must get wrong (or must refuse),
+  pinned as expected failures. Without them you cannot tell a strict metric
+  from a broken target.
+- For RAG cases add `context` (what the generator saw) and `retrievalContext`
+  (what retrieval returned) so grounding metrics have something to check.
+- Report evals to observability (Langfuse eval reporting, trace refs via
+  `resolveEvalTraceRef`) when runs must be auditable — see the cookbook's
+  langfuse eval-reporting example.