@respan/cli 0.9.0 → 0.10.0

package/dist/lib/skill-refs.generated.js

@@ -0,0 +1,1168 @@
+// AUTO-GENERATED by scripts/embed-skill-refs.mjs
+// Source: skills/references/{tracing,prompts,evals,monitors}.md
+// Do not edit directly — edit the markdown sources and rebuild.
+export const TRACING_MD = `# Tracing Reference
+
+Advanced tracing configuration: decorators, context propagation, processors, and span attributes.
+
+Docs: \`https://respan.ai/docs/documentation/features/tracing/traces/tracing-sdk.md\`
+
+---
+
+## Decorators
+
+Wrap functions to create structured span hierarchies. All decorators share the same signature.
+
+### Types
+
+| Decorator | Purpose |
+|-----------|---------|
+| \`@workflow\` / \`withWorkflow\` | Root span — top-level pipeline |
+| \`@task\` / \`withTask\` | Step within a workflow |
+| \`@agent\` / \`withAgent\` | Agent loop span |
+| \`@tool\` / \`withTool\` | Tool/function call span |
+
+### Python
+
+\`\`\`python
+from respan import Respan, workflow, task, agent, tool
+
+Respan()
+
+@workflow(name="write_article")
+def write_article(topic: str):
+    outline = generate_outline(topic)
+    return write_draft(outline)
+
+@task(name="generate_outline")
+def generate_outline(topic: str):
+    return client.chat.completions.create(...)
+
+@task(name="write_draft")
+def write_draft(outline: str):
+    return client.chat.completions.create(...)
+\`\`\`
+
+### TypeScript
+
+\`\`\`typescript
+import { Respan, withWorkflow, withTask } from "@respan/respan";
+
+const respan = new Respan();
+await respan.initialize();
+
+const result = await withWorkflow({ name: "write_article" }, async () => {
+  const outline = await withTask({ name: "generate_outline" }, async () => {
+    return await client.chat.completions.create({...});
+  });
+  return await withTask({ name: "write_draft" }, async () => {
+    return await client.chat.completions.create({...});
+  });
+});
+\`\`\`
+
+### Decorator Parameters
+
+\`\`\`python
+@workflow(
+    name="my_workflow",               # Display name (default: function name)
+    version=1,                        # Version number
+    processors="my_processor",        # Route to specific processor(s)
+    sample_rate=0.5,                  # 0.0-1.0, fraction of spans exported
+    export_filter={                   # Conditional export
+        "metadata.env": {"operator": "", "value": "production"}
+    },
+)
+\`\`\`
+
+All parameters are optional. Without any, the function name is used as the span name.
+
+### Async Support
+
+Decorators work with \`async def\`, generators, and async generators automatically.
+
+---
+
+## Context Propagation
+
+Attach attributes to **all spans** within a scope, including auto-instrumented LLM calls.
+
+### propagate_attributes
+
+\`\`\`python
+from respan import Respan, propagate_attributes
+
+Respan()
+
+with propagate_attributes(
+    customer_identifier="user_123",
+    thread_identifier="conv_abc",
+    environment="production",
+    metadata={"plan": "pro", "team": "search"},
+):
+    # All spans here (including auto-instrumented OpenAI calls) get these attributes
+    result = run_pipeline()
+\`\`\`
+
+#### Available attributes
+
+| Attribute | Description |
+|-----------|-------------|
+| \`customer_identifier\` | User/customer ID — enables per-user analytics |
+| \`customer_email\` | Customer email |
+| \`customer_name\` | Customer display name |
+| \`thread_identifier\` | Conversation thread ID — groups related calls |
+| \`custom_identifier\` | Indexed custom identifier for fast queries |
+| \`group_identifier\` | Group related traces together |
+| \`environment\` | Environment name (e.g. \`"production"\`, \`"staging"\`) |
+| \`metadata\` | Dict of custom key-value pairs (merged in nested contexts) |
+| \`prompt\` | Dict with \`prompt_id\` and \`variables\` for prompt logging |
+
+#### TypeScript
+
+\`\`\`typescript
+const result = await respan.propagateAttributes(
+  {
+    customerIdentifier: "user_123",
+    threadIdentifier: "conv_abc",
+    metadata: { plan: "pro" },
+  },
+  async () => {
+    return await runPipeline();
+  }
+);
+\`\`\`
+
+#### Nesting behavior
+
+- Nested \`propagate_attributes\` calls merge with the outer context
+- \`metadata\` dicts are merged (not replaced)
+- For duplicate keys, inner values override outer
+- Async-safe via \`contextvars\`
+
+### respan_span_attributes
+
+Attaches attributes to the **current active span only** (not auto-instrumented child spans):
+
+\`\`\`python
+from respan import respan_span_attributes
+
+with respan_span_attributes({
+    "customer_identifier": "user-123",
+    "metadata": {"priority": "high"},
+}):
+    pass
+\`\`\`
+
+Use \`propagate_attributes\` when you need attributes on all nested spans (most common). Use \`respan_span_attributes\` when you only want to tag the current span.
+
+---
+
+## Imperative Span Creation
+
+For runtime-determined span names, use the client API instead of decorators:
+
+\`\`\`python
+from respan import get_client
+
+client = get_client()
+
+with client.start_span(name="dynamic_step", kind="task") as span:
+    result = do_work()
+    # span is automatically closed and exported
+\`\`\`
+
+---
+
+## Updating the Current Span
+
+Add attributes or change status on the active span at runtime:
+
+\`\`\`python
+from respan import get_client
+
+client = get_client()
+
+# Inside a decorated function or start_span context:
+client.update_current_span(
+    respan_params={
+        "customer_identifier": "user_123",
+        "metadata": {"step": "final"},
+    },
+    name="renamed_span",
+)
+
+# Add events
+client.add_event("checkpoint_reached", attributes={"items": 42})
+
+# Record exceptions (span continues)
+try:
+    risky_operation()
+except Exception as e:
+    client.record_exception(e)
+\`\`\`
+
+---
+
+## Custom Processors
+
+Route spans to multiple destinations with filtering:
+
+\`\`\`python
+from respan_tracing import RespanTelemetry
+
+telemetry = RespanTelemetry(api_key="...", base_url="...")
+
+# Add a second exporter for debug logging
+from opentelemetry.sdk.trace.export import ConsoleSpanExporter
+
+telemetry.add_processor(
+    exporter=ConsoleSpanExporter(),
+    name="debug",
+    filter_fn=lambda span: span.attributes.get("environment") == "debug",
+)
+\`\`\`
+
+Route specific spans to a processor using the \`processors\` parameter on decorators:
+
+\`\`\`python
+@task(name="sensitive_step", processors="debug")
+def sensitive_step():
+    ...
+\`\`\`
+
+---
+
+## Flush
+
+Always call \`flush()\` before process exit to ensure all spans are exported:
+
+\`\`\`python
+respan = Respan()
+# ... your code ...
+respan.flush()
+\`\`\`
+
+In serverless/Lambda: call \`flush()\` at the end of every handler invocation.
+
+---
+
+## Trace Hierarchy Example
+
+\`\`\`
+workflow: handle_request
+  +-- task: classify_intent
+  |     +-- llm: openai.chat (auto-captured)
+  +-- agent: support_agent
+  |     +-- tool: lookup_order
+  |     +-- llm: openai.chat (auto-captured)
+  |     +-- tool: process_refund
+  +-- task: generate_response
+        +-- llm: openai.chat (auto-captured)
+\`\`\`
+
+LLM calls within decorated functions are automatically nested as child spans.
+`;
+export const GATEWAY_MD = `# Gateway Reference
+
+Route LLM calls through the Respan proxy for logging, caching, key management, fallbacks, and model switching.
+
+Docs: \`https://respan.ai/docs/documentation/features/gateway/gateway-quickstart.md\`
+
+---
+
+## Setup
+
+Point the LLM client's \`base_url\` at the Respan gateway. The Respan API key authenticates both the proxy and tracing.
+
+### Python (OpenAI SDK)
+
+\`\`\`python
+import os
+from openai import OpenAI
+
+client = OpenAI(
+    api_key=os.environ["RESPAN_API_KEY"],
+    base_url=os.getenv("RESPAN_BASE_URL", "https://api.respan.ai/api"),
+)
+
+response = client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[{"role": "user", "content": "Hello!"}],
+)
+\`\`\`
+
+### TypeScript (OpenAI SDK)
+
+\`\`\`typescript
+import OpenAI from "openai";
+
+const client = new OpenAI({
+  apiKey: process.env.RESPAN_API_KEY,
+  baseURL: process.env.RESPAN_BASE_URL || "https://api.respan.ai/api",
+});
+
+const response = await client.chat.completions.create({
+  model: "gpt-4o-mini",
+  messages: [{ role: "user", content: "Hello!" }],
+});
+\`\`\`
+
+### Python (Anthropic SDK)
+
+\`\`\`python
+import os
+import anthropic
+
+client = anthropic.Anthropic(
+    api_key=os.environ["RESPAN_API_KEY"],
+    base_url=os.getenv("RESPAN_BASE_URL", "https://api.respan.ai/api"),
+)
+\`\`\`
+
+### Environment variables
+
+Set once, works everywhere:
+
+\`\`\`bash
+export RESPAN_API_KEY="your-key"
+export OPENAI_API_KEY="$RESPAN_API_KEY"
+export OPENAI_BASE_URL="https://api.respan.ai/api"
+\`\`\`
+
+---
+
+## Gateway + Tracing (Both)
+
+Combine SDK tracing and gateway routing:
+
+### Python
+
+\`\`\`python
+import os
+from openai import OpenAI
+from respan import Respan
+
+respan_api_key = os.environ["RESPAN_API_KEY"]
+respan_base_url = os.getenv("RESPAN_BASE_URL", "https://api.respan.ai/api")
+
+respan = Respan(api_key=respan_api_key, base_url=respan_base_url)
+
+client = OpenAI(
+    api_key=respan_api_key,
+    base_url=respan_base_url,
+)
+\`\`\`
+
+### TypeScript
+
+\`\`\`typescript
+import OpenAI from "openai";
+import { Respan } from "@respan/respan";
+
+const respanApiKey = process.env.RESPAN_API_KEY!;
+const respanBaseUrl = process.env.RESPAN_BASE_URL || "https://api.respan.ai/api";
+
+const respan = new Respan({ apiKey: respanApiKey, baseUrl: respanBaseUrl });
+await respan.initialize();
+
+const client = new OpenAI({
+  apiKey: respanApiKey,
+  baseURL: respanBaseUrl,
+});
+\`\`\`
+
+---
+
+## Framework-Specific Gateway Setup
+
+### CrewAI
+
+\`\`\`python
+import os
+os.environ["OPENAI_API_KEY"] = os.environ["RESPAN_API_KEY"]
+os.environ["OPENAI_BASE_URL"] = os.getenv("RESPAN_BASE_URL", "https://api.respan.ai/api")
+
+from crewai import Agent, Task, Crew
+# CrewAI uses the env vars automatically
+\`\`\`
+
+### Haystack
+
+\`\`\`python
+import os
+os.environ["OPENAI_API_KEY"] = os.environ["RESPAN_API_KEY"]
+os.environ["OPENAI_BASE_URL"] = os.getenv("RESPAN_BASE_URL", "https://api.respan.ai/api")
+
+from haystack.components.generators import OpenAIGenerator
+# OpenAIGenerator picks up env vars automatically
+\`\`\`
+
+### OpenAI Agents SDK
+
+\`\`\`python
+import os
+os.environ["OPENAI_API_KEY"] = os.environ["RESPAN_API_KEY"]
+os.environ["OPENAI_BASE_URL"] = os.getenv("RESPAN_BASE_URL", "https://api.respan.ai/api")
+\`\`\`
+
+### Pydantic AI
+
+\`\`\`python
+import os
+os.environ["OPENAI_API_KEY"] = os.environ["RESPAN_API_KEY"]
+os.environ["OPENAI_BASE_URL"] = os.getenv("RESPAN_BASE_URL", "https://api.respan.ai/api")
+
+from pydantic_ai import Agent
+agent = Agent(model="openai:gpt-4o-mini")
+\`\`\`
+
+---
+
+## Model Switching
+
+The gateway supports 250+ models across providers. Use any model name:
+
+\`\`\`python
+# OpenAI models
+response = client.chat.completions.create(model="gpt-4o", ...)
+
+# Anthropic models (via OpenAI SDK!)
+response = client.chat.completions.create(model="claude-sonnet-4-20250514", ...)
+
+# Google models
+response = client.chat.completions.create(model="gemini-2.0-flash", ...)
+
+# Open-source models
+response = client.chat.completions.create(model="llama-3.1-70b", ...)
+\`\`\`
+
+No code changes needed — just change the model string. View all models at \`https://platform.respan.ai/platform/models\`.
+
+---
+
+## Advanced Features
+
+### Fallback Models
+
+Automatic failover if the primary model fails:
+
+\`\`\`python
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[...],
+    extra_body={
+        "fallback_models": ["claude-sonnet-4-20250514", "gemini-2.0-flash"],
+    },
+)
+\`\`\`
+
+### Retries
+
+\`\`\`python
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[...],
+    extra_body={
+        "retries": 3,
+    },
+)
+\`\`\`
+
+### Caching
+
+Enable response caching to save cost on repeated calls:
+
+\`\`\`python
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[...],
+    extra_body={
+        "cache_enabled": True,
+        "cache_ttl": 3600,  # seconds
+    },
+)
+\`\`\`
+
+### Streaming
+
+Streaming works normally through the gateway:
+
+\`\`\`python
+stream = client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[{"role": "user", "content": "Tell me a story"}],
+    stream=True,
+)
+for chunk in stream:
+    print(chunk.choices[0].delta.content, end="")
+\`\`\`
+
+### Disable Logging
+
+For sensitive data, disable logging on specific requests:
+
+\`\`\`python
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[...],
+    extra_body={
+        "disable_log": True,
+    },
+)
+\`\`\`
+
+---
+
+## Key Benefits
+
+- **Unified key management** — one Respan API key for all providers
+- **Automatic logging** — every request logged as a span
+- **Cost tracking** — per-request cost tracking across providers
+- **Caching** — cache repeated LLM calls to reduce cost and latency
+- **Fallbacks** — automatic failover between providers
+- **Model switching** — swap models without code changes
+- **~50-150ms added latency** — minimal overhead
+`;
+export const PROMPTS_MD = `# Prompts Reference
+
+Create, version, deploy, and use prompt templates from the Respan platform.
+
+Docs: \`https://respan.ai/docs/documentation/features/prompt-management/prompt-management-quickstart.md\`
+
+---
+
+## Overview
+
+Prompts are version-controlled templates stored on the Respan platform. The workflow is:
+
+1. **Create** a prompt with a name
+2. **Add versions** with messages, model config, and \`{{variables}}\`
+3. **Deploy** the version you want live
+4. **Fetch** the prompt in code using the API or MCP tools
+5. **Iterate** — commit new versions and deploy without code changes
+
+---
+
+## Create a Prompt
+
+### Via MCP
+
+\`\`\`
+Use create_prompt to create a prompt:
+  name: "classify_intent"
+  description: "Classifies user messages into intent categories"
+
+Then use create_prompt_version to add content:
+  prompt_id: "<id from above>"
+  model: "gpt-4o-mini"
+  messages:
+    - role: "system"
+      content: "You are an intent classifier. Classify the user message into one of: question, complaint, request, feedback."
+    - role: "user"
+      content: "{{user_message}}"
+\`\`\`
+
+### Via CLI
+
+\`\`\`bash
+respan prompts create --name "classify_intent"
+respan prompts versions <prompt-id>
+\`\`\`
+
+---
+
+## Variables
+
+Use \`{{variable_name}}\` in message content. Variables are filled at runtime.
+
+\`\`\`json
+{
+  "messages": [
+    {"role": "system", "content": "You are a {{role}} assistant."},
+    {"role": "user", "content": "{{user_input}}"}
+  ]
+}
+\`\`\`
+
+### Jinja templates
+
+Prompts support Jinja2 syntax:
+
+- **Conditionals:** \`{% if context %}Use this context: {{context}}{% endif %}\`
+- **JSON access:** \`{{ input.key }}\`
+- **Filters:** \`{{ var | upper }}\`
+- **Loops:** \`{% for item in items %}{{item}}{% endfor %}\`
+- **Default values:** Set via the API or platform UI
+
+---
+
+## Versioning and Deployment
+
+Each prompt can have multiple versions. Only one version is **deployed** (live) at a time.
+
+### Workflow
+
+1. \`create_prompt_version\` — creates a new version (always starts as NOT deployed)
+2. Test the version in the playground or via API
+3. Deploy via platform UI when ready
+4. Code using \`prompt_id\` automatically gets the deployed version
+
+### Version pinning
+
+\`\`\`python
+# Always use the deployed version (default)
+prompt = get_prompt(prompt_id="abc123")
+
+# Pin to a specific version
+prompt = get_prompt(prompt_id="abc123", version=3)
+\`\`\`
+
+---
+
+## Using Prompts in Code
+
+### Python (via Gateway)
+
+Fetch the prompt config and pass to the LLM:
+
+\`\`\`python
+import os
+from openai import OpenAI
+
+client = OpenAI(
+    api_key=os.environ["RESPAN_API_KEY"],
+    base_url=os.getenv("RESPAN_BASE_URL", "https://api.respan.ai/api"),
+)
+
+# The gateway resolves the prompt template and fills variables
+response = client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[],  # Messages come from the prompt template
+    extra_body={
+        "prompt_id": "your-prompt-id",
+        "prompt_variables": {
+            "user_message": "I want to return my order",
+        },
+    },
+)
+\`\`\`
+
+### Linking traces to prompts
+
+Use \`propagate_attributes\` to tag traces with the prompt used:
+
+\`\`\`python
+from respan import propagate_attributes
+
+with propagate_attributes(
+    prompt={"prompt_id": "abc123", "variables": {"user_message": "Hello"}},
+):
+    response = client.chat.completions.create(...)
+\`\`\`
+
+---
+
+## Prompt Composition
+
+Variables can reference other prompts (child prompts rendered first, injected as text).
+
+Configure in the platform UI by changing a variable's type from **Text** to **Prompt**.
+
+Max nesting depth: 2 (parent -> child).
+
+---
+
+## Structured Output (JSON Schema)
+
+Attach a JSON schema to a prompt version for structured responses:
+
+\`\`\`json
+{
+  "response_format": {
+    "type": "json_schema",
+    "json_schema": {
+      "name": "extraction",
+      "strict": true,
+      "schema": {
+        "type": "object",
+        "properties": {
+          "intent": {"type": "string"},
+          "confidence": {"type": "number"}
+        },
+        "required": ["intent", "confidence"],
+        "additionalProperties": false
+      }
+    }
+  }
+}
+\`\`\`
+
+---
+
+## MCP Tools Reference
+
+| Action | Tool |
+|--------|------|
+| List all prompts | \`list_prompts\` |
+| Get prompt details | \`get_prompt_detail(prompt_id)\` |
+| Create a prompt | \`create_prompt(name, description)\` |
+| Update name/description | \`update_prompt(prompt_id, name, description)\` |
+| Create a version | \`create_prompt_version(prompt_id, messages, model, ...)\` |
+| Update a version | \`update_prompt_version(prompt_id, version, ...)\` |
+| List versions | \`list_prompt_versions(prompt_id)\` |
+| Get version detail | \`get_prompt_version_detail(prompt_id, version)\` |
+
+---
+
+## CLI Commands
+
+\`\`\`bash
+respan prompts list                    # List all prompts
+respan prompts get <id>                # Get prompt details
+respan prompts create --name <name>    # Create a prompt
+respan prompts update <id>             # Update a prompt
+respan prompts versions <id>           # List prompt versions
+\`\`\`
+`;
+export const EVALS_MD = `# Evals Reference
+
+Build evaluation pipelines: datasets, evaluators, and experiments.
+
+Docs: \`https://respan.ai/docs/documentation/features/evals/concepts.md\`
+
+---
+
+## Overview
+
+Two evaluation modes:
+
+- **Offline** — build datasets, run experiments across prompt/model variants, compare scores
+- **Online** — automatically score live production traffic (see [monitors.md](monitors.md))
+
+Offline eval workflow:
+
+\`\`\`
+Create dataset -> Create evaluator -> Create experiment -> Run & compare
+\`\`\`
+
+---
+
+## Datasets
+
+A dataset is a collection of test rows (input/output pairs) for experiments.
+
+### Create from scratch
+
+\`\`\`
+Use create_dataset:
+  name: "intent_classification_testset"
+  description: "Test cases for intent classifier"
+
+Then add rows with create_dataset_span:
+  dataset_id: "<id>"
+  input: '{"user_message": "I want a refund"}'
+  output: '{"intent": "refund_request"}'
+\`\`\`
+
+### Create from production traces
+
+Sample real spans from traced data:
+
+\`\`\`
+1. Use list_traces or list_logs to find relevant spans
+2. Use add_spans_to_dataset to add span IDs to a dataset:
+     dataset_id: "<id>"
+     span_ids: ["span_1", "span_2", "span_3"]
+\`\`\`
+
+Or use the platform UI: Datasets -> Insert by sampling (filter, set sampling %, preview, confirm).
+
+### CSV import
+
+Upload CSV via the platform UI. Columns map to prompt variables. Optional \`ideal_output\` column for expected outputs. Max 500 rows per import.
+
+### MCP tools
+
+| Action | Tool |
+|--------|------|
+| Create dataset | \`create_dataset(name, description)\` |
+| List datasets | \`list_datasets\` |
+| Get dataset | \`get_dataset(dataset_id)\` |
+| Update dataset | \`update_dataset(dataset_id, name, description)\` |
+| Add a row manually | \`create_dataset_span(dataset_id, input, output)\` |
+| Add existing spans | \`add_spans_to_dataset(dataset_id, span_ids)\` |
+| List rows | \`list_dataset_spans(dataset_id)\` |
+| Get a row | \`retrieve_dataset_span(dataset_id, log_id)\` |
+| Update a row | \`update_dataset_span(dataset_id, log_id, ...)\` |
+
+---
+
+## Evaluators
+
+An evaluator scores LLM outputs. It is a workflow built from **graders** connected with conditions.
+
+### Grader types
+
+| Type | How it works |
+|------|-------------|
+| **LLM grader** | A language model judges the output. Uses variables: \`{{output}}\`, \`{{input}}\`, \`{{expected_output}}\`, \`{{metadata}}\`, \`{{metrics}}\` |
+| **Code grader** | A Python function returns a score. Signature: \`main(eval_inputs)\` where \`eval_inputs["output"]\` is required |
+| **Human grader** | Team members review and score outputs manually |
+
+### Score value types
+
+| Type | Use case |
+|------|----------|
+| \`numerical\` | Rating scales (e.g. 0-5) |
+| \`boolean\` | Pass/fail checks |
+| \`categorical\` | Multi-choice classifications |
+| \`comment\` | Qualitative text feedback |
+
+### Create an evaluator
+
+\`\`\`
+Use create_evaluator:
+  name: "response_quality"
+  score_value_type: "numerical"
+  type: "llm"
+  description: "Scores response quality 1-5"
+  evaluator_slug: "response-quality"
+\`\`\`
+
+For complex evaluators with grader workflows, conditions, and compute blocks — use the platform UI (Evaluators -> Create).
+
+### Evaluator workflow blocks
+
+- **Markers:** \`Original input\`, \`Final result\` (entry/exit)
+- **Graders:** LLM, Code, or Human scoring
+- **Conditions:** \`If...Then...Else\` branching based on scores
+- **Compute:** Average/weighted average of multiple grader scores
+- **Metrics:** Built-in: completion tokens, cost, latency, model, prompt tokens, total tokens
+- **Constants:** Fixed threshold values
+
+### MCP tools
+
+| Action | Tool |
+|--------|------|
+| List evaluators | \`list_evaluators\` |
+| Get evaluator | \`get_evaluator(evaluator_id)\` |
+| Create evaluator | \`create_evaluator(name, score_value_type, ...)\` |
+| Update evaluator | \`update_evaluator(evaluator_id, ...)\` |
+| Run evaluator | \`run_evaluator(evaluator_id, dataset_id or log_ids)\` |
+
+---
+
+## Experiments
+
+An experiment runs evaluators across a dataset to compare prompt/model configurations.
+
+### Workflow
+
+1. Create a dataset with test rows
+2. Create evaluator(s) to score outputs
+3. Create an experiment linking dataset + evaluators
+4. Run the experiment — it generates outputs for each row and scores them
+5. Compare results across configurations
+
+### Create an experiment
+
+\`\`\`
+Use create_experiment:
+  name: "v2_vs_v3_comparison"
+  dataset_id: "<dataset-id>"
+  evaluator_slugs: ["response-quality", "hallucination-check"]
+  workflows:
+    - type: "prompt"
+      config: { prompt_id: "<prompt-v2-id>" }
+    - type: "prompt"
+      config: { prompt_id: "<prompt-v3-id>" }
+\`\`\`
+
+### Workflow types
+
+| Type | Description |
+|------|-------------|
+| \`prompt\` | Run a saved prompt template against dataset rows |
+| \`completion\` | Run a model completion with config |
+| \`custom\` | User-defined logic |
+
+### MCP tools
+
+| Action | Tool |
+|--------|------|
+| List experiments | \`list_experiments\` |
+| Get experiment | \`get_experiment(experiment_id)\` |
+| Create experiment | \`create_experiment(name, dataset_id, ...)\` |
+| List experiment spans | \`list_experiment_spans(experiment_id)\` |
+| Get experiment span | \`get_experiment_span(experiment_id, log_id)\` |
+| Update experiment span | \`update_experiment_span(experiment_id, log_id, ...)\` |
+| Get summary stats | \`get_experiment_spans_summary(experiment_id, start_time, end_time)\` |
+
+---
+
+## End-to-End Example
+
+\`\`\`
+Step 1: Create dataset
+  create_dataset(name="qa_testset")
+  create_dataset_span(dataset_id="...", input="What is Python?", output="A programming language")
+  create_dataset_span(dataset_id="...", input="Who made Rust?", output="Mozilla Research")
+
+Step 2: Create evaluator
+  create_evaluator(
+    name="accuracy_check",
+    score_value_type="boolean",
+    type="llm",
+    evaluator_slug="accuracy-check"
+  )
+
+Step 3: Create experiment
+  create_experiment(
+    name="gpt4o_vs_claude",
+    dataset_id="...",
+    evaluator_slugs=["accuracy-check"]
+  )
+
+Step 4: Review results
+  list_experiment_spans(experiment_id="...")
+  get_experiment_spans_summary(experiment_id="...", start_time="...", end_time="...")
+\`\`\`
+
+---
+
+## CLI Commands
+
+\`\`\`bash
+respan datasets list                   # List datasets
+respan datasets create --name <name>   # Create a dataset
+respan datasets get <id>               # Get dataset details
+respan datasets spans <id>             # List dataset spans
+respan evaluators list                 # List evaluators
+respan evaluators get <id>             # Get evaluator details
+respan evaluators create               # Create an evaluator
+respan evaluators run <id>             # Run an evaluator
+respan experiments list                # List experiments
+respan experiments get <id>            # Get experiment details
+respan experiments create              # Create an experiment
+respan eval <file>                     # Run a one-shot eval defined in a JSON file
+\`\`\`
+
+## One-Shot Evals via JSON
+
+\`respan eval <file.json>\` runs a complete eval pipeline from a single JSON spec —
+creates the prompt, dataset, evaluators, and experiment in one command.
+IDs are written back into the file so reruns reuse existing resources.
+
+### Minimal example
+
+\`\`\`json
+{
+  "name": "Movie matcher",
+  "prompt": {
+    "name": "movie-matcher",
+    "model": "gpt-5-mini",
+    "messages": [
+      { "role": "system", "content": "Identify the movie. Reply with only the title." },
+      { "role": "user", "content": "{{description}}" }
+    ]
+  },
+  "dataset": {
+    "name": "movie-descriptions",
+    "rows": [
+      { "description": "A detective investigates seven deadly sins.", "expected_output": "Se7en" },
+      { "description": "A hacker learns reality is a simulation.", "expected_output": "The Matrix" }
+    ]
+  },
+  "experiment": {
+    "evaluators": [
+      {
+        "name": "Title match",
+        "type": "llm",
+        "score_value_type": "boolean",
+        "llm_config": {
+          "model": "openai/gpt-5-mini",
+          "evaluator_definition": "Expected: {{expected_output}}. Got: {{output}}. Correct? Reply true or false."
+        }
+      }
+    ]
+  }
+}
+\`\`\`
+
+Run:
+
+\`\`\`bash
+respan eval movie-matcher.eval.json
+\`\`\`
+
+The runner creates the prompt (and deploys version 1), creates the dataset (and bulk-loads rows),
+creates each evaluator + its workflow (and deploys), then creates the experiment linking them.
+Each step writes the resulting IDs back to the JSON file so subsequent runs are idempotent —
+re-running with the same file reuses the existing resources.
+`;
+export const MONITORS_MD = `# Monitors & Automation Reference
+
+Set up monitoring alerts, online evaluations, and automated workflows.
+
+Docs: \`https://respan.ai/docs/documentation/features/monitoring/monitors.md\`
+
+---
+
+## Monitors
+
+A monitor watches a metric over time and sends a notification when a threshold is crossed.
+
+### Examples
+
+- Error rate > 5% over 10 minutes
+- Total cost > $1.00 over 5 minutes
+- Average latency > 3 seconds over 15 minutes
+- P95 latency > 5 seconds over 1 hour
+
+### Setup
+
+Monitors are configured in the platform UI: **Monitoring -> Monitors -> Create**.
+
+A monitor has two parts:
+
+#### 1. Trigger
+
+Template: \`When [metric] of [source] is [threshold] over [time window]\`
+
+- **Metric:** cost, latency, error rate, token usage, request count, etc.
+- **Source:** all requests, specific model, specific customer, etc.
+- **Threshold:** condition (greater than, less than, equals)
+- **Time window:** evaluation period (1 min, 5 min, 10 min, 1 hour, etc.)
+
+Add **Where** conditions to narrow scope:
+- Filter by model, project, environment, customer, or custom metadata
+
+#### 2. Notifications
+
+Destinations:
+
+| Destination | Details |
+|-------------|---------|
+| **Email** | Customizable subject and body |
+| **Slack** | Customizable message |
+| **Webhook** | Raw URL — receives JSON payload |
+
+A single monitor can send to multiple destinations.
+
+### Manage
+
+- **Deploy** to make a monitor live
+- **Pause/Resume** deployed monitors
+- **Send test alert** to validate before deploying
+- **View versions** and restore earlier versions as drafts
+- **Create from chart** — select a metric on the dashboard, click to create a monitor with metric/source/interval prefilled
+
+---
+
+## Online Evaluations (Automation)
+
+Automatically score production traffic in real time using deployed evaluators.
+
+### Setup
+
+Configure in the platform UI: **Evaluators -> Automations -> Create**.
+
+1. **Create a condition** — filter rules for which logs to evaluate:
+   - **Single log** — evaluate individual spans matching criteria
+   - **Aggregation** — evaluate aggregated metrics over a window
+
+2. **Create an automation** — connect:
+   - Condition (which logs)
+   - Evaluator (which scoring workflow)
+   - Sampling rate (0.0-1.0, fraction of matching logs to evaluate)
+
+3. **Check results** — view eval scores in the automation table
+
+### Use case
+
+> "Score 10% of production chat responses for hallucination using my LLM evaluator"
+
+\`\`\`
+Condition: log_type = "chat" AND model = "gpt-4o"
+Evaluator: hallucination-check
+Sampling: 0.1
+\`\`\`
+
+---
+
+## Webhooks
+
+Receive event notifications via HTTP webhooks.
+
+### Setup
+
+1. Go to platform: **Settings -> Webhooks -> Create**
+2. Enter your endpoint URL
+3. Select event types to subscribe to
+4. Optionally set a webhook secret for HMAC-SHA256 verification
+
+### Verification
+
+Respan signs webhook payloads with HMAC-SHA256. Verify the signature:
+
+\`\`\`python
+import hashlib
+import hmac
+
+def verify_webhook(payload: bytes, signature: str, secret: str) -> bool:
+    expected = hmac.new(
+        secret.encode(),
+        payload,
+        hashlib.sha256,
+    ).hexdigest()
+    return hmac.compare_digest(expected, signature)
+\`\`\`
+
+The signature is sent in the request headers.
+
+---
+
+## Metrics Dashboard
+
+Track key indicators in the platform: **Dashboard -> Metrics**.
+
+### Available metrics
+
+- **Requests:** total count, requests per model/customer/API key
+- **Tokens:** prompt tokens, completion tokens, total tokens
+- **Cost:** per-request, per-model, per-customer
+- **Latency:** average, P50, P95, P99
+- **Errors:** error count, error rate
+- **Time to first token (TTFT):** streaming latency
+
+### Breakdowns
+
+Slice metrics by:
+- Model
+- Customer (\`customer_identifier\`)
+- API key
+- Prompt template
+- Environment
+- Custom metadata keys
+
+### Per-user analytics
+
+Use \`customer_identifier\` in spans to enable:
+- Per-user cost tracking and budgets
+- Per-user rate limiting
+- User-level quality metrics
+
+---
+
+## CLI Commands
+
+\`\`\`bash
+respan traces list --limit 10          # List recent traces
+respan traces get <trace-id>           # Get trace details
+respan traces summary                  # Trace summary stats
+respan logs list --limit 10            # List recent logs
+respan logs get <log-id>               # Get log details
+respan logs summary                    # Log summary stats
+\`\`\`
+`;