@respan/cli 0.8.0 → 0.10.0

package/dist/lib/eval/runner.js

@@ -0,0 +1,292 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+import { parseEvalFile } from './schema.js';
+export async function runEval(filePath, client, authHeader, baseUrl, log = () => { }) {
+    const raw = readFileSync(filePath, 'utf8');
+    const evalFile = parseEvalFile(raw);
+    const promptResult = await ensurePrompt(evalFile, client, authHeader, log);
+    persist(filePath, evalFile);
+    const datasetResult = await ensureDataset(evalFile, client, authHeader, log);
+    persist(filePath, evalFile);
+    const evaluatorResult = await ensureEvaluatorWorkflows(evalFile, client, authHeader, baseUrl, log);
+    persist(filePath, evalFile);
+    const experimentResult = await ensureExperiment(evalFile, promptResult.id, datasetResult.id, evaluatorResult.workflow_ids, client, authHeader, log);
+    persist(filePath, evalFile);
+    return {
+        prompt_id: promptResult.id,
+        prompt_version_id: promptResult.version_id,
+        prompt_version_deployed: promptResult.deployed_version,
+        dataset_id: datasetResult.id,
+        evaluator_workflow_ids: evaluatorResult.workflow_ids,
+        experiment_id: experimentResult.id,
+        created_prompt: promptResult.created,
+        created_dataset: datasetResult.created,
+        created_experiment: experimentResult.created,
+        evaluators_created: evaluatorResult.created_count,
+        rows_added: datasetResult.rows_added,
+    };
+}
+async function ensurePrompt(evalFile, client, Authorization, log) {
+    const { prompt } = evalFile;
+    if (prompt.id) {
+        log(`prompt: reusing ${prompt.id}`);
+        return { id: prompt.id, version_id: prompt.version_id, created: false };
+    }
+    log(`prompt: creating "${prompt.name}"`);
+    const created = (await client.prompts.createPrompt({
+        Authorization,
+        name: prompt.name,
+        ...(prompt.description ? { description: prompt.description } : {}),
+    }));
+    const promptId = String(created.id ?? created.prompt_id ?? '');
+    if (!promptId)
+        throw new Error('createPrompt: response missing id');
+    prompt.id = promptId;
+    log(`prompt: creating version on ${promptId}`);
+    const version = (await client.prompts.createPromptVersion({
+        Authorization,
+        prompt_id: promptId,
+        messages: prompt.messages,
+        model: prompt.model,
+        ...(prompt.temperature !== undefined ? { temperature: prompt.temperature } : {}),
+        ...(prompt.max_tokens !== undefined ? { max_tokens: prompt.max_tokens } : {}),
+        ...(prompt.top_p !== undefined ? { top_p: prompt.top_p } : {}),
+    }));
+    const versionId = String(version.id ?? version.version_id ?? '');
+    if (versionId)
+        prompt.version_id = versionId;
+    const versionNumber = typeof version.version === 'number'
+        ? version.version
+        : Number.parseInt(String(version.version ?? '1'), 10);
+    log(`prompt: committing draft version ${versionNumber}`);
+    await client.prompts.commitPromptVersion({ Authorization, prompt_id: promptId });
+    log(`prompt: deploying version ${versionNumber}`);
+    await client.prompts.deployPromptVersion({ Authorization, prompt_id: promptId, version: versionNumber });
+    return {
+        id: promptId,
+        version_id: versionId || undefined,
+        deployed_version: versionNumber,
+        created: true,
+    };
+}
+async function ensureDataset(evalFile, client, Authorization, log) {
+    const { dataset } = evalFile;
+    if (dataset.id) {
+        log(`dataset: reusing ${dataset.id}`);
+        return { id: dataset.id, created: false, rows_added: 0 };
+    }
+    log(`dataset: creating "${dataset.name}"`);
+    const created = (await client.datasets.createDataset({
+        Authorization,
+        name: dataset.name,
+        is_empty: true,
+        ...(dataset.description ? { description: dataset.description } : {}),
+    }));
+    const datasetId = String(created.id ?? created.dataset_id ?? '');
+    if (!datasetId)
+        throw new Error('createDataset: response missing id');
+    dataset.id = datasetId;
+    const logs = dataset.rows.map((row) => {
+        const { expected_output, ...variables } = row;
+        return {
+            input: variables,
+            ...(expected_output !== undefined ? { expected_output } : {}),
+        };
+    });
+    const result = (await client.datasets.bulkCreateDatasetLogs({
+        Authorization,
+        dataset_id: datasetId,
+        logs,
+    }));
+    const successCount = typeof result.success_count === 'number' ? result.success_count : 0;
+    const errorCount = typeof result.error_count === 'number' ? result.error_count : 0;
+    log(`dataset: added ${successCount}/${logs.length} row(s)` + (errorCount ? ` (${errorCount} errors)` : ''));
+    return { id: datasetId, created: true, rows_added: successCount };
+}
+async function ensureEvaluatorWorkflows(evalFile, client, Authorization, baseUrl, log) {
+    const experiment = evalFile.experiment ?? (evalFile.experiment = {});
+    const evaluators = experiment.evaluators ?? [];
+    const explicitWorkflowIds = experiment.evaluator_workflow_ids ?? [];
+    let createdCount = 0;
+    const workflowIds = [];
+    // Resolve user-supplied workflow IDs to their deployed-version row IDs.
+    // The experiment API expects version IDs (the `evaluator_workflow_ids` field
+    // name is misleading), so we look up each workflow's deployed version.
+    for (const wfId of explicitWorkflowIds) {
+        const versionRowId = await getDeployedWorkflowVersionId(Authorization, baseUrl, wfId);
+        if (!versionRowId) {
+            throw new Error(`workflow ${wfId} (from experiment.evaluator_workflow_ids) has no deployed version.`);
+        }
+        workflowIds.push(versionRowId);
+    }
+    for (const ev of evaluators) {
+        if (!ev.evaluator_id) {
+            log(`evaluator: creating "${ev.name}" (${ev.type})`);
+            const created = (await client.evaluators.createEvaluator({
+                Authorization,
+                name: ev.name,
+                type: ev.type,
+                score_value_type: ev.score_value_type,
+                ...(ev.description ? { description: ev.description } : {}),
+                ...(ev.score_config ? { score_config: ev.score_config } : {}),
+                ...(ev.passing_conditions ? { passing_conditions: ev.passing_conditions } : {}),
+                ...(ev.llm_config ? { llm_config: ev.llm_config } : {}),
+                ...(ev.code_config ? { code_config: ev.code_config } : {}),
+            }));
+            const evaluatorId = String(created.id ?? created.evaluator_id ?? '');
+            if (!evaluatorId)
+                throw new Error('createEvaluator: response missing id');
+            ev.evaluator_id = evaluatorId;
+            createdCount += 1;
+        }
+        else {
+            log(`evaluator: reusing ${ev.evaluator_id}`);
+        }
+        if (!ev.workflow_id) {
+            const workflowName = `${ev.name} workflow`;
+            log(`workflow: creating "${workflowName}" wrapping evaluator ${ev.evaluator_id}`);
+            const nodeIdSeed = ev.evaluator_id.slice(0, 8);
+            const blocklyNodeId = safeNodeId(nodeIdSeed);
+            const taskId = `blockly_hidden_eval_task_${blocklyNodeId}`;
+            const taskLabel = `blockly_hidden_eval_${blocklyNodeId}`;
+            const outputField = outputFieldForScoreType(ev.score_value_type);
+            const taskConfig = {
+                evaluator_id: ev.evaluator_id,
+                score_value_type: ev.score_value_type,
+                score_config: ev.score_config ?? {},
+                is_auto_persist_enabled: true,
+                _blockly_hidden_eval: true,
+                _blockly_is_result: true,
+                _blockly_node_id: blocklyNodeId,
+                ...(outputField ? { _blockly_output_field: outputField } : {}),
+                _blockly_evaluator_kind: ev.type,
+                ...(ev.llm_config ? { llm_config: ev.llm_config } : {}),
+                ...(ev.code_config ? { code_config: ev.code_config } : {}),
+            };
+            const createdWf = (await client.workflows.createWorkflow({
+                Authorization,
+                type: 'evaluators',
+                trigger_event_type: 'eval_only',
+                name: workflowName,
+                tasks: [{
+                        id: taskId,
+                        type: 'eval',
+                        generation_method: ev.type,
+                        label: taskLabel,
+                        config: taskConfig,
+                        next: null,
+                    }],
+            }));
+            const workflowId = String(createdWf.workflow_id ?? createdWf.id ?? '');
+            if (!workflowId)
+                throw new Error('createWorkflow: response missing workflow_id');
+            ev.workflow_id = workflowId;
+            log(`workflow: committing draft v1`);
+            await commitWorkflow(Authorization, baseUrl, workflowId);
+            log(`workflow: deploying v1`);
+            const deployed = (await client.workflows.deployWorkflow({
+                Authorization,
+                workflow_id: workflowId,
+                version: 1,
+            }));
+            const deployedVersion = typeof deployed.version === 'number' ? deployed.version : 1;
+            ev.workflow_version_deployed = deployedVersion;
+            const versionRowId = String(deployed.id ?? '');
+            if (versionRowId)
+                ev.workflow_version_id = versionRowId;
+        }
+        else {
+            log(`workflow: reusing ${ev.workflow_id}`);
+        }
+        if (!ev.workflow_version_id) {
+            log(`workflow: looking up deployed version row id for ${ev.workflow_id}`);
+            const versionRowId = await getDeployedWorkflowVersionId(Authorization, baseUrl, ev.workflow_id);
+            if (!versionRowId) {
+                throw new Error(`workflow ${ev.workflow_id} has no deployed version; rerun after committing/deploying.`);
+            }
+            ev.workflow_version_id = versionRowId;
+        }
+        workflowIds.push(ev.workflow_version_id);
+    }
+    return { workflow_ids: workflowIds, created_count: createdCount };
+}
+async function ensureExperiment(evalFile, promptId, datasetId, workflowIds, client, Authorization, log) {
+    const experiment = evalFile.experiment ?? (evalFile.experiment = {});
+    if (experiment.id) {
+        log(`experiment: reusing ${experiment.id}`);
+        return { id: experiment.id, created: false };
+    }
+    const name = experiment.name ?? evalFile.experiment_name ?? `${evalFile.name} run`;
+    const evaluatorSlugs = experiment.evaluator_slugs ?? [];
+    if (workflowIds.length === 0 && evaluatorSlugs.length === 0) {
+        throw new Error('experiment requires at least one evaluator. Add `experiment.evaluators[]` to your eval file ' +
+            '(or `experiment.evaluator_workflow_ids` / `experiment.evaluator_slugs` for existing ones).');
+    }
+    const evaluatorSummary = workflowIds.length > 0
+        ? `evaluator_workflow_ids: ${workflowIds.join(', ')}`
+        : `evaluator_slugs: ${evaluatorSlugs.join(', ')}`;
+    log(`experiment: creating "${name}" with ${evaluatorSummary}`);
+    const created = (await client.experiments.createExperiment({
+        Authorization,
+        name,
+        dataset_id: datasetId,
+        ...(experiment.description ? { description: experiment.description } : {}),
+        workflow: [{ type: 'prompt', config: { prompt_id: promptId } }],
+        ...(workflowIds.length > 0 ? { evaluator_workflow_ids: workflowIds } : {}),
+        ...(evaluatorSlugs.length > 0 ? { evaluator_slugs: evaluatorSlugs } : {}),
+    }));
+    const experimentId = String(created.id ?? created.experiment_id ?? '');
+    if (!experimentId)
+        throw new Error('createExperiment: response missing id');
+    experiment.id = experimentId;
+    if (!experiment.name)
+        experiment.name = name;
+    return { id: experimentId, created: true };
+}
+function safeNodeId(raw) {
+    const sanitized = raw.replace(/[^a-zA-Z0-9_]/g, '_').slice(0, 30);
+    return sanitized || 'node';
+}
+function outputFieldForScoreType(scoreValueType) {
+    switch (scoreValueType) {
+        case 'boolean':
+            return 'boolean_value';
+        case 'single_select':
+        case 'text':
+        case 'comment':
+            return 'string_value';
+        case 'multi_select':
+        case 'json':
+            return null;
+        default:
+            return 'primary_score';
+    }
+}
+async function getDeployedWorkflowVersionId(Authorization, baseUrl, workflowId) {
+    const url = `${normalizeBaseUrl(baseUrl)}/api/workflows/${encodeURIComponent(workflowId)}/versions/`;
+    const response = await fetch(url, { headers: { Authorization } });
+    if (!response.ok)
+        return null;
+    const body = (await response.json());
+    const results = body.results ?? [];
+    const deployed = results.find((v) => v.is_enabled === true && v.is_read_only === true)
+        ?? results.find((v) => v.is_read_only === true);
+    return deployed ? String(deployed.id ?? '') : null;
+}
+function normalizeBaseUrl(baseUrl) {
+    return baseUrl.replace(/\/+$/, '').replace(/\/api$/, '');
+}
+async function commitWorkflow(Authorization, baseUrl, workflowId) {
+    const url = `${normalizeBaseUrl(baseUrl)}/api/workflows/${encodeURIComponent(workflowId)}/commits/`;
+    const response = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', Authorization },
+        body: JSON.stringify({}),
+    });
+    if (!response.ok) {
+        const text = await response.text().catch(() => '');
+        throw new Error(`commitWorkflow ${response.status}: ${text || response.statusText}`);
+    }
+}
+function persist(filePath, evalFile) {
+    writeFileSync(filePath, JSON.stringify(evalFile, null, 2) + '\n', 'utf8');
+}

package/dist/lib/eval/schema.d.ts

@@ -0,0 +1,67 @@
+export interface EvalPromptMessage {
+    role: 'system' | 'user' | 'assistant';
+    content: string;
+}
+export interface EvalPromptBlock {
+    id?: string;
+    version_id?: string;
+    name: string;
+    description?: string;
+    model: string;
+    messages: EvalPromptMessage[];
+    temperature?: number;
+    max_tokens?: number;
+    top_p?: number;
+}
+export interface EvalDatasetBlock {
+    id?: string;
+    name: string;
+    description?: string;
+    rows: Record<string, unknown>[];
+}
+export interface EvalEvaluatorBlock {
+    evaluator_id?: string;
+    workflow_id?: string;
+    workflow_version_id?: string;
+    workflow_version_deployed?: number;
+    name: string;
+    type: 'llm' | 'code' | 'human';
+    score_value_type: 'numerical' | 'boolean' | 'percentage' | 'single_select' | 'multi_select' | 'json' | 'text' | 'categorical' | 'comment';
+    description?: string;
+    score_config?: {
+        min_score?: number;
+        max_score?: number;
+        choices?: {
+            name?: string;
+            value?: string;
+        }[];
+    };
+    passing_conditions?: Record<string, unknown>;
+    llm_config?: {
+        model: string;
+        evaluator_definition: string;
+        scoring_rubric?: string;
+        temperature?: number;
+        max_tokens?: number;
+        top_p?: number;
+    };
+    code_config?: {
+        eval_code_snippet: string;
+    };
+}
+export interface EvalExperimentBlock {
+    id?: string;
+    name?: string;
+    description?: string;
+    evaluators?: EvalEvaluatorBlock[];
+    evaluator_slugs?: string[];
+    evaluator_workflow_ids?: string[];
+}
+export interface EvalFile {
+    name: string;
+    experiment_name?: string;
+    prompt: EvalPromptBlock;
+    dataset: EvalDatasetBlock;
+    experiment?: EvalExperimentBlock;
+}
+export declare function parseEvalFile(raw: string): EvalFile;

package/dist/lib/eval/schema.js

@@ -0,0 +1,22 @@
+export function parseEvalFile(raw) {
+    const data = JSON.parse(raw);
+    if (!data.name)
+        throw new Error('eval file: `name` is required');
+    if (!data.prompt)
+        throw new Error('eval file: `prompt` block is required');
+    if (!data.prompt.name)
+        throw new Error('eval file: `prompt.name` is required');
+    if (!data.prompt.model)
+        throw new Error('eval file: `prompt.model` is required');
+    if (!Array.isArray(data.prompt.messages) || data.prompt.messages.length === 0) {
+        throw new Error('eval file: `prompt.messages` must be a non-empty array');
+    }
+    if (!data.dataset)
+        throw new Error('eval file: `dataset` block is required');
+    if (!data.dataset.name)
+        throw new Error('eval file: `dataset.name` is required');
+    if (!Array.isArray(data.dataset.rows)) {
+        throw new Error('eval file: `dataset.rows` must be an array');
+    }
+    return data;
+}

package/dist/lib/skill-refs.generated.d.ts

@@ -0,0 +1,5 @@
+export declare const TRACING_MD = "# Tracing Reference\n\nAdvanced tracing configuration: decorators, context propagation, processors, and span attributes.\n\nDocs: `https://respan.ai/docs/documentation/features/tracing/traces/tracing-sdk.md`\n\n---\n\n## Decorators\n\nWrap functions to create structured span hierarchies. All decorators share the same signature.\n\n### Types\n\n| Decorator | Purpose |\n|-----------|---------|\n| `@workflow` / `withWorkflow` | Root span \u2014 top-level pipeline |\n| `@task` / `withTask` | Step within a workflow |\n| `@agent` / `withAgent` | Agent loop span |\n| `@tool` / `withTool` | Tool/function call span |\n\n### Python\n\n```python\nfrom respan import Respan, workflow, task, agent, tool\n\nRespan()\n\n@workflow(name=\"write_article\")\ndef write_article(topic: str):\n    outline = generate_outline(topic)\n    return write_draft(outline)\n\n@task(name=\"generate_outline\")\ndef generate_outline(topic: str):\n    return client.chat.completions.create(...)\n\n@task(name=\"write_draft\")\ndef write_draft(outline: str):\n    return client.chat.completions.create(...)\n```\n\n### TypeScript\n\n```typescript\nimport { Respan, withWorkflow, withTask } from \"@respan/respan\";\n\nconst respan = new Respan();\nawait respan.initialize();\n\nconst result = await withWorkflow({ name: \"write_article\" }, async () => {\n  const outline = await withTask({ name: \"generate_outline\" }, async () => {\n    return await client.chat.completions.create({...});\n  });\n  return await withTask({ name: \"write_draft\" }, async () => {\n    return await client.chat.completions.create({...});\n  });\n});\n```\n\n### Decorator Parameters\n\n```python\n@workflow(\n    name=\"my_workflow\",               # Display name (default: function name)\n    version=1,                        # Version number\n    processors=\"my_processor\",        # Route to specific processor(s)\n    sample_rate=0.5,                  # 0.0-1.0, fraction of spans exported\n    export_filter={                   # Conditional export\n        \"metadata.env\": {\"operator\": \"\", \"value\": \"production\"}\n    },\n)\n```\n\nAll parameters are optional. Without any, the function name is used as the span name.\n\n### Async Support\n\nDecorators work with `async def`, generators, and async generators automatically.\n\n---\n\n## Context Propagation\n\nAttach attributes to **all spans** within a scope, including auto-instrumented LLM calls.\n\n### propagate_attributes\n\n```python\nfrom respan import Respan, propagate_attributes\n\nRespan()\n\nwith propagate_attributes(\n    customer_identifier=\"user_123\",\n    thread_identifier=\"conv_abc\",\n    environment=\"production\",\n    metadata={\"plan\": \"pro\", \"team\": \"search\"},\n):\n    # All spans here (including auto-instrumented OpenAI calls) get these attributes\n    result = run_pipeline()\n```\n\n#### Available attributes\n\n| Attribute | Description |\n|-----------|-------------|\n| `customer_identifier` | User/customer ID \u2014 enables per-user analytics |\n| `customer_email` | Customer email |\n| `customer_name` | Customer display name |\n| `thread_identifier` | Conversation thread ID \u2014 groups related calls |\n| `custom_identifier` | Indexed custom identifier for fast queries |\n| `group_identifier` | Group related traces together |\n| `environment` | Environment name (e.g. `\"production\"`, `\"staging\"`) |\n| `metadata` | Dict of custom key-value pairs (merged in nested contexts) |\n| `prompt` | Dict with `prompt_id` and `variables` for prompt logging |\n\n#### TypeScript\n\n```typescript\nconst result = await respan.propagateAttributes(\n  {\n    customerIdentifier: \"user_123\",\n    threadIdentifier: \"conv_abc\",\n    metadata: { plan: \"pro\" },\n  },\n  async () => {\n    return await runPipeline();\n  }\n);\n```\n\n#### Nesting behavior\n\n- Nested `propagate_attributes` calls merge with the outer context\n- `metadata` dicts are merged (not replaced)\n- For duplicate keys, inner values override outer\n- Async-safe via `contextvars`\n\n### respan_span_attributes\n\nAttaches attributes to the **current active span only** (not auto-instrumented child spans):\n\n```python\nfrom respan import respan_span_attributes\n\nwith respan_span_attributes({\n    \"customer_identifier\": \"user-123\",\n    \"metadata\": {\"priority\": \"high\"},\n}):\n    pass\n```\n\nUse `propagate_attributes` when you need attributes on all nested spans (most common). Use `respan_span_attributes` when you only want to tag the current span.\n\n---\n\n## Imperative Span Creation\n\nFor runtime-determined span names, use the client API instead of decorators:\n\n```python\nfrom respan import get_client\n\nclient = get_client()\n\nwith client.start_span(name=\"dynamic_step\", kind=\"task\") as span:\n    result = do_work()\n    # span is automatically closed and exported\n```\n\n---\n\n## Updating the Current Span\n\nAdd attributes or change status on the active span at runtime:\n\n```python\nfrom respan import get_client\n\nclient = get_client()\n\n# Inside a decorated function or start_span context:\nclient.update_current_span(\n    respan_params={\n        \"customer_identifier\": \"user_123\",\n        \"metadata\": {\"step\": \"final\"},\n    },\n    name=\"renamed_span\",\n)\n\n# Add events\nclient.add_event(\"checkpoint_reached\", attributes={\"items\": 42})\n\n# Record exceptions (span continues)\ntry:\n    risky_operation()\nexcept Exception as e:\n    client.record_exception(e)\n```\n\n---\n\n## Custom Processors\n\nRoute spans to multiple destinations with filtering:\n\n```python\nfrom respan_tracing import RespanTelemetry\n\ntelemetry = RespanTelemetry(api_key=\"...\", base_url=\"...\")\n\n# Add a second exporter for debug logging\nfrom opentelemetry.sdk.trace.export import ConsoleSpanExporter\n\ntelemetry.add_processor(\n    exporter=ConsoleSpanExporter(),\n    name=\"debug\",\n    filter_fn=lambda span: span.attributes.get(\"environment\") == \"debug\",\n)\n```\n\nRoute specific spans to a processor using the `processors` parameter on decorators:\n\n```python\n@task(name=\"sensitive_step\", processors=\"debug\")\ndef sensitive_step():\n    ...\n```\n\n---\n\n## Flush\n\nAlways call `flush()` before process exit to ensure all spans are exported:\n\n```python\nrespan = Respan()\n# ... your code ...\nrespan.flush()\n```\n\nIn serverless/Lambda: call `flush()` at the end of every handler invocation.\n\n---\n\n## Trace Hierarchy Example\n\n```\nworkflow: handle_request\n  +-- task: classify_intent\n  |     +-- llm: openai.chat (auto-captured)\n  +-- agent: support_agent\n  |     +-- tool: lookup_order\n  |     +-- llm: openai.chat (auto-captured)\n  |     +-- tool: process_refund\n  +-- task: generate_response\n        +-- llm: openai.chat (auto-captured)\n```\n\nLLM calls within decorated functions are automatically nested as child spans.\n";
+export declare const GATEWAY_MD = "# Gateway Reference\n\nRoute LLM calls through the Respan proxy for logging, caching, key management, fallbacks, and model switching.\n\nDocs: `https://respan.ai/docs/documentation/features/gateway/gateway-quickstart.md`\n\n---\n\n## Setup\n\nPoint the LLM client's `base_url` at the Respan gateway. The Respan API key authenticates both the proxy and tracing.\n\n### Python (OpenAI SDK)\n\n```python\nimport os\nfrom openai import OpenAI\n\nclient = OpenAI(\n    api_key=os.environ[\"RESPAN_API_KEY\"],\n    base_url=os.getenv(\"RESPAN_BASE_URL\", \"https://api.respan.ai/api\"),\n)\n\nresponse = client.chat.completions.create(\n    model=\"gpt-4o-mini\",\n    messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n)\n```\n\n### TypeScript (OpenAI SDK)\n\n```typescript\nimport OpenAI from \"openai\";\n\nconst client = new OpenAI({\n  apiKey: process.env.RESPAN_API_KEY,\n  baseURL: process.env.RESPAN_BASE_URL || \"https://api.respan.ai/api\",\n});\n\nconst response = await client.chat.completions.create({\n  model: \"gpt-4o-mini\",\n  messages: [{ role: \"user\", content: \"Hello!\" }],\n});\n```\n\n### Python (Anthropic SDK)\n\n```python\nimport os\nimport anthropic\n\nclient = anthropic.Anthropic(\n    api_key=os.environ[\"RESPAN_API_KEY\"],\n    base_url=os.getenv(\"RESPAN_BASE_URL\", \"https://api.respan.ai/api\"),\n)\n```\n\n### Environment variables\n\nSet once, works everywhere:\n\n```bash\nexport RESPAN_API_KEY=\"your-key\"\nexport OPENAI_API_KEY=\"$RESPAN_API_KEY\"\nexport OPENAI_BASE_URL=\"https://api.respan.ai/api\"\n```\n\n---\n\n## Gateway + Tracing (Both)\n\nCombine SDK tracing and gateway routing:\n\n### Python\n\n```python\nimport os\nfrom openai import OpenAI\nfrom respan import Respan\n\nrespan_api_key = os.environ[\"RESPAN_API_KEY\"]\nrespan_base_url = os.getenv(\"RESPAN_BASE_URL\", \"https://api.respan.ai/api\")\n\nrespan = Respan(api_key=respan_api_key, base_url=respan_base_url)\n\nclient = OpenAI(\n    api_key=respan_api_key,\n    base_url=respan_base_url,\n)\n```\n\n### TypeScript\n\n```typescript\nimport OpenAI from \"openai\";\nimport { Respan } from \"@respan/respan\";\n\nconst respanApiKey = process.env.RESPAN_API_KEY!;\nconst respanBaseUrl = process.env.RESPAN_BASE_URL || \"https://api.respan.ai/api\";\n\nconst respan = new Respan({ apiKey: respanApiKey, baseUrl: respanBaseUrl });\nawait respan.initialize();\n\nconst client = new OpenAI({\n  apiKey: respanApiKey,\n  baseURL: respanBaseUrl,\n});\n```\n\n---\n\n## Framework-Specific Gateway Setup\n\n### CrewAI\n\n```python\nimport os\nos.environ[\"OPENAI_API_KEY\"] = os.environ[\"RESPAN_API_KEY\"]\nos.environ[\"OPENAI_BASE_URL\"] = os.getenv(\"RESPAN_BASE_URL\", \"https://api.respan.ai/api\")\n\nfrom crewai import Agent, Task, Crew\n# CrewAI uses the env vars automatically\n```\n\n### Haystack\n\n```python\nimport os\nos.environ[\"OPENAI_API_KEY\"] = os.environ[\"RESPAN_API_KEY\"]\nos.environ[\"OPENAI_BASE_URL\"] = os.getenv(\"RESPAN_BASE_URL\", \"https://api.respan.ai/api\")\n\nfrom haystack.components.generators import OpenAIGenerator\n# OpenAIGenerator picks up env vars automatically\n```\n\n### OpenAI Agents SDK\n\n```python\nimport os\nos.environ[\"OPENAI_API_KEY\"] = os.environ[\"RESPAN_API_KEY\"]\nos.environ[\"OPENAI_BASE_URL\"] = os.getenv(\"RESPAN_BASE_URL\", \"https://api.respan.ai/api\")\n```\n\n### Pydantic AI\n\n```python\nimport os\nos.environ[\"OPENAI_API_KEY\"] = os.environ[\"RESPAN_API_KEY\"]\nos.environ[\"OPENAI_BASE_URL\"] = os.getenv(\"RESPAN_BASE_URL\", \"https://api.respan.ai/api\")\n\nfrom pydantic_ai import Agent\nagent = Agent(model=\"openai:gpt-4o-mini\")\n```\n\n---\n\n## Model Switching\n\nThe gateway supports 250+ models across providers. Use any model name:\n\n```python\n# OpenAI models\nresponse = client.chat.completions.create(model=\"gpt-4o\", ...)\n\n# Anthropic models (via OpenAI SDK!)\nresponse = client.chat.completions.create(model=\"claude-sonnet-4-20250514\", ...)\n\n# Google models\nresponse = client.chat.completions.create(model=\"gemini-2.0-flash\", ...)\n\n# Open-source models\nresponse = client.chat.completions.create(model=\"llama-3.1-70b\", ...)\n```\n\nNo code changes needed \u2014 just change the model string. View all models at `https://platform.respan.ai/platform/models`.\n\n---\n\n## Advanced Features\n\n### Fallback Models\n\nAutomatic failover if the primary model fails:\n\n```python\nresponse = client.chat.completions.create(\n    model=\"gpt-4o\",\n    messages=[...],\n    extra_body={\n        \"fallback_models\": [\"claude-sonnet-4-20250514\", \"gemini-2.0-flash\"],\n    },\n)\n```\n\n### Retries\n\n```python\nresponse = client.chat.completions.create(\n    model=\"gpt-4o\",\n    messages=[...],\n    extra_body={\n        \"retries\": 3,\n    },\n)\n```\n\n### Caching\n\nEnable response caching to save cost on repeated calls:\n\n```python\nresponse = client.chat.completions.create(\n    model=\"gpt-4o\",\n    messages=[...],\n    extra_body={\n        \"cache_enabled\": True,\n        \"cache_ttl\": 3600,  # seconds\n    },\n)\n```\n\n### Streaming\n\nStreaming works normally through the gateway:\n\n```python\nstream = client.chat.completions.create(\n    model=\"gpt-4o-mini\",\n    messages=[{\"role\": \"user\", \"content\": \"Tell me a story\"}],\n    stream=True,\n)\nfor chunk in stream:\n    print(chunk.choices[0].delta.content, end=\"\")\n```\n\n### Disable Logging\n\nFor sensitive data, disable logging on specific requests:\n\n```python\nresponse = client.chat.completions.create(\n    model=\"gpt-4o\",\n    messages=[...],\n    extra_body={\n        \"disable_log\": True,\n    },\n)\n```\n\n---\n\n## Key Benefits\n\n- **Unified key management** \u2014 one Respan API key for all providers\n- **Automatic logging** \u2014 every request logged as a span\n- **Cost tracking** \u2014 per-request cost tracking across providers\n- **Caching** \u2014 cache repeated LLM calls to reduce cost and latency\n- **Fallbacks** \u2014 automatic failover between providers\n- **Model switching** \u2014 swap models without code changes\n- **~50-150ms added latency** \u2014 minimal overhead\n";
+export declare const PROMPTS_MD = "# Prompts Reference\n\nCreate, version, deploy, and use prompt templates from the Respan platform.\n\nDocs: `https://respan.ai/docs/documentation/features/prompt-management/prompt-management-quickstart.md`\n\n---\n\n## Overview\n\nPrompts are version-controlled templates stored on the Respan platform. The workflow is:\n\n1. **Create** a prompt with a name\n2. **Add versions** with messages, model config, and `{{variables}}`\n3. **Deploy** the version you want live\n4. **Fetch** the prompt in code using the API or MCP tools\n5. **Iterate** \u2014 commit new versions and deploy without code changes\n\n---\n\n## Create a Prompt\n\n### Via MCP\n\n```\nUse create_prompt to create a prompt:\n  name: \"classify_intent\"\n  description: \"Classifies user messages into intent categories\"\n\nThen use create_prompt_version to add content:\n  prompt_id: \"<id from above>\"\n  model: \"gpt-4o-mini\"\n  messages:\n    - role: \"system\"\n      content: \"You are an intent classifier. Classify the user message into one of: question, complaint, request, feedback.\"\n    - role: \"user\"\n      content: \"{{user_message}}\"\n```\n\n### Via CLI\n\n```bash\nrespan prompts create --name \"classify_intent\"\nrespan prompts versions <prompt-id>\n```\n\n---\n\n## Variables\n\nUse `{{variable_name}}` in message content. Variables are filled at runtime.\n\n```json\n{\n  \"messages\": [\n    {\"role\": \"system\", \"content\": \"You are a {{role}} assistant.\"},\n    {\"role\": \"user\", \"content\": \"{{user_input}}\"}\n  ]\n}\n```\n\n### Jinja templates\n\nPrompts support Jinja2 syntax:\n\n- **Conditionals:** `{% if context %}Use this context: {{context}}{% endif %}`\n- **JSON access:** `{{ input.key }}`\n- **Filters:** `{{ var | upper }}`\n- **Loops:** `{% for item in items %}{{item}}{% endfor %}`\n- **Default values:** Set via the API or platform UI\n\n---\n\n## Versioning and Deployment\n\nEach prompt can have multiple versions. Only one version is **deployed** (live) at a time.\n\n### Workflow\n\n1. `create_prompt_version` \u2014 creates a new version (always starts as NOT deployed)\n2. Test the version in the playground or via API\n3. Deploy via platform UI when ready\n4. Code using `prompt_id` automatically gets the deployed version\n\n### Version pinning\n\n```python\n# Always use the deployed version (default)\nprompt = get_prompt(prompt_id=\"abc123\")\n\n# Pin to a specific version\nprompt = get_prompt(prompt_id=\"abc123\", version=3)\n```\n\n---\n\n## Using Prompts in Code\n\n### Python (via Gateway)\n\nFetch the prompt config and pass to the LLM:\n\n```python\nimport os\nfrom openai import OpenAI\n\nclient = OpenAI(\n    api_key=os.environ[\"RESPAN_API_KEY\"],\n    base_url=os.getenv(\"RESPAN_BASE_URL\", \"https://api.respan.ai/api\"),\n)\n\n# The gateway resolves the prompt template and fills variables\nresponse = client.chat.completions.create(\n    model=\"gpt-4o-mini\",\n    messages=[],  # Messages come from the prompt template\n    extra_body={\n        \"prompt_id\": \"your-prompt-id\",\n        \"prompt_variables\": {\n            \"user_message\": \"I want to return my order\",\n        },\n    },\n)\n```\n\n### Linking traces to prompts\n\nUse `propagate_attributes` to tag traces with the prompt used:\n\n```python\nfrom respan import propagate_attributes\n\nwith propagate_attributes(\n    prompt={\"prompt_id\": \"abc123\", \"variables\": {\"user_message\": \"Hello\"}},\n):\n    response = client.chat.completions.create(...)\n```\n\n---\n\n## Prompt Composition\n\nVariables can reference other prompts (child prompts rendered first, injected as text).\n\nConfigure in the platform UI by changing a variable's type from **Text** to **Prompt**.\n\nMax nesting depth: 2 (parent -> child).\n\n---\n\n## Structured Output (JSON Schema)\n\nAttach a JSON schema to a prompt version for structured responses:\n\n```json\n{\n  \"response_format\": {\n    \"type\": \"json_schema\",\n    \"json_schema\": {\n      \"name\": \"extraction\",\n      \"strict\": true,\n      \"schema\": {\n        \"type\": \"object\",\n        \"properties\": {\n          \"intent\": {\"type\": \"string\"},\n          \"confidence\": {\"type\": \"number\"}\n        },\n        \"required\": [\"intent\", \"confidence\"],\n        \"additionalProperties\": false\n      }\n    }\n  }\n}\n```\n\n---\n\n## MCP Tools Reference\n\n| Action | Tool |\n|--------|------|\n| List all prompts | `list_prompts` |\n| Get prompt details | `get_prompt_detail(prompt_id)` |\n| Create a prompt | `create_prompt(name, description)` |\n| Update name/description | `update_prompt(prompt_id, name, description)` |\n| Create a version | `create_prompt_version(prompt_id, messages, model, ...)` |\n| Update a version | `update_prompt_version(prompt_id, version, ...)` |\n| List versions | `list_prompt_versions(prompt_id)` |\n| Get version detail | `get_prompt_version_detail(prompt_id, version)` |\n\n---\n\n## CLI Commands\n\n```bash\nrespan prompts list                    # List all prompts\nrespan prompts get <id>                # Get prompt details\nrespan prompts create --name <name>    # Create a prompt\nrespan prompts update <id>             # Update a prompt\nrespan prompts versions <id>           # List prompt versions\n```\n";
+export declare const EVALS_MD = "# Evals Reference\n\nBuild evaluation pipelines: datasets, evaluators, and experiments.\n\nDocs: `https://respan.ai/docs/documentation/features/evals/concepts.md`\n\n---\n\n## Overview\n\nTwo evaluation modes:\n\n- **Offline** \u2014 build datasets, run experiments across prompt/model variants, compare scores\n- **Online** \u2014 automatically score live production traffic (see [monitors.md](monitors.md))\n\nOffline eval workflow:\n\n```\nCreate dataset -> Create evaluator -> Create experiment -> Run & compare\n```\n\n---\n\n## Datasets\n\nA dataset is a collection of test rows (input/output pairs) for experiments.\n\n### Create from scratch\n\n```\nUse create_dataset:\n  name: \"intent_classification_testset\"\n  description: \"Test cases for intent classifier\"\n\nThen add rows with create_dataset_span:\n  dataset_id: \"<id>\"\n  input: '{\"user_message\": \"I want a refund\"}'\n  output: '{\"intent\": \"refund_request\"}'\n```\n\n### Create from production traces\n\nSample real spans from traced data:\n\n```\n1. Use list_traces or list_logs to find relevant spans\n2. Use add_spans_to_dataset to add span IDs to a dataset:\n     dataset_id: \"<id>\"\n     span_ids: [\"span_1\", \"span_2\", \"span_3\"]\n```\n\nOr use the platform UI: Datasets -> Insert by sampling (filter, set sampling %, preview, confirm).\n\n### CSV import\n\nUpload CSV via the platform UI. Columns map to prompt variables. Optional `ideal_output` column for expected outputs. Max 500 rows per import.\n\n### MCP tools\n\n| Action | Tool |\n|--------|------|\n| Create dataset | `create_dataset(name, description)` |\n| List datasets | `list_datasets` |\n| Get dataset | `get_dataset(dataset_id)` |\n| Update dataset | `update_dataset(dataset_id, name, description)` |\n| Add a row manually | `create_dataset_span(dataset_id, input, output)` |\n| Add existing spans | `add_spans_to_dataset(dataset_id, span_ids)` |\n| List rows | `list_dataset_spans(dataset_id)` |\n| Get a row | `retrieve_dataset_span(dataset_id, log_id)` |\n| Update a row | `update_dataset_span(dataset_id, log_id, ...)` |\n\n---\n\n## Evaluators\n\nAn evaluator scores LLM outputs. It is a workflow built from **graders** connected with conditions.\n\n### Grader types\n\n| Type | How it works |\n|------|-------------|\n| **LLM grader** | A language model judges the output. Uses variables: `{{output}}`, `{{input}}`, `{{expected_output}}`, `{{metadata}}`, `{{metrics}}` |\n| **Code grader** | A Python function returns a score. Signature: `main(eval_inputs)` where `eval_inputs[\"output\"]` is required |\n| **Human grader** | Team members review and score outputs manually |\n\n### Score value types\n\n| Type | Use case |\n|------|----------|\n| `numerical` | Rating scales (e.g. 0-5) |\n| `boolean` | Pass/fail checks |\n| `categorical` | Multi-choice classifications |\n| `comment` | Qualitative text feedback |\n\n### Create an evaluator\n\n```\nUse create_evaluator:\n  name: \"response_quality\"\n  score_value_type: \"numerical\"\n  type: \"llm\"\n  description: \"Scores response quality 1-5\"\n  evaluator_slug: \"response-quality\"\n```\n\nFor complex evaluators with grader workflows, conditions, and compute blocks \u2014 use the platform UI (Evaluators -> Create).\n\n### Evaluator workflow blocks\n\n- **Markers:** `Original input`, `Final result` (entry/exit)\n- **Graders:** LLM, Code, or Human scoring\n- **Conditions:** `If...Then...Else` branching based on scores\n- **Compute:** Average/weighted average of multiple grader scores\n- **Metrics:** Built-in: completion tokens, cost, latency, model, prompt tokens, total tokens\n- **Constants:** Fixed threshold values\n\n### MCP tools\n\n| Action | Tool |\n|--------|------|\n| List evaluators | `list_evaluators` |\n| Get evaluator | `get_evaluator(evaluator_id)` |\n| Create evaluator | `create_evaluator(name, score_value_type, ...)` |\n| Update evaluator | `update_evaluator(evaluator_id, ...)` |\n| Run evaluator | `run_evaluator(evaluator_id, dataset_id or log_ids)` |\n\n---\n\n## Experiments\n\nAn experiment runs evaluators across a dataset to compare prompt/model configurations.\n\n### Workflow\n\n1. Create a dataset with test rows\n2. Create evaluator(s) to score outputs\n3. Create an experiment linking dataset + evaluators\n4. Run the experiment \u2014 it generates outputs for each row and scores them\n5. Compare results across configurations\n\n### Create an experiment\n\n```\nUse create_experiment:\n  name: \"v2_vs_v3_comparison\"\n  dataset_id: \"<dataset-id>\"\n  evaluator_slugs: [\"response-quality\", \"hallucination-check\"]\n  workflows:\n    - type: \"prompt\"\n      config: { prompt_id: \"<prompt-v2-id>\" }\n    - type: \"prompt\"\n      config: { prompt_id: \"<prompt-v3-id>\" }\n```\n\n### Workflow types\n\n| Type | Description |\n|------|-------------|\n| `prompt` | Run a saved prompt template against dataset rows |\n| `completion` | Run a model completion with config |\n| `custom` | User-defined logic |\n\n### MCP tools\n\n| Action | Tool |\n|--------|------|\n| List experiments | `list_experiments` |\n| Get experiment | `get_experiment(experiment_id)` |\n| Create experiment | `create_experiment(name, dataset_id, ...)` |\n| List experiment spans | `list_experiment_spans(experiment_id)` |\n| Get experiment span | `get_experiment_span(experiment_id, log_id)` |\n| Update experiment span | `update_experiment_span(experiment_id, log_id, ...)` |\n| Get summary stats | `get_experiment_spans_summary(experiment_id, start_time, end_time)` |\n\n---\n\n## End-to-End Example\n\n```\nStep 1: Create dataset\n  create_dataset(name=\"qa_testset\")\n  create_dataset_span(dataset_id=\"...\", input=\"What is Python?\", output=\"A programming language\")\n  create_dataset_span(dataset_id=\"...\", input=\"Who made Rust?\", output=\"Mozilla Research\")\n\nStep 2: Create evaluator\n  create_evaluator(\n    name=\"accuracy_check\",\n    score_value_type=\"boolean\",\n    type=\"llm\",\n    evaluator_slug=\"accuracy-check\"\n  )\n\nStep 3: Create experiment\n  create_experiment(\n    name=\"gpt4o_vs_claude\",\n    dataset_id=\"...\",\n    evaluator_slugs=[\"accuracy-check\"]\n  )\n\nStep 4: Review results\n  list_experiment_spans(experiment_id=\"...\")\n  get_experiment_spans_summary(experiment_id=\"...\", start_time=\"...\", end_time=\"...\")\n```\n\n---\n\n## CLI Commands\n\n```bash\nrespan datasets list                   # List datasets\nrespan datasets create --name <name>   # Create a dataset\nrespan datasets get <id>               # Get dataset details\nrespan datasets spans <id>             # List dataset spans\nrespan evaluators list                 # List evaluators\nrespan evaluators get <id>             # Get evaluator details\nrespan evaluators create               # Create an evaluator\nrespan evaluators run <id>             # Run an evaluator\nrespan experiments list                # List experiments\nrespan experiments get <id>            # Get experiment details\nrespan experiments create              # Create an experiment\nrespan eval <file>                     # Run a one-shot eval defined in a JSON file\n```\n\n## One-Shot Evals via JSON\n\n`respan eval <file.json>` runs a complete eval pipeline from a single JSON spec \u2014\ncreates the prompt, dataset, evaluators, and experiment in one command.\nIDs are written back into the file so reruns reuse existing resources.\n\n### Minimal example\n\n```json\n{\n  \"name\": \"Movie matcher\",\n  \"prompt\": {\n    \"name\": \"movie-matcher\",\n    \"model\": \"gpt-5-mini\",\n    \"messages\": [\n      { \"role\": \"system\", \"content\": \"Identify the movie. Reply with only the title.\" },\n      { \"role\": \"user\", \"content\": \"{{description}}\" }\n    ]\n  },\n  \"dataset\": {\n    \"name\": \"movie-descriptions\",\n    \"rows\": [\n      { \"description\": \"A detective investigates seven deadly sins.\", \"expected_output\": \"Se7en\" },\n      { \"description\": \"A hacker learns reality is a simulation.\", \"expected_output\": \"The Matrix\" }\n    ]\n  },\n  \"experiment\": {\n    \"evaluators\": [\n      {\n        \"name\": \"Title match\",\n        \"type\": \"llm\",\n        \"score_value_type\": \"boolean\",\n        \"llm_config\": {\n          \"model\": \"openai/gpt-5-mini\",\n          \"evaluator_definition\": \"Expected: {{expected_output}}. Got: {{output}}. Correct? Reply true or false.\"\n        }\n      }\n    ]\n  }\n}\n```\n\nRun:\n\n```bash\nrespan eval movie-matcher.eval.json\n```\n\nThe runner creates the prompt (and deploys version 1), creates the dataset (and bulk-loads rows),\ncreates each evaluator + its workflow (and deploys), then creates the experiment linking them.\nEach step writes the resulting IDs back to the JSON file so subsequent runs are idempotent \u2014\nre-running with the same file reuses the existing resources.\n";
+export declare const MONITORS_MD = "# Monitors & Automation Reference\n\nSet up monitoring alerts, online evaluations, and automated workflows.\n\nDocs: `https://respan.ai/docs/documentation/features/monitoring/monitors.md`\n\n---\n\n## Monitors\n\nA monitor watches a metric over time and sends a notification when a threshold is crossed.\n\n### Examples\n\n- Error rate > 5% over 10 minutes\n- Total cost > $1.00 over 5 minutes\n- Average latency > 3 seconds over 15 minutes\n- P95 latency > 5 seconds over 1 hour\n\n### Setup\n\nMonitors are configured in the platform UI: **Monitoring -> Monitors -> Create**.\n\nA monitor has two parts:\n\n#### 1. Trigger\n\nTemplate: `When [metric] of [source] is [threshold] over [time window]`\n\n- **Metric:** cost, latency, error rate, token usage, request count, etc.\n- **Source:** all requests, specific model, specific customer, etc.\n- **Threshold:** condition (greater than, less than, equals)\n- **Time window:** evaluation period (1 min, 5 min, 10 min, 1 hour, etc.)\n\nAdd **Where** conditions to narrow scope:\n- Filter by model, project, environment, customer, or custom metadata\n\n#### 2. Notifications\n\nDestinations:\n\n| Destination | Details |\n|-------------|---------|\n| **Email** | Customizable subject and body |\n| **Slack** | Customizable message |\n| **Webhook** | Raw URL \u2014 receives JSON payload |\n\nA single monitor can send to multiple destinations.\n\n### Manage\n\n- **Deploy** to make a monitor live\n- **Pause/Resume** deployed monitors\n- **Send test alert** to validate before deploying\n- **View versions** and restore earlier versions as drafts\n- **Create from chart** \u2014 select a metric on the dashboard, click to create a monitor with metric/source/interval prefilled\n\n---\n\n## Online Evaluations (Automation)\n\nAutomatically score production traffic in real time using deployed evaluators.\n\n### Setup\n\nConfigure in the platform UI: **Evaluators -> Automations -> Create**.\n\n1. **Create a condition** \u2014 filter rules for which logs to evaluate:\n   - **Single log** \u2014 evaluate individual spans matching criteria\n   - **Aggregation** \u2014 evaluate aggregated metrics over a window\n\n2. **Create an automation** \u2014 connect:\n   - Condition (which logs)\n   - Evaluator (which scoring workflow)\n   - Sampling rate (0.0-1.0, fraction of matching logs to evaluate)\n\n3. **Check results** \u2014 view eval scores in the automation table\n\n### Use case\n\n> \"Score 10% of production chat responses for hallucination using my LLM evaluator\"\n\n```\nCondition: log_type = \"chat\" AND model = \"gpt-4o\"\nEvaluator: hallucination-check\nSampling: 0.1\n```\n\n---\n\n## Webhooks\n\nReceive event notifications via HTTP webhooks.\n\n### Setup\n\n1. Go to platform: **Settings -> Webhooks -> Create**\n2. Enter your endpoint URL\n3. Select event types to subscribe to\n4. Optionally set a webhook secret for HMAC-SHA256 verification\n\n### Verification\n\nRespan signs webhook payloads with HMAC-SHA256. Verify the signature:\n\n```python\nimport hashlib\nimport hmac\n\ndef verify_webhook(payload: bytes, signature: str, secret: str) -> bool:\n    expected = hmac.new(\n        secret.encode(),\n        payload,\n        hashlib.sha256,\n    ).hexdigest()\n    return hmac.compare_digest(expected, signature)\n```\n\nThe signature is sent in the request headers.\n\n---\n\n## Metrics Dashboard\n\nTrack key indicators in the platform: **Dashboard -> Metrics**.\n\n### Available metrics\n\n- **Requests:** total count, requests per model/customer/API key\n- **Tokens:** prompt tokens, completion tokens, total tokens\n- **Cost:** per-request, per-model, per-customer\n- **Latency:** average, P50, P95, P99\n- **Errors:** error count, error rate\n- **Time to first token (TTFT):** streaming latency\n\n### Breakdowns\n\nSlice metrics by:\n- Model\n- Customer (`customer_identifier`)\n- API key\n- Prompt template\n- Environment\n- Custom metadata keys\n\n### Per-user analytics\n\nUse `customer_identifier` in spans to enable:\n- Per-user cost tracking and budgets\n- Per-user rate limiting\n- User-level quality metrics\n\n---\n\n## CLI Commands\n\n```bash\nrespan traces list --limit 10          # List recent traces\nrespan traces get <trace-id>           # Get trace details\nrespan traces summary                  # Trace summary stats\nrespan logs list --limit 10            # List recent logs\nrespan logs get <log-id>               # Get log details\nrespan logs summary                    # Log summary stats\n```\n";