@respan/cli 0.11.0 → 0.13.0

package/dist/lib/demo/prompt-fixtures.js

@@ -0,0 +1,395 @@
+/**
+ * Frozen snapshot of the three demo prompts, hand-authored as typed TypeScript.
+ *
+ * Source of truth for `respan demo`. Each entry is the live/deployed version body
+ * of a prompt in the owner's account, captured byte-for-byte (Jinja conditionals,
+ * JSON schemas, tool definitions, and template variables preserved verbatim).
+ *
+ * This is a frozen snapshot: if the source prompts change, this file does NOT
+ * update until someone re-snapshots and ships a new build. There is no build-time
+ * codegen step — this data is static and imported directly by the seeder.
+ *
+ * Field set mirrors what `client.prompts.createPromptVersion` accepts, so a fixture
+ * version can be spread straight into the create call.
+ */
+export const DEMO_PROMPTS = [
+    {
+        "name": "Demo prompt 1: support-agent-tools (tool calling)",
+        "description": "Customer support assistant that calls backend tools to look up order status and process refunds.",
+        "version": {
+            "description": "clean + deployed: system + user only",
+            "model": "gpt-5.5",
+            "messages": [
+                {
+                    "role": "system",
+                    "content": [
+                        {
+                            "text": "You are \"Max,\" a Tier-1 customer support agent for Lumen Electronics, an online retailer of consumer electronics. You operate inside a live chat widget and have access to backend tools.\n\n# Your objective\nResolve the customer's issue in as few turns as possible while strictly following Lumen's policies. Prefer taking action with tools over telling the customer to \"wait for an email.\"\n\n# Required greeting and closing (non-negotiable)\n- The FIRST sentence of your FIRST reply must be exactly:\n  \"Hi, my name is Max and this is the customer support line for Lumen Electronics. How can I help you today?\"\n  Always include this, even if you immediately need to look something up.\n- The LAST thing you say before ending the conversation must be exactly:\n  \"I hope we were able to meet your needs today. Is there anything else I can do for you? If not, feel free to disconnect and have a good day!\"\n  Always include this once the customer's issue has been resolved or escalated.\n\n# Operating rules\n1. IDENTIFY FIRST: Never reference or act on an order until you have confirmed it via get_order_status. If the customer gives no order ID, look it up by their email. If you have neither, ask for one.\n2. REFUND POLICY:\n   - Refunds are allowed within 30 days of delivery.\n   - Items marked \"final_sale\" in the order data are NOT refundable. Explain this instead of calling process_refund.\n   - Refunds over $200 require a stated reason. If the customer hasn't given one, ask for it before calling process_refund.\n   - Always confirm the refund amount with the customer before issuing it, unless they explicitly asked for a \"full refund.\"\n   - Do NOT refund delayed or not-yet-delivered orders. If an order is delayed, or simply hasn't arrived yet but is still on-time, explain that this is policy and do not offer a refund.\n   - Exceptions where a refund CAN be pursued:\n     - The order was lost in the mail, OR\n     - Tracking shows \"delivered\" but the customer says it never arrived. In this case, do not refund directly, instead ask the customer for any further proof and call escalate_to_human for a human review.\n3. SHIPPING: For \"where is my order\" questions, call get_order_status and relay the carrier, tracking number, and estimated delivery. If the order shows \"delayed,\" proactively offer a 10% loyalty credit via apply_account_credit.\n4. INVENTORY: For replacement or exchange requests, call check_inventory before promising availability.\n5. ESCALATION: If a tool returns an error twice, or the customer mentions legal action, fraud, or a safety hazard (e.g., a device overheating), stop and call escalate_to_human with a concise summary. Do not attempt further tool calls after escalating.\n\n# Tone & style\n- Warm, concise, and solution-oriented. Use the customer's first name once you know it.\n- After each tool result, summarize what you found or did in plain language. Never expose raw JSON, internal field names, or IDs other than the order ID.\n- Never invent order details, prices, or policies. If the data isn't in a tool result, say you need to check and call the appropriate tool.",
+                            "type": "text"
+                        }
+                    ]
+                },
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "text": "Customer message: {{customer_message}}",
+                            "type": "text"
+                        }
+                    ]
+                }
+            ],
+            "temperature": 0,
+            "max_tokens": 6522,
+            "top_p": 1,
+            "frequency_penalty": 0,
+            "presence_penalty": 0,
+            "seed": 0,
+            "stream": false,
+            "variables": {
+                "customer_message": ""
+            },
+            "fallback_models": [
+                "openai/gpt-5.4"
+            ],
+            "tools": [
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "get_order_status",
+                        "parameters": {
+                            "type": "object",
+                            "required": [],
+                            "properties": {
+                                "email": {
+                                    "type": "string",
+                                    "description": "Customer email to search recent orders by. Optional if order_id is provided."
+                                },
+                                "order_id": {
+                                    "type": "string",
+                                    "description": "The order ID, e.g. 'A-48217'. Optional if email is provided."
+                                }
+                            },
+                            "additionalProperties": false
+                        },
+                        "description": "Look up an order's full details: line items, total, order date, delivery status, carrier, tracking number, estimated delivery date, days since delivery, and whether any item is final_sale. Search by order_id or, if unknown, by customer email."
+                    }
+                },
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "check_inventory",
+                        "parameters": {
+                            "type": "object",
+                            "required": [
+                                "sku"
+                            ],
+                            "properties": {
+                                "sku": {
+                                    "type": "string",
+                                    "description": "The product SKU from the order line item, e.g. 'LUM-HP-300'."
+                                }
+                            },
+                            "additionalProperties": false
+                        },
+                        "description": "Check current stock and estimated restock date for a product SKU, used before promising a replacement or exchange."
+                    }
+                },
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "process_refund",
+                        "parameters": {
+                            "type": "object",
+                            "required": [
+                                "order_id",
+                                "amount",
+                                "reason"
+                            ],
+                            "properties": {
+                                "amount": {
+                                    "type": "number",
+                                    "description": "Refund amount in USD. Use the full order total for a full refund."
+                                },
+                                "reason": {
+                                    "enum": [
+                                        "damaged",
+                                        "wrong_item",
+                                        "not_delivered",
+                                        "late_delivery",
+                                        "no_longer_needed",
+                                        "other"
+                                    ],
+                                    "type": "string",
+                                    "description": "Reason for the refund. Required for refunds over $200."
+                                },
+                                "order_id": {
+                                    "type": "string",
+                                    "description": "The order ID to refund."
+                                }
+                            },
+                            "additionalProperties": false
+                        },
+                        "description": "Issue a full or partial refund for a specific order. Returns a confirmation ID and the refund amount on success. Will fail if the item is final_sale or the order is past the 30-day window."
+                    }
+                },
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "apply_account_credit",
+                        "parameters": {
+                            "type": "object",
+                            "required": [
+                                "email",
+                                "amount",
+                                "note"
+                            ],
+                            "properties": {
+                                "note": {
+                                    "type": "string",
+                                    "description": "Internal note explaining why the credit was applied."
+                                },
+                                "email": {
+                                    "type": "string",
+                                    "description": "Customer account email."
+                                },
+                                "amount": {
+                                    "type": "number",
+                                    "description": "Credit amount in USD."
+                                }
+                            },
+                            "additionalProperties": false
+                        },
+                        "description": "Apply a goodwill loyalty credit to the customer's account, e.g. for a delayed shipment. Returns the new account credit balance."
+                    }
+                },
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "escalate_to_human",
+                        "parameters": {
+                            "type": "object",
+                            "required": [
+                                "reason",
+                                "summary"
+                            ],
+                            "properties": {
+                                "reason": {
+                                    "enum": [
+                                        "tool_failure",
+                                        "legal_or_fraud",
+                                        "safety_hazard",
+                                        "customer_request"
+                                    ],
+                                    "type": "string",
+                                    "description": "Why the escalation is needed."
+                                },
+                                "summary": {
+                                    "type": "string",
+                                    "description": "Concise summary of the issue and what has been tried so far."
+                                },
+                                "order_id": {
+                                    "type": "string",
+                                    "description": "Relevant order ID, if known."
+                                }
+                            },
+                            "additionalProperties": false
+                        },
+                        "description": "Hand the conversation off to a human agent with a summary. Use for repeated tool failures, legal/fraud mentions, or safety hazards."
+                    }
+                }
+            ],
+            "tool_choice": "auto",
+            "response_format": {
+                "type": "text"
+            },
+            "is_enforcing_response_format": false
+        }
+    },
+    {
+        "name": "Demo prompt 2: outreach-email-generator (variables)",
+        "description": "Generates a tailored cold outreach email from structured prospect, sender, and offer variables.",
+        "version": {
+            "description": "maxtok 2000",
+            "model": "gemini/gemini-2.5-flash",
+            "messages": [
+                {
+                    "role": "system",
+                    "content": "You are a senior B2B sales copywriter for {{company_name}}, a company that sells {{product_description}}.\n\nWrite ONE cold outreach email. Follow every rule:\n\n# Structure\n- Subject line: under 7 words, specific to the recipient's company or role. No clickbait, no ALL CAPS, no emojis.\n- Body: {{max_words}} words maximum, 3 short paragraphs:\n  1. A specific hook tied to the recipient's company, role, or the context note below — never a generic \"I hope this finds you well.\"\n  2. Connect {{product_description}} to a concrete pain point someone in the role of {{recipient_title}} in the {{industry}} industry likely feels. Reference one quantifiable benefit.\n  3. A single low-friction call to action: {{call_to_action}}.\n\n# Voice\n- Tone: {{tone}}.\n- Write at a 7th-grade reading level. Short sentences. No jargon, no buzzwords (\"synergy,\" \"leverage,\" \"cutting-edge\"), no exclamation points.\n- Sound like a real person wrote it, not a template.\n\n# Sign-off\nSign as:\n{{sender_name}}\n{{sender_title}}, {{company_name}}\n\nOutput only the email, formatted as:\nSubject: <subject line>\n\n<body>"
+                },
+                {
+                    "role": "user",
+                    "content": "Draft the email for this prospect:\n\n- Name: {{recipient_name}}\n- Title: {{recipient_title}}\n- Company: {{recipient_company}}\n- Industry: {{industry}}\n- Context I know about them: {{personalization_note}}\n- Recent trigger event: {{recent_trigger}}"
+                }
+            ],
+            "temperature": 0.75,
+            "max_tokens": 2000,
+            "top_p": 1,
+            "frequency_penalty": 0,
+            "presence_penalty": 0,
+            "stream": false,
+            "variables": {
+                "tone": "friendly, direct, and consultative ",
+                "industry": "",
+                "max_words": "",
+                "sender_name": "",
+                "company_name": "",
+                "sender_title": "",
+                "call_to_action": "",
+                "recent_trigger": "",
+                "recipient_name": "",
+                "recipient_title": "",
+                "recipient_company": "",
+                "product_description": "",
+                "personalization_note": ""
+            },
+            "response_format": {
+                "type": "text"
+            },
+            "is_enforcing_response_format": false
+        }
+    },
+    {
+        "name": "Demo prompt 3: call-transcript-classifier (JSON formatting)",
+        "description": "Classifies a customer service call transcript and extracts structured QA, sentiment, and outcome data as strict JSON.",
+        "version": {
+            "description": "classifier ready",
+            "model": "openai/gpt-5-mini",
+            "messages": [
+                {
+                    "role": "system",
+                    "content": "You are a call-analytics engine for a customer service team. You receive a full transcript of a recorded phone call between a customer and a support agent. Speaker turns are labeled (e.g., \"Agent:\", \"Customer:\").\n\nAnalyze the ENTIRE transcript and return a single structured classification. Follow these rules:\n\n# Classification rules\n- primary_reason: the main reason the customer called. Choose the single best-fit category.\n- resolution_status: judge whether the customer's issue was actually resolved by the end of the call, based on what was said - not on the agent's optimism.\n- customer_sentiment_start / customer_sentiment_end: assess the customer's tone at the beginning vs. the end of the call separately, so sentiment shifts are visible.\n- escalation_requested: true if the customer asked for a manager/supervisor, threatened to cancel, or mentioned legal action, regulators, or chargebacks.\n- followup_required: true if anything was promised but not completed on the call (a callback, an email, a shipment, an investigation).\n\n# Extraction rules\n- order_ids: extract every order ID mentioned (format like \"A-48217\"). Empty array if none.\n- products_mentioned: list distinct product names or models discussed.\n- agent_name: the agent's name if stated, otherwise null.\n- summary: 1-2 neutral sentences, under 40 words, describing the call and its outcome.\n- key_quote: the single most representative customer quote (verbatim, trimmed to under 25 words).\n\n# QA rules\n- agent_followed_greeting: true if the agent identified themselves and the company near the start.\n- agent_empathy_shown: true if the agent acknowledged the customer's frustration or thanked them for patience when warranted.\n- compliance_flags: list any of the allowed flags observed; empty array if none.\n\nBase every field strictly on the transcript. Do not infer facts that were not said."
+                },
+                {
+                    "role": "user",
+                    "content": "Classify the following customer service call transcript:\n\n{{call_transcript}}"
+                }
+            ],
+            "temperature": 0.7,
+            "max_tokens": 4096,
+            "top_p": 1,
+            "frequency_penalty": 0,
+            "presence_penalty": 0,
+            "stream": false,
+            "response_format": {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "call_classification",
+                    "schema": {
+                        "type": "object",
+                        "required": [
+                            "primary_reason",
+                            "resolution_status",
+                            "customer_sentiment_start",
+                            "customer_sentiment_end",
+                            "escalation_requested",
+                            "followup_required",
+                            "order_ids",
+                            "products_mentioned",
+                            "agent_name",
+                            "summary",
+                            "key_quote",
+                            "agent_followed_greeting",
+                            "agent_empathy_shown",
+                            "compliance_flags"
+                        ],
+                        "properties": {
+                            "summary": {
+                                "type": "string"
+                            },
+                            "key_quote": {
+                                "type": "string"
+                            },
+                            "order_ids": {
+                                "type": "array",
+                                "items": {
+                                    "type": "string"
+                                }
+                            },
+                            "agent_name": {
+                                "type": [
+                                    "string",
+                                    "null"
+                                ]
+                            },
+                            "primary_reason": {
+                                "enum": [
+                                    "shipping_delay",
+                                    "billing_dispute",
+                                    "product_defect",
+                                    "return_or_refund",
+                                    "technical_support",
+                                    "account_management",
+                                    "general_inquiry",
+                                    "other"
+                                ],
+                                "type": "string"
+                            },
+                            "compliance_flags": {
+                                "type": "array",
+                                "items": {
+                                    "enum": [
+                                        "no_identity_verification",
+                                        "promise_without_confirmation",
+                                        "rude_or_dismissive",
+                                        "shared_sensitive_data",
+                                        "no_resolution_offered"
+                                    ],
+                                    "type": "string"
+                                }
+                            },
+                            "followup_required": {
+                                "type": "boolean"
+                            },
+                            "resolution_status": {
+                                "enum": [
+                                    "resolved",
+                                    "partially_resolved",
+                                    "unresolved",
+                                    "escalated"
+                                ],
+                                "type": "string"
+                            },
+                            "products_mentioned": {
+                                "type": "array",
+                                "items": {
+                                    "type": "string"
+                                }
+                            },
+                            "agent_empathy_shown": {
+                                "type": "boolean"
+                            },
+                            "escalation_requested": {
+                                "type": "boolean"
+                            },
+                            "customer_sentiment_end": {
+                                "enum": [
+                                    "positive",
+                                    "neutral",
+                                    "frustrated",
+                                    "angry"
+                                ],
+                                "type": "string"
+                            },
+                            "agent_followed_greeting": {
+                                "type": "boolean"
+                            },
+                            "customer_sentiment_start": {
+                                "enum": [
+                                    "positive",
+                                    "neutral",
+                                    "frustrated",
+                                    "angry"
+                                ],
+                                "type": "string"
+                            }
+                        },
+                        "additionalProperties": false
+                    },
+                    "strict": true
+                }
+            },
+            "is_enforcing_response_format": true
+        }
+    }
+];

package/dist/lib/demo/seed-datasets.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Datasets seeder for `respan demo`.
+ *
+ * Creates the demo datasets (see dataset-fixtures.ts) and bulk-loads their rows,
+ * then tears them down for `respan demo clear`. Deduplicates by dataset name so
+ * re-runs never create duplicates. All dataset-specific logic lives here; the
+ * orchestrator stays service-agnostic.
+ *
+ * Rows are written via `bulkCreateDatasetLogs` (POST .../logs/bulk/), which is
+ * synchronous and returns success/error counts. Each row's `input` object becomes
+ * the row's mapped variables; `expected_output` is JSON-stringified so the grader
+ * sees valid JSON (a raw object is coerced to single-quoted python-repr).
+ */
+import { Seeder } from './types.js';
+export declare const datasetsSeeder: Seeder;

package/dist/lib/demo/seed-datasets.js

@@ -0,0 +1,107 @@
+/**
+ * Datasets seeder for `respan demo`.
+ *
+ * Creates the demo datasets (see dataset-fixtures.ts) and bulk-loads their rows,
+ * then tears them down for `respan demo clear`. Deduplicates by dataset name so
+ * re-runs never create duplicates. All dataset-specific logic lives here; the
+ * orchestrator stays service-agnostic.
+ *
+ * Rows are written via `bulkCreateDatasetLogs` (POST .../logs/bulk/), which is
+ * synchronous and returns success/error counts. Each row's `input` object becomes
+ * the row's mapped variables; `expected_output` is JSON-stringified so the grader
+ * sees valid JSON (a raw object is coerced to single-quoted python-repr).
+ */
+import { DEMO_DATASETS } from './dataset-fixtures.js';
+/** Datasets in the account whose name exactly matches a demo fixture name.
+ *
+ * Listed in full and matched client-side (case-sensitive, exact) rather than via
+ * server filters, which keeps dedup independent of filter-operator semantics.
+ * Returns every match so duplicates are all found (for clear). */
+async function findExistingDatasets(ctx) {
+    const wanted = new Set(DEMO_DATASETS.map((d) => d.name));
+    const response = await ctx.client.datasets.listDatasets({
+        Authorization: ctx.authHeader,
+        page_size: 100,
+    });
+    const found = [];
+    for (const row of response.results ?? []) {
+        const name = typeof row.name === 'string' ? row.name : '';
+        const id = String(row.id ?? row.dataset_id ?? '');
+        if (id && wanted.has(name))
+            found.push({ name, id });
+    }
+    return found;
+}
+/** Create one dataset and bulk-load its rows. Returns the new dataset id. */
+async function createFromFixture(ctx, fixture) {
+    const created = (await ctx.client.datasets.createDataset({
+        Authorization: ctx.authHeader,
+        name: fixture.name,
+        is_empty: true,
+        ...(fixture.description ? { description: fixture.description } : {}),
+    }));
+    const datasetId = String(created.id ?? created.dataset_id ?? '');
+    if (!datasetId) {
+        throw new Error(`createDataset did not return an id for "${fixture.name}".`);
+    }
+    const logs = fixture.rows.map((row) => ({
+        input: row.input,
+        ...(row.expected_output !== undefined
+            ? {
+                expected_output: typeof row.expected_output === 'string'
+                    ? row.expected_output
+                    : JSON.stringify(row.expected_output),
+            }
+            : {}),
+    }));
+    const result = (await ctx.client.datasets.bulkCreateDatasetLogs({
+        Authorization: ctx.authHeader,
+        dataset_id: datasetId,
+        logs,
+    }));
+    const errorCount = typeof result.error_count === 'number' ? result.error_count : 0;
+    if (errorCount > 0) {
+        const detail = Array.isArray(result.errors) ? `: ${JSON.stringify(result.errors).slice(0, 200)}` : '';
+        throw new Error(`bulkCreateDatasetLogs reported ${errorCount} error(s) for "${fixture.name}"${detail}`);
+    }
+    return datasetId;
+}
+export const datasetsSeeder = {
+    service: 'datasets',
+    label: 'Datasets',
+    findExisting(ctx) {
+        return findExistingDatasets(ctx);
+    },
+    async run(ctx) {
+        const existingNames = new Set((await findExistingDatasets(ctx)).map((d) => d.name));
+        const items = [];
+        for (const fixture of DEMO_DATASETS) {
+            if (existingNames.has(fixture.name)) {
+                items.push({ name: fixture.name, status: 'skipped', reason: 'already exists' });
+                continue;
+            }
+            const id = await createFromFixture(ctx, fixture);
+            items.push({ name: fixture.name, status: 'created', id });
+        }
+        return { service: this.service, label: this.label, items };
+    },
+    async clear(ctx) {
+        const existing = await findExistingDatasets(ctx);
+        const items = [];
+        for (const fixture of DEMO_DATASETS) {
+            const matches = existing.filter((d) => d.name === fixture.name);
+            if (matches.length === 0) {
+                items.push({ name: fixture.name, status: 'skipped', reason: 'not found' });
+                continue;
+            }
+            for (const match of matches) {
+                await ctx.client.datasets.deleteDataset({
+                    Authorization: ctx.authHeader,
+                    dataset_id: match.id,
+                });
+                items.push({ name: match.name, status: 'deleted', id: match.id });
+            }
+        }
+        return { service: this.service, label: this.label, items };
+    },
+};

package/dist/lib/demo/seed-evaluators.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Evaluators seeder for `respan demo`.
+ *
+ * Recreates the demo evaluators (see evaluator-fixtures.ts) in the authenticated
+ * account and tears them down for `respan demo clear`. Each evaluator is two kinds
+ * of resource: grader record(s) and a Blockly workflow that references them.
+ *
+ * Recreate recipe (the engine — LLM vs human — lives in the workflow node, not the
+ * grader record, so the captured graph is replayed verbatim):
+ *   1. create each grader (or reuse one with the same name)
+ *   2. map each grader's captured `originalId` to its new id
+ *   3. deep-clone the task graph and rewrite every eval task's `config.evaluator_id`
+ *      via that map (task ids are kept verbatim so the compute task's
+ *      `state.<task_id>` input references stay valid)
+ *   4. createWorkflow -> commit -> deploy v1
+ *
+ * Dedup is by workflow name (the user-facing evaluator name). The SDK has no commit
+ * method, so commit is a raw POST to `/api/workflows/{id}/commits/` (as in
+ * lib/eval/runner.ts). All evaluator-specific logic lives here.
+ */
+import { Seeder } from './types.js';
+export declare const evaluatorsSeeder: Seeder;