@respan/cli 0.11.0 → 0.12.0

package/dist/commands/demo/clear.d.ts

@@ -0,0 +1,23 @@
+import { BaseCommand } from '../../lib/base-command.js';
+/**
+ * Tear down what `respan demo` created. Each seeder owns how it identifies its own
+ * demo resources — prompts by exact fixture name, traces by the `demo` workflow —
+ * so a resource that was manually renamed/relabeled may not be matched, the same
+ * identity contract used when seeding.
+ */
+export default class DemoClear extends BaseCommand {
+    static description: string;
+    static examples: string[];
+    static flags: {
+        yes: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        'api-key': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'base-url': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        profile: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        json: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        csv: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        verbose: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+    };
+    run(): Promise<void>;
+    private previewDeletions;
+    private printSummary;
+}

package/dist/commands/demo/clear.js

@@ -0,0 +1,114 @@
+import { Flags } from '@oclif/core';
+import { confirm } from '@inquirer/prompts';
+import { BaseCommand } from '../../lib/base-command.js';
+import { getActiveProfile, getCredential } from '../../lib/config.js';
+import { GREEN, DIM, BOLD, RESET } from '../../lib/colors.js';
+import { SEEDERS } from '../../lib/demo/seeders.js';
+/**
+ * Tear down what `respan demo` created. Each seeder owns how it identifies its own
+ * demo resources — prompts by exact fixture name, traces by the `demo` workflow —
+ * so a resource that was manually renamed/relabeled may not be matched, the same
+ * identity contract used when seeding.
+ */
+class DemoClear extends BaseCommand {
+    async run() {
+        const { flags } = await this.parse(DemoClear);
+        this.globalFlags = flags;
+        const profile = flags.profile || getActiveProfile();
+        if (!getCredential(profile) && !flags['api-key']) {
+            this.error('Not authenticated. Run `respan auth login` (or set RESPAN_API_KEY) before running `respan demo clear`.', { exit: 1 });
+        }
+        try {
+            const client = this.getClient();
+            const ctx = {
+                client,
+                authHeader: this.getAuthHeader(),
+                baseUrl: this.getBaseUrl(),
+            };
+            // Preview what exists before deleting anything.
+            const existing = [];
+            for (const seeder of SEEDERS) {
+                const matches = await this.spin(`Finding ${seeder.label}`, () => seeder.findExisting(ctx));
+                existing.push({ label: seeder.label, matches });
+            }
+            const total = existing.reduce((n, e) => n + e.matches.length, 0);
+            if (total === 0) {
+                this.log('');
+                this.log(`Nothing to clear in account ${DIM}(${profile})${RESET}.`);
+                this.log('');
+                return;
+            }
+            if (!flags.yes && !flags.json) {
+                this.previewDeletions(profile, existing);
+                if (!process.stdin.isTTY) {
+                    this.error('Refusing to delete without confirmation. Re-run with --yes.', { exit: 1 });
+                }
+                const proceed = await confirm({
+                    message: `Delete ${total} demo resource(s) from account "${profile}"?`,
+                    default: false,
+                });
+                if (!proceed) {
+                    this.log('Aborted. Nothing was deleted.');
+                    return;
+                }
+            }
+            const results = [];
+            for (const seeder of SEEDERS) {
+                const result = await this.spin(`Clearing ${seeder.label}`, () => seeder.clear(ctx));
+                results.push(result);
+            }
+            this.printSummary(profile, results);
+        }
+        catch (error) {
+            this.handleError(error);
+        }
+    }
+    previewDeletions(profile, existing) {
+        this.log('');
+        this.log(`${BOLD}respan demo clear${RESET} will delete from account ${DIM}(${profile})${RESET}:`);
+        for (const { label, matches } of existing) {
+            if (matches.length === 0)
+                continue;
+            this.log('');
+            this.log(`  ${BOLD}${label}${RESET}`);
+            for (const m of matches) {
+                this.log(`  ${DIM}- ${m.name}  (id: ${m.id})${RESET}`);
+            }
+        }
+        this.log('');
+    }
+    printSummary(profile, results) {
+        if (this.globalFlags.json) {
+            this.outputResult(results);
+            return;
+        }
+        this.log('');
+        this.log(`${GREEN}✓${RESET} ${BOLD}respan demo clear${RESET} — account ${DIM}(${profile})${RESET}`);
+        for (const result of results) {
+            if (result.items.length === 0)
+                continue; // e.g. traces, which are never deleted
+            this.log('');
+            this.log(`  ${BOLD}${result.label}${RESET}`);
+            for (const item of result.items) {
+                if (item.status === 'deleted') {
+                    this.log(`  ${GREEN}Deleted${RESET}   ${item.name}  ${DIM}(id: ${item.id})${RESET}`);
+                }
+                else {
+                    this.log(`  ${DIM}Skipped   ${item.name}  (${item.reason})${RESET}`);
+                }
+            }
+        }
+        const deletedCount = results.reduce((n, r) => n + r.items.filter((i) => i.status === 'deleted').length, 0);
+        const skippedCount = results.reduce((n, r) => n + r.items.filter((i) => i.status === 'skipped').length, 0);
+        this.log('');
+        this.log(`  ${DIM}${deletedCount} deleted, ${skippedCount} not found${RESET}`);
+        this.log('');
+    }
+}
+DemoClear.description = 'Delete the sample prompts, datasets, and evaluators created by `respan demo`. Demo traces are left in place (each run is timestamped history). Idempotent — safe to re-run.';
+DemoClear.examples = ['<%= config.bin %> demo clear', '<%= config.bin %> demo clear --yes'];
+DemoClear.flags = {
+    ...BaseCommand.baseFlags,
+    yes: Flags.boolean({ char: 'y', description: 'Skip the confirmation prompt', default: false }),
+};
+export default DemoClear;

package/dist/commands/demo/index.d.ts

@@ -0,0 +1,15 @@
+import { BaseCommand } from '../../lib/base-command.js';
+export default class Demo extends BaseCommand {
+    static description: string;
+    static examples: string[];
+    static flags: {
+        'api-key': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'base-url': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        profile: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        json: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        csv: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        verbose: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+    };
+    run(): Promise<void>;
+    private printSummary;
+}

package/dist/commands/demo/index.js

@@ -0,0 +1,81 @@
+import { BaseCommand } from '../../lib/base-command.js';
+import { getActiveProfile, getCredential } from '../../lib/config.js';
+import { GREEN, DIM, BOLD, RESET } from '../../lib/colors.js';
+import { SEEDERS } from '../../lib/demo/seeders.js';
+const PLATFORM_BASE = 'https://platform.respan.ai/platform';
+/** Dashboard link for a seeded service, deep-linked/filtered where possible so the
+ * user lands directly on what was just created. Traces are pre-filtered to the
+ * `demo` workflow so they're isolated from any existing traffic. */
+function viewLink(service) {
+    switch (service) {
+        case 'prompts':
+            return `${PLATFORM_BASE}/prompts`;
+        case 'datasets':
+            return `${PLATFORM_BASE}/datasets`;
+        case 'evaluators':
+            return `${PLATFORM_BASE}/evaluators`;
+        case 'traces':
+            return `${PLATFORM_BASE}/traces?span_workflow_name=demo`;
+        default:
+            return undefined;
+    }
+}
+class Demo extends BaseCommand {
+    async run() {
+        const { flags } = await this.parse(Demo);
+        this.globalFlags = flags;
+        const profile = flags.profile || getActiveProfile();
+        if (!getCredential(profile) && !flags['api-key']) {
+            this.error('Not authenticated. Run `respan auth login` (or set RESPAN_API_KEY) before running `respan demo`.', { exit: 1 });
+        }
+        try {
+            const client = this.getClient();
+            const ctx = {
+                client,
+                authHeader: this.getAuthHeader(),
+                baseUrl: this.getBaseUrl(),
+            };
+            const results = [];
+            for (const seeder of SEEDERS) {
+                const result = await this.spin(`Seeding ${seeder.label}`, () => seeder.run(ctx));
+                results.push(result);
+            }
+            this.printSummary(profile, results);
+        }
+        catch (error) {
+            this.handleError(error);
+        }
+    }
+    printSummary(profile, results) {
+        if (this.globalFlags.json) {
+            this.outputResult(results);
+            return;
+        }
+        this.log('');
+        this.log(`${GREEN}✓${RESET} ${BOLD}respan demo${RESET} — seeded into account ${DIM}(${profile})${RESET}`);
+        for (const result of results) {
+            this.log('');
+            this.log(`  ${BOLD}${result.label}${RESET}`);
+            for (const item of result.items) {
+                if (item.status === 'created') {
+                    this.log(`  ${GREEN}Created${RESET}   ${item.name}  ${DIM}(id: ${item.id})${RESET}`);
+                }
+                else {
+                    this.log(`  ${DIM}Skipped   ${item.name}  (${item.reason})${RESET}`);
+                }
+            }
+            const link = viewLink(result.service);
+            if (link)
+                this.log(`  ${DIM}View:${RESET} ${link}`);
+        }
+        const createdCount = results.reduce((n, r) => n + r.items.filter((i) => i.status === 'created').length, 0);
+        const skippedCount = results.reduce((n, r) => n + r.items.filter((i) => i.status === 'skipped').length, 0);
+        this.log('');
+        this.log(`  ${DIM}${createdCount} created, ${skippedCount} skipped${RESET}`);
+        this.log('');
+    }
+}
+Demo.description = 'Seed your account with sample data (prompts and traces) so you have something to explore. Idempotent — safe to re-run.';
+Demo.examples = ['<%= config.bin %> demo'];
+Demo.flags = { ...BaseCommand.baseFlags };
+export default Demo;

package/dist/lib/demo/dataset-fixtures.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Frozen demo datasets for `respan demo`, hand-authored as typed TypeScript.
+ *
+ * Two datasets, numbered to match the demo prompt/evaluator they pair with:
+ *   - "Demo dataset 1: support messages"  -> prompt 1 / evaluator 1
+ *       Rows are customer messages that probe Lumen's refund/escalation policy
+ *       edges. No `expected_output` — evaluator 1 scores compliance from the
+ *       reply alone (it uses {{input}}/{{output}} only).
+ *   - "Demo dataset 3: call transcripts"  -> prompt 3 / evaluator 3
+ *       Rows are call transcripts, each with a gold-label classification in
+ *       `expected_output` (evaluator 3 compares against {{expected_output}}).
+ *
+ * `input` is an object of the prompt's template variables (`customer_message` /
+ * `call_transcript`); it is stored stringified and parsed into the row's mapped
+ * fields. `expected_output` is authored here as an object for readability and
+ * JSON-stringified at seed time — passing a raw object makes the platform coerce
+ * it to single-quoted python-repr (invalid JSON the grader would then see).
+ */
+/** One dataset row: template variables plus an optional gold label. */
+export interface DemoDatasetRow {
+    /** The prompt's template variables for this row. */
+    input: Record<string, unknown>;
+    /** Gold label (object; stringified to JSON at seed time). */
+    expected_output?: unknown;
+}
+/** One demo dataset: name/description plus its rows. */
+export interface DemoDatasetFixture {
+    name: string;
+    description?: string;
+    rows: DemoDatasetRow[];
+}
+export declare const DEMO_DATASETS: DemoDatasetFixture[];

package/dist/lib/demo/dataset-fixtures.js

@@ -0,0 +1,194 @@
+/**
+ * Frozen demo datasets for `respan demo`, hand-authored as typed TypeScript.
+ *
+ * Two datasets, numbered to match the demo prompt/evaluator they pair with:
+ *   - "Demo dataset 1: support messages"  -> prompt 1 / evaluator 1
+ *       Rows are customer messages that probe Lumen's refund/escalation policy
+ *       edges. No `expected_output` — evaluator 1 scores compliance from the
+ *       reply alone (it uses {{input}}/{{output}} only).
+ *   - "Demo dataset 3: call transcripts"  -> prompt 3 / evaluator 3
+ *       Rows are call transcripts, each with a gold-label classification in
+ *       `expected_output` (evaluator 3 compares against {{expected_output}}).
+ *
+ * `input` is an object of the prompt's template variables (`customer_message` /
+ * `call_transcript`); it is stored stringified and parsed into the row's mapped
+ * fields. `expected_output` is authored here as an object for readability and
+ * JSON-stringified at seed time — passing a raw object makes the platform coerce
+ * it to single-quoted python-repr (invalid JSON the grader would then see).
+ */
+export const DEMO_DATASETS = [
+    {
+        name: 'Demo dataset 1: support messages',
+        description: 'Customer messages that probe Lumen Electronics refund/escalation policy edges, for evaluating the support agent (prompt 1).',
+        rows: [
+            {
+                input: {
+                    customer_message: "Hi, my wireless earbuds (order A-50231) arrived two days ago but the left one won't charge. I'd like a refund, please.",
+                },
+            },
+            {
+                input: {
+                    customer_message: "I bought the clearance Lumen soundbar (order A-49120, it was marked final sale) last week and I've changed my mind. I'd like my money back.",
+                },
+            },
+            {
+                input: {
+                    customer_message: "Order A-51002 was supposed to arrive yesterday and it's still not here. I want a refund for the delay.",
+                },
+            },
+            {
+                input: {
+                    customer_message: 'Tracking says my package (A-48817) was delivered three days ago, but I never received it. I want a full refund.',
+                },
+            },
+            {
+                input: {
+                    customer_message: 'Please process a refund on order A-52340 for $349.99.',
+                },
+            },
+            {
+                input: {
+                    customer_message: "My Lumen 4K monitor (order A-53001) stopped working after two days — it was $279. I'd like a refund because it's defective.",
+                },
+            },
+            {
+                input: {
+                    customer_message: "Where is my order A-50777? I just want to know when it'll arrive.",
+                },
+            },
+        ],
+    },
+    {
+        name: 'Demo dataset 3: call transcripts',
+        description: 'Support call transcripts with gold-label classifications, for evaluating the call-transcript classifier (prompt 3).',
+        rows: [
+            {
+                input: {
+                    call_transcript: 'Agent: Hi, this is Dana at Lumen Electronics support. How can I help you today?\n' +
+                        "Customer: My monitor, order A-51440, arrived with a cracked panel. I'm pretty annoyed.\n" +
+                        "Agent: I'm really sorry about that. I've created a replacement order — it ships today and you'll get tracking by email.\n" +
+                        'Customer: Oh, great. Thank you, that was easy.\n' +
+                        'Agent: My pleasure. Have a great day.',
+                },
+                expected_output: {
+                    primary_reason: 'product_defect',
+                    resolution_status: 'resolved',
+                    customer_sentiment_start: 'frustrated',
+                    customer_sentiment_end: 'positive',
+                    escalation_requested: false,
+                    followup_required: true,
+                    order_ids: ['A-51440'],
+                    products_mentioned: ['Lumen Monitor'],
+                    agent_name: 'Dana',
+                    summary: 'Customer reported a cracked monitor panel (A-51440); agent created a same-day replacement shipment.',
+                    key_quote: 'My monitor, order A-51440, arrived with a cracked panel.',
+                    agent_followed_greeting: true,
+                    agent_empathy_shown: true,
+                    compliance_flags: [],
+                },
+            },
+            {
+                input: {
+                    call_transcript: 'Agent: Thanks for calling Lumen Electronics, this is Marcus.\n' +
+                        "Customer: I was charged twice for order A-52210 and nobody's fixed it. I want a manager, now.\n" +
+                        "Agent: I can see two charges. I'm not able to reverse it from my side today.\n" +
+                        "Customer: Unacceptable. Escalate this or I'm calling my bank for a chargeback.\n" +
+                        "Agent: I'll escalate to a supervisor who will call you back within 24 hours.\n" +
+                        'Customer: Fine.',
+                },
+                expected_output: {
+                    primary_reason: 'billing_dispute',
+                    resolution_status: 'escalated',
+                    customer_sentiment_start: 'angry',
+                    customer_sentiment_end: 'frustrated',
+                    escalation_requested: true,
+                    followup_required: true,
+                    order_ids: ['A-52210'],
+                    products_mentioned: [],
+                    agent_name: 'Marcus',
+                    summary: 'Customer disputed a double charge on A-52210 and demanded a manager; agent escalated to a supervisor with a 24-hour callback.',
+                    key_quote: "I was charged twice for order A-52210 and nobody's fixed it.",
+                    agent_followed_greeting: true,
+                    agent_empathy_shown: false,
+                    compliance_flags: ['no_identity_verification'],
+                },
+            },
+            {
+                input: {
+                    call_transcript: 'Agent: Hi, my name is Max and this is the customer support line for Lumen Electronics. How can I help you today?\n' +
+                        'Customer: Finally. I ordered the Lumen Wireless Headphones almost two weeks ago, order A-48217, and they still have not shown up. This is ridiculous.\n' +
+                        "Agent: I'm really sorry about the wait, and I understand how frustrating that is. It looks like A-48217 was delayed at the carrier's sorting facility. I've flagged it and I'll email you an update within 24 hours.\n" +
+                        "Customer: Okay, I'd appreciate that.\n" +
+                        'Agent: Thank you for your patience.',
+                },
+                expected_output: {
+                    primary_reason: 'shipping_delay',
+                    resolution_status: 'partially_resolved',
+                    customer_sentiment_start: 'frustrated',
+                    customer_sentiment_end: 'neutral',
+                    escalation_requested: false,
+                    followup_required: true,
+                    order_ids: ['A-48217'],
+                    products_mentioned: ['Lumen Wireless Headphones'],
+                    agent_name: 'Max',
+                    summary: 'Customer reported headphones order A-48217 was delayed; agent identified a carrier delay and promised an emailed update within 24 hours.',
+                    key_quote: 'I ordered the Lumen Wireless Headphones almost two weeks ago and they still have not shown up.',
+                    agent_followed_greeting: true,
+                    agent_empathy_shown: true,
+                    compliance_flags: [],
+                },
+            },
+            {
+                input: {
+                    call_transcript: "Agent: Hello, you've reached Lumen Electronics, this is Priya.\n" +
+                        "Customer: Hi, I'd like to return my speaker, order A-50988. It works fine, I just don't need it.\n" +
+                        'Agent: No problem. It is within our 30-day window, so I have issued a full refund of $59.99 and emailed a prepaid return label.\n' +
+                        'Customer: Perfect, thanks so much.\n' +
+                        "Agent: You're welcome!",
+                },
+                expected_output: {
+                    primary_reason: 'return_or_refund',
+                    resolution_status: 'resolved',
+                    customer_sentiment_start: 'neutral',
+                    customer_sentiment_end: 'positive',
+                    escalation_requested: false,
+                    followup_required: true,
+                    order_ids: ['A-50988'],
+                    products_mentioned: ['Lumen Speaker'],
+                    agent_name: 'Priya',
+                    summary: 'Customer requested a return for unwanted speaker A-50988; agent issued a full refund and sent a prepaid return label.',
+                    key_quote: "I'd like to return my speaker, order A-50988. It works fine, I just don't need it.",
+                    agent_followed_greeting: true,
+                    agent_empathy_shown: false,
+                    compliance_flags: [],
+                },
+            },
+            {
+                input: {
+                    call_transcript: "Customer: My Lumen smart hub won't connect to wifi no matter what I try.\n" +
+                        'Agent: Have you tried resetting it? Hold the button for ten seconds.\n' +
+                        'Customer: Yes, three times. Still nothing.\n' +
+                        "Agent: I'm not sure what else to suggest. I'll open a ticket and someone from tech will email you.\n" +
+                        'Customer: So you can’t actually help me right now?\n' +
+                        'Agent: Not today, sorry.',
+                },
+                expected_output: {
+                    primary_reason: 'technical_support',
+                    resolution_status: 'unresolved',
+                    customer_sentiment_start: 'frustrated',
+                    customer_sentiment_end: 'frustrated',
+                    escalation_requested: false,
+                    followup_required: true,
+                    order_ids: [],
+                    products_mentioned: ['Lumen Smart Hub'],
+                    agent_name: null,
+                    summary: 'Customer’s smart hub would not connect to wifi; a basic reset failed and the agent opened a ticket for tech to follow up by email.',
+                    key_quote: "My Lumen smart hub won't connect to wifi no matter what I try.",
+                    agent_followed_greeting: false,
+                    agent_empathy_shown: false,
+                    compliance_flags: ['no_identity_verification', 'no_resolution_offered'],
+                },
+            },
+        ],
+    },
+];

package/dist/lib/demo/evaluator-fixtures.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Frozen snapshot of the demo evaluators, captured byte-for-byte from the
+ * owner's deployed evaluators (graders + Blockly workflow task graphs).
+ *
+ * Source of truth for `respan demo`. Each entry is one user-facing evaluator: the
+ * deployed workflow's task graph plus the grader record(s) it references. The
+ * grader `type` is captured verbatim ("human" is the stored default) — the actual
+ * LLM engine lives in each eval task's `generation_method`/`_blockly_evaluator_kind`,
+ * so replaying the graph reproduces the verified scoring behavior exactly.
+ *
+ * This is a frozen snapshot: if the source evaluators change, this file does NOT
+ * update until someone re-snapshots and ships a new build.
+ *
+ * Recreate recipe (see seed-evaluators.ts): create each grader -> map its
+ * `originalId` to the new id -> rewrite every eval task's `config.evaluator_id`
+ * via that map -> createWorkflow -> commit -> deploy. Task ids are kept verbatim
+ * so the compute task's `state.<task_id>` input references stay valid.
+ */
+/** A grader record referenced by an evaluator's workflow. */
+export interface DemoEvaluatorGrader {
+    name: string;
+    /** The grader's id inside the captured task graph. Remapped to the freshly
+     * created grader id at seed time so the eval tasks point at the new record. */
+    originalId: string;
+    type: string;
+    score_value_type: string;
+    score_config: Record<string, unknown>;
+    passing_conditions: Record<string, unknown>;
+    llm_config: Record<string, unknown>;
+}
+/** One demo evaluator: the deployed workflow graph plus its grader(s). */
+export interface DemoEvaluatorFixture {
+    /** Workflow name (the user-facing evaluator name); used for dedup. */
+    workflowName: string;
+    description?: string;
+    graders: DemoEvaluatorGrader[];
+    /** Verbatim deployed workflow task graph. Only each eval task's
+     * `config.evaluator_id` is rewritten at seed time. */
+    tasks: Record<string, unknown>[];
+}
+export declare const DEMO_EVALUATORS: DemoEvaluatorFixture[];