@respan/cli 0.11.0 → 0.12.0

package/dist/lib/demo/seed-evaluators.js

@@ -0,0 +1,180 @@
+/**
+ * Evaluators seeder for `respan demo`.
+ *
+ * Recreates the demo evaluators (see evaluator-fixtures.ts) in the authenticated
+ * account and tears them down for `respan demo clear`. Each evaluator is two kinds
+ * of resource: grader record(s) and a Blockly workflow that references them.
+ *
+ * Recreate recipe (the engine — LLM vs human — lives in the workflow node, not the
+ * grader record, so the captured graph is replayed verbatim):
+ *   1. create each grader (or reuse one with the same name)
+ *   2. map each grader's captured `originalId` to its new id
+ *   3. deep-clone the task graph and rewrite every eval task's `config.evaluator_id`
+ *      via that map (task ids are kept verbatim so the compute task's
+ *      `state.<task_id>` input references stay valid)
+ *   4. createWorkflow -> commit -> deploy v1
+ *
+ * Dedup is by workflow name (the user-facing evaluator name). The SDK has no commit
+ * method, so commit is a raw POST to `/api/workflows/{id}/commits/` (as in
+ * lib/eval/runner.ts). All evaluator-specific logic lives here.
+ */
+import { DEMO_EVALUATORS } from './evaluator-fixtures.js';
+const WORKFLOW_NAMES = new Set(DEMO_EVALUATORS.map((e) => e.workflowName));
+const GRADER_NAMES = new Set(DEMO_EVALUATORS.flatMap((e) => e.graders.map((g) => g.name)));
+/** Demo evaluator workflows present in the account, matched by exact name. */
+async function findExistingWorkflows(ctx) {
+    const response = await ctx.client.workflows.listWorkflows({
+        Authorization: ctx.authHeader,
+        page_size: 100,
+    });
+    const found = [];
+    for (const row of response.results ?? []) {
+        const name = typeof row.name === 'string' ? row.name : '';
+        const id = String(row.workflow_id ?? row.id ?? '');
+        if (id && WORKFLOW_NAMES.has(name))
+            found.push({ name, id });
+    }
+    return found;
+}
+/** Demo grader records present in the account, matched by exact name. */
+async function findExistingGraders(ctx) {
+    const response = await ctx.client.evaluators.listEvaluators({ Authorization: ctx.authHeader });
+    const found = [];
+    for (const row of response.results ?? []) {
+        const name = typeof row.name === 'string' ? row.name : '';
+        const id = String(row.id ?? '');
+        if (id && GRADER_NAMES.has(name))
+            found.push({ name, id });
+    }
+    return found;
+}
+/** Create each grader the evaluator needs (reusing one of the same name if it
+ * already exists) and return a map from captured `originalId` to the live id. */
+async function ensureGraders(ctx, fixture, existingByName) {
+    const idMap = new Map();
+    for (const grader of fixture.graders) {
+        let id = existingByName.get(grader.name);
+        if (!id) {
+            const created = (await ctx.client.evaluators.createEvaluator({
+                Authorization: ctx.authHeader,
+                name: grader.name,
+                type: grader.type,
+                score_value_type: grader.score_value_type,
+                score_config: grader.score_config,
+                passing_conditions: grader.passing_conditions,
+                llm_config: grader.llm_config,
+            }));
+            id = String(created.id ?? created.evaluator_id ?? '');
+            if (!id)
+                throw new Error(`createEvaluator did not return an id for grader "${grader.name}".`);
+            existingByName.set(grader.name, id);
+        }
+        idMap.set(grader.originalId, id);
+    }
+    return idMap;
+}
+/** Deep-clone the captured task graph and point every eval task at the live grader
+ * id. Tasks without an `evaluator_id` (transform/compute) pass through unchanged. */
+function remapTasks(tasks, idMap) {
+    const clone = JSON.parse(JSON.stringify(tasks));
+    for (const task of clone) {
+        const config = task.config;
+        const original = config?.evaluator_id;
+        if (typeof original === 'string') {
+            const live = idMap.get(original);
+            if (!live)
+                throw new Error(`task references unknown grader id ${original}`);
+            config.evaluator_id = live;
+        }
+    }
+    return clone;
+}
+function normalizeBaseUrl(baseUrl) {
+    return baseUrl.replace(/\/+$/, '').replace(/\/api$/, '');
+}
+/** Commit the workflow's draft (no SDK method — raw POST, as in runner.ts). */
+async function commitWorkflow(ctx, workflowId) {
+    const url = `${normalizeBaseUrl(ctx.baseUrl)}/api/workflows/${encodeURIComponent(workflowId)}/commits/`;
+    const response = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', Authorization: ctx.authHeader },
+        body: JSON.stringify({}),
+    });
+    if (!response.ok) {
+        const text = await response.text().catch(() => '');
+        throw new Error(`commitWorkflow ${response.status}: ${text || response.statusText}`);
+    }
+}
+/** Create the grader(s) and the deployed workflow for one evaluator. */
+async function createFromFixture(ctx, fixture, existingGradersByName) {
+    const idMap = await ensureGraders(ctx, fixture, existingGradersByName);
+    const tasks = remapTasks(fixture.tasks, idMap);
+    const created = (await ctx.client.workflows.createWorkflow({
+        Authorization: ctx.authHeader,
+        type: 'evaluators',
+        trigger_event_type: 'eval_only',
+        name: fixture.workflowName,
+        tasks: tasks,
+    }));
+    const workflowId = String(created.workflow_id ?? created.id ?? '');
+    if (!workflowId)
+        throw new Error(`createWorkflow did not return an id for "${fixture.workflowName}".`);
+    await commitWorkflow(ctx, workflowId);
+    await ctx.client.workflows.deployWorkflow({
+        Authorization: ctx.authHeader,
+        workflow_id: workflowId,
+        version: 1,
+    });
+    return workflowId;
+}
+export const evaluatorsSeeder = {
+    service: 'evaluators',
+    label: 'Evaluators',
+    /** Both workflows and graders, so the clear preview/teardown covers every
+     * resource the seeder owns. */
+    async findExisting(ctx) {
+        const [workflows, graders] = await Promise.all([
+            findExistingWorkflows(ctx),
+            findExistingGraders(ctx),
+        ]);
+        return [...workflows, ...graders];
+    },
+    async run(ctx) {
+        const existingWorkflows = new Set((await findExistingWorkflows(ctx)).map((w) => w.name));
+        const existingGradersByName = new Map((await findExistingGraders(ctx)).map((g) => [g.name, g.id]));
+        const items = [];
+        for (const fixture of DEMO_EVALUATORS) {
+            if (existingWorkflows.has(fixture.workflowName)) {
+                items.push({ name: fixture.workflowName, status: 'skipped', reason: 'already exists' });
+                continue;
+            }
+            const id = await createFromFixture(ctx, fixture, existingGradersByName);
+            items.push({ name: fixture.workflowName, status: 'created', id });
+        }
+        return { service: this.service, label: this.label, items };
+    },
+    async clear(ctx) {
+        const items = [];
+        // Delete workflows first, then the grader records they referenced.
+        for (const match of await findExistingWorkflows(ctx)) {
+            await ctx.client.workflows.deleteWorkflow({
+                Authorization: ctx.authHeader,
+                workflow_id: match.id,
+            });
+            items.push({ name: match.name, status: 'deleted', id: match.id });
+        }
+        for (const match of await findExistingGraders(ctx)) {
+            await ctx.client.evaluators.deleteEvaluator({
+                Authorization: ctx.authHeader,
+                evaluator_id: match.id,
+            });
+            items.push({ name: match.name, status: 'deleted', id: match.id });
+        }
+        if (items.length === 0) {
+            for (const fixture of DEMO_EVALUATORS) {
+                items.push({ name: fixture.workflowName, status: 'skipped', reason: 'not found' });
+            }
+        }
+        return { service: this.service, label: this.label, items };
+    },
+};

package/dist/lib/demo/seed-prompts.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Prompts seeder for `respan demo`.
+ *
+ * Recreates the three demo prompts (name + single deployed live version) in the
+ * authenticated account, and tears them down again for `respan demo clear`.
+ * Deduplicates by prompt name so re-runs never create duplicates. All
+ * prompt-specific logic lives here; the orchestrator stays service-agnostic.
+ */
+import { Seeder } from './types.js';
+export declare const promptsSeeder: Seeder;

package/dist/lib/demo/seed-prompts.js

@@ -0,0 +1,111 @@
+/**
+ * Prompts seeder for `respan demo`.
+ *
+ * Recreates the three demo prompts (name + single deployed live version) in the
+ * authenticated account, and tears them down again for `respan demo clear`.
+ * Deduplicates by prompt name so re-runs never create duplicates. All
+ * prompt-specific logic lives here; the orchestrator stays service-agnostic.
+ */
+import { DEMO_PROMPTS } from './prompt-fixtures.js';
+/** Prompts in the account whose name exactly matches a demo fixture name.
+ *
+ * Queried with the `name` `iexact` filter, one query per fixture name. NOTE: the
+ * `icontains`/`iexact` filters do NOT OR-match across a multi-value array — a
+ * single `value: [a, b, c]` matches nothing — so each name must be queried on
+ * its own. Server matching is case-insensitive, so we re-confirm with an exact,
+ * case-sensitive match client-side. Returns every match (so duplicates are all
+ * found, for clear). */
+async function findExistingPrompts(ctx) {
+    const found = [];
+    for (const fixture of DEMO_PROMPTS) {
+        const response = await ctx.client.prompts.listPrompts({
+            Authorization: ctx.authHeader,
+            page_size: 100,
+            filters: { name: { operator: 'iexact', value: [fixture.name] } },
+        });
+        for (const row of response.results ?? []) {
+            if (row.id && row.name === fixture.name) {
+                found.push({ name: row.name, id: row.id });
+            }
+        }
+    }
+    return found;
+}
+/** Create one prompt, then commit and deploy its version so it is the live
+ * version immediately.
+ *
+ * The management API treats a freshly created version as an uncommitted draft:
+ * `deploy: true` on `createPromptVersion` is a no-op for a draft, and deploying
+ * one directly fails ("Cannot deploy a draft version. Commit it first."). The
+ * working sequence is create → commit → deploy. (Committing opens a fresh draft
+ * on top of the committed version — the same editing model the platform uses
+ * for hand-edited prompts.) */
+async function createFromFixture(ctx, fixture) {
+    const prompt = await ctx.client.prompts.createPrompt({
+        Authorization: ctx.authHeader,
+        name: fixture.name,
+        ...(fixture.description ? { description: fixture.description } : {}),
+    });
+    const promptId = prompt.id;
+    if (!promptId) {
+        throw new Error(`createPrompt did not return an id for "${fixture.name}".`);
+    }
+    const version = await ctx.client.prompts.createPromptVersion({
+        Authorization: ctx.authHeader,
+        prompt_id: promptId,
+        ...fixture.version,
+    });
+    const versionNumber = version.version;
+    if (versionNumber == null) {
+        throw new Error(`createPromptVersion did not return a version number for "${fixture.name}".`);
+    }
+    await ctx.client.prompts.commitPromptVersion({
+        Authorization: ctx.authHeader,
+        prompt_id: promptId,
+    });
+    await ctx.client.prompts.deployPromptVersion({
+        Authorization: ctx.authHeader,
+        prompt_id: promptId,
+        version: versionNumber,
+    });
+    return promptId;
+}
+export const promptsSeeder = {
+    service: 'prompts',
+    label: 'Prompts',
+    findExisting(ctx) {
+        return findExistingPrompts(ctx);
+    },
+    async run(ctx) {
+        const existingNames = new Set((await findExistingPrompts(ctx)).map((p) => p.name));
+        const items = [];
+        for (const fixture of DEMO_PROMPTS) {
+            if (existingNames.has(fixture.name)) {
+                items.push({ name: fixture.name, status: 'skipped', reason: 'already exists' });
+                continue;
+            }
+            const id = await createFromFixture(ctx, fixture);
+            items.push({ name: fixture.name, status: 'created', id });
+        }
+        return { service: this.service, label: this.label, items };
+    },
+    async clear(ctx) {
+        const existing = await findExistingPrompts(ctx);
+        const items = [];
+        for (const fixture of DEMO_PROMPTS) {
+            const matches = existing.filter((p) => p.name === fixture.name);
+            if (matches.length === 0) {
+                items.push({ name: fixture.name, status: 'skipped', reason: 'not found' });
+                continue;
+            }
+            for (const match of matches) {
+                await ctx.client.prompts.deletePrompt({
+                    Authorization: ctx.authHeader,
+                    prompt_id: match.id,
+                });
+                items.push({ name: match.name, status: 'deleted', id: match.id });
+            }
+        }
+        return { service: this.service, label: this.label, items };
+    },
+};

package/dist/lib/demo/seed-traces.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Traces seeder for `respan demo`.
+ *
+ * Ingests the supplied multi-agent service-desk traces (see trace-fixtures.ts)
+ * into the authenticated account. All trace-specific logic lives here; the
+ * orchestrator stays service-agnostic.
+ *
+ * Traces are treated as append-only run history: every `respan demo` run ingests a
+ * fresh batch stamped at the current time, with freshly-generated trace/span ids.
+ * Trace id + timestamps differ on every run, so runs never collide — there is no
+ * dedup-skip and no teardown (`respan demo clear` leaves traces in place).
+ *
+ * Two workflow identifiers, on purpose:
+ *   - `traceloop.workflow.name` is set to the shared constant `demo`
+ *     (`span_workflow_name`, a filterable field) on every span, so one marker
+ *     drives the dashboard filter and link across all demo traces.
+ *   - the root span's NAME is set to the fixture's unique `workflow`
+ *     (e.g. `demo.meridian-portal_outage-12`) — shown in the dashboard Workflow
+ *     column, and unique per scenario. The platform dedups identical span CONTENT
+ *     within a run (independent of trace id / timestamp), and the dataset repeats
+ *     some scenarios with a byte-identical root span; a unique root name keeps each
+ *     one distinct so all 12 ingest without a rejected span.
+ *
+ * Ingestion is a raw OTLP POST to `/v2/traces` (the SDK does not expose it).
+ */
+import { Seeder } from './types.js';
+export declare const tracesSeeder: Seeder;

package/dist/lib/demo/seed-traces.js

@@ -0,0 +1,158 @@
+/**
+ * Traces seeder for `respan demo`.
+ *
+ * Ingests the supplied multi-agent service-desk traces (see trace-fixtures.ts)
+ * into the authenticated account. All trace-specific logic lives here; the
+ * orchestrator stays service-agnostic.
+ *
+ * Traces are treated as append-only run history: every `respan demo` run ingests a
+ * fresh batch stamped at the current time, with freshly-generated trace/span ids.
+ * Trace id + timestamps differ on every run, so runs never collide — there is no
+ * dedup-skip and no teardown (`respan demo clear` leaves traces in place).
+ *
+ * Two workflow identifiers, on purpose:
+ *   - `traceloop.workflow.name` is set to the shared constant `demo`
+ *     (`span_workflow_name`, a filterable field) on every span, so one marker
+ *     drives the dashboard filter and link across all demo traces.
+ *   - the root span's NAME is set to the fixture's unique `workflow`
+ *     (e.g. `demo.meridian-portal_outage-12`) — shown in the dashboard Workflow
+ *     column, and unique per scenario. The platform dedups identical span CONTENT
+ *     within a run (independent of trace id / timestamp), and the dataset repeats
+ *     some scenarios with a byte-identical root span; a unique root name keeps each
+ *     one distinct so all 12 ingest without a rejected span.
+ *
+ * Ingestion is a raw OTLP POST to `/v2/traces` (the SDK does not expose it).
+ */
+import { DEMO_TRACES } from './trace-fixtures.js';
+import { randomBytes } from 'node:crypto';
+/** Shared, filterable workflow marker stamped on every demo span. */
+const DEMO_WORKFLOW = 'demo';
+/** OTLP attribute that becomes the trace's `span_workflow_name`. */
+const WORKFLOW_ATTR = 'traceloop.workflow.name';
+/** Spread the whole batch across the last few minutes so the traces read as fresh,
+ * distinct activity and ALL land inside a "last 5 minutes" view. We target a
+ * 4-minute window (under 5, with margin) and divide it evenly across the traces,
+ * so the newest ends ~now and the oldest ends ~4 minutes ago. */
+const SEED_WINDOW_MS = 4 * 60 * 1000;
+const STAGGER_MS = DEMO_TRACES.length > 1 ? Math.floor(SEED_WINDOW_MS / (DEMO_TRACES.length - 1)) : 0;
+/** Set (or replace) a string OTLP attribute on an attributes array. */
+function setStringAttr(attrs, key, value) {
+    const existing = attrs.find((a) => a.key === key);
+    if (existing)
+        existing.value = { stringValue: value };
+    else
+        attrs.push({ key, value: { stringValue: value } });
+}
+/** Resolve the OTLP ingest endpoint from the API base URL. Mirrors the hooks'
+ * resolver: append `/v2/traces`, inserting `/api` when the base lacks it. */
+function resolveTracesEndpoint(baseUrl) {
+    const normalized = (baseUrl || 'https://api.respan.ai').replace(/\/+$/, '');
+    return normalized.endsWith('/api')
+        ? `${normalized}/v2/traces`
+        : `${normalized}/api/v2/traces`;
+}
+/** Deep-clone a fixture body, stamp the shared `demo` workflow attribute on every
+ * span, set the root span's name to `rootName` (unique per trace — shown in the
+ * Workflow column and keeps the root span from colliding with a repeated scenario),
+ * assign a fresh trace id, remap every span id, and shift all timestamps so the
+ * trace ends at `targetEndMs`. Bodies are pure JSON, so JSON round-trip is a safe,
+ * fully-typed clone. */
+function restamp(body, targetEndMs, rootName) {
+    const clone = JSON.parse(JSON.stringify(body));
+    const traceId = randomBytes(16).toString('hex'); // 32 hex chars
+    const spans = [];
+    for (const rs of clone.resourceSpans ?? []) {
+        for (const ss of rs.scopeSpans ?? []) {
+            for (const s of ss.spans ?? []) {
+                // Shared filterable marker on every span...
+                s.attributes = s.attributes ?? [];
+                setStringAttr(s.attributes, WORKFLOW_ATTR, DEMO_WORKFLOW);
+                // ...unique display name on the root span.
+                if (!s.parentSpanId)
+                    s.name = rootName;
+                spans.push(s);
+            }
+        }
+    }
+    const idMap = new Map();
+    for (const s of spans) {
+        if (!idMap.has(s.spanId))
+            idMap.set(s.spanId, randomBytes(8).toString('hex')); // 16 hex chars
+    }
+    let maxEnd = 0n;
+    for (const s of spans) {
+        const e = BigInt(s.endTimeUnixNano);
+        if (e > maxEnd)
+            maxEnd = e;
+    }
+    const targetEndNanos = BigInt(Math.round(targetEndMs)) * 1000000n;
+    const delta = targetEndNanos - maxEnd;
+    for (const s of spans) {
+        s.traceId = traceId;
+        s.spanId = idMap.get(s.spanId);
+        if (s.parentSpanId)
+            s.parentSpanId = idMap.get(s.parentSpanId) ?? s.parentSpanId;
+        s.startTimeUnixNano = String(BigInt(s.startTimeUnixNano) + delta);
+        s.endTimeUnixNano = String(BigInt(s.endTimeUnixNano) + delta);
+    }
+    return { body: clone, traceId };
+}
+/** POST one OTLP body to the ingest endpoint. Throws on a 4xx/5xx response, and
+ * also on an OTLP `partialSuccess` that rejected spans — so a dropped span surfaces
+ * as a loud error instead of a silently span-short trace. */
+async function ingest(endpoint, authHeader, body) {
+    const res = await fetch(endpoint, {
+        method: 'POST',
+        headers: {
+            Authorization: authHeader,
+            'Content-Type': 'application/json',
+            'X-Respan-Dogfood': '1', // anti-recursion: don't trace the ingest itself
+        },
+        body: JSON.stringify(body),
+    });
+    const text = await res.text().catch(() => '');
+    if (res.status >= 400) {
+        throw new Error(`Trace ingest failed (HTTP ${res.status}): ${text.slice(0, 200)}`);
+    }
+    // OTLP returns 200 with a partialSuccess block when some spans are rejected.
+    let rejected = 0;
+    let message = '';
+    try {
+        const parsed = JSON.parse(text);
+        rejected = Number(parsed.partialSuccess?.rejectedSpans ?? 0);
+        message = parsed.partialSuccess?.errorMessage ?? '';
+    }
+    catch {
+        // Non-JSON success body — nothing to inspect.
+    }
+    if (rejected > 0) {
+        throw new Error(`Trace ingest rejected ${rejected} span(s)${message ? `: ${message}` : ''}.`);
+    }
+}
+export const tracesSeeder = {
+    service: 'traces',
+    label: 'Traces',
+    // Traces are append-only run history — nothing to find for dedup or teardown.
+    findExisting() {
+        return Promise.resolve([]);
+    },
+    async run(ctx) {
+        const endpoint = resolveTracesEndpoint(ctx.baseUrl);
+        const now = Date.now();
+        const items = [];
+        for (let i = 0; i < DEMO_TRACES.length; i++) {
+            const fixture = DEMO_TRACES[i];
+            // Fresh trace id + timestamps every run (so runs never collide), staggered
+            // backwards from "now", each with a unique root-span name.
+            const { body, traceId } = restamp(fixture.body, now - i * STAGGER_MS, fixture.workflow);
+            await ingest(endpoint, ctx.authHeader, body);
+            items.push({ name: fixture.name, status: 'created', id: traceId });
+        }
+        return { service: this.service, label: this.label, items };
+    },
+    // Traces are intentionally left in place — each run is timestamped history, and
+    // `respan demo clear` only removes deletable resources (prompts).
+    clear() {
+        return Promise.resolve({ service: this.service, label: this.label, items: [] });
+    },
+};

package/dist/lib/demo/seeders.d.ts

@@ -0,0 +1,8 @@
+/**
+ * The registry of demo seeders, shared by `respan demo` and `respan demo clear`.
+ *
+ * To add a service later (traces, datasets, evaluators), implement a `Seeder` in
+ * its own `seed-*.ts` and append it here — no other change is required.
+ */
+import { Seeder } from './types.js';
+export declare const SEEDERS: Seeder[];

package/dist/lib/demo/seeders.js

@@ -0,0 +1,11 @@
+/**
+ * The registry of demo seeders, shared by `respan demo` and `respan demo clear`.
+ *
+ * To add a service later (traces, datasets, evaluators), implement a `Seeder` in
+ * its own `seed-*.ts` and append it here — no other change is required.
+ */
+import { promptsSeeder } from './seed-prompts.js';
+import { datasetsSeeder } from './seed-datasets.js';
+import { evaluatorsSeeder } from './seed-evaluators.js';
+import { tracesSeeder } from './seed-traces.js';
+export const SEEDERS = [promptsSeeder, datasetsSeeder, evaluatorsSeeder, tracesSeeder];

package/dist/lib/demo/trace-fixtures.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Frozen snapshot of the demo traces, supplied by the team as ready-to-POST OTLP v2
+ * bodies (the `trace-bodies/` dataset: a 22-span, 6-LLM-call multi-agent AI Service
+ * Desk run across two tenants). Source of truth for the traces portion of `respan demo`.
+ *
+ * Each `body` is a complete OTLP/JSON payload for `POST /api/v2/traces`, captured
+ * VERBATIM. The seeder (`seed-traces.ts`) re-stamps fresh trace/span ids and recent
+ * timestamps on each fresh seed, and sets the root span's name to `workflow` (below)
+ * so it shows in the dashboard Workflow column and is unique per trace — the dataset
+ * repeats some scenarios with a byte-identical root span, and a unique root name keeps
+ * the platform from rejecting it as a duplicate workflow span.
+ *
+ * The filterable `span_workflow_name` marker stays a shared constant (see seed-traces.ts)
+ * so dedup, teardown, and the dashboard link match all demo traces at once.
+ *
+ * Frozen snapshot: not regenerated at build time, imported directly. No runtime JSON.
+ */
+/** A complete OTLP/JSON trace body for `POST /api/v2/traces`. */
+export interface OtlpTraceBody {
+    resourceSpans: Record<string, unknown>[];
+}
+/** One demo trace. */
+export interface DemoTraceFixture {
+    /** Human label for the seed summary. */
+    name: string;
+    /** Unique workflow id set as the root span name (shown in the Workflow column). */
+    workflow: string;
+    /** The OTLP body to ingest. */
+    body: OtlpTraceBody;
+}
+export declare const DEMO_TRACES: DemoTraceFixture[];