@nodetool-ai/cli 0.7.0-rc.23 → 0.8.2

package/dist/diagnose.js

@@ -0,0 +1,418 @@
+/**
+ * Pure failure-diagnosis aggregator for `nodetool agent diagnose <job_id>`.
+ *
+ * When a run fails the cause is scattered across the job record, the structured
+ * processing-message stream, and the OpenTelemetry trace. `diagnoseRun` folds
+ * those three sources into one `DiagnosisReport` that names the failing
+ * node/step, the error, the last `llm.chat`/`llm.stream` span at or before the
+ * failure (with tokens, model, and a truncated response), a memory snapshot, and
+ * a deterministic "likely fix locus" pointing at where to start.
+ *
+ * The function is pure and deterministic — it only reads its inputs and never
+ * touches the filesystem, env, or clock. The CLI wiring in
+ * `commands/agent.ts` gathers the real inputs and calls it. Keep all I/O out of
+ * this module so it stays unit-testable under the CLI vitest stub setup, which
+ * stubs every `@nodetool-ai/*` runtime package — only `import type` is used.
+ */
+import { isFiniteNumber, isObjectLike, isString } from "./predicates.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+const RESPONSE_PREVIEW_MAX = 500;
+const VALUE_PREVIEW_MAX = 120;
+const MEMORY_KEY_LIMIT = 40;
+const FAILED_NODE_STATUSES = new Set(["error", "failed"]);
+/** Job statuses that mean the run did not succeed. */
+const FAILED_JOB_STATUSES = new Set(["failed", "error", "cancelled"]);
+const LLM_SPAN_NAMES = new Set(["llm.chat", "llm.stream"]);
+function str(v) {
+    return isString(v) ? v : null;
+}
+function num(v) {
+    return isFiniteNumber(v) ? v : null;
+}
+function truncate(s, max) {
+    const oneLine = s.replace(/\s+/g, " ").trim();
+    return oneLine.length > max ? `${oneLine.slice(0, max)}…[${oneLine.length} chars]` : oneLine;
+}
+/** First non-empty string among the candidates. */
+function firstString(...candidates) {
+    for (const c of candidates) {
+        const s = str(c);
+        if (s && s.trim().length > 0)
+            return s;
+    }
+    return null;
+}
+const NO_SITE = {
+    kind: "unknown",
+    nodeId: null,
+    nodeType: null,
+    nodeName: null,
+    stepId: null,
+    stepLabel: null
+};
+/**
+ * Scan the message stream for the first failure. A failing `node_update`
+ * (status error/failed) wins; otherwise a failed `task_update`/`step_result`
+ * (an agent step); otherwise a top-level `error` message. The scan keeps the
+ * earliest failure so the report points at the root cause, not a downstream
+ * cascade.
+ */
+function scanMessages(messages) {
+    let nodeFailure = null;
+    let nodeError = null;
+    let stepFailure = null;
+    let stepError = null;
+    let topError = null;
+    let jobError = null;
+    for (const raw of messages) {
+        const msg = raw;
+        switch (msg.type) {
+            case "node_update": {
+                const status = str(msg.status);
+                if (status && FAILED_NODE_STATUSES.has(status) && !nodeFailure) {
+                    nodeFailure = {
+                        kind: "node",
+                        nodeId: str(msg.node_id),
+                        nodeType: str(msg.node_type),
+                        nodeName: str(msg.node_name),
+                        stepId: null,
+                        stepLabel: null
+                    };
+                    nodeError = firstString(msg.error) ?? status;
+                }
+                break;
+            }
+            case "task_update": {
+                const event = str(msg.event);
+                const task = msg.task ?? {};
+                const step = msg.step ?? {};
+                const failed = event === "step_failed" ||
+                    str(step.status) === "failed" ||
+                    str(task.status) === "failed";
+                if (failed && !stepFailure) {
+                    stepFailure = {
+                        kind: "step",
+                        nodeId: str(msg.node_id),
+                        nodeType: null,
+                        nodeName: firstString(task.title, task.name),
+                        stepId: firstString(step.id),
+                        stepLabel: firstString(step.instructions, step.name, task.title)
+                    };
+                    stepError = firstString(step.error, task.error);
+                }
+                break;
+            }
+            case "step_result": {
+                const stepErr = firstString(msg.error);
+                if (stepErr && !stepFailure) {
+                    const step = msg.step ?? {};
+                    stepFailure = {
+                        kind: "step",
+                        nodeId: null,
+                        nodeType: null,
+                        nodeName: null,
+                        stepId: firstString(step.id),
+                        stepLabel: firstString(step.instructions, step.name)
+                    };
+                    stepError = stepErr;
+                }
+                break;
+            }
+            case "job_update": {
+                const status = str(msg.status);
+                if (status && FAILED_JOB_STATUSES.has(status)) {
+                    jobError ??= firstString(msg.error, msg.message);
+                }
+                break;
+            }
+            case "error": {
+                topError ??= firstString(msg.message);
+                break;
+            }
+            default:
+                break;
+        }
+    }
+    // Precedence: a failing node is the most specific locus, then an agent step,
+    // then a job/top-level error with no attributable node.
+    if (nodeFailure) {
+        return {
+            failure: nodeFailure,
+            error: nodeError ?? stepError ?? topError ?? jobError,
+            failedAtMs: null,
+            found: true
+        };
+    }
+    if (stepFailure) {
+        return {
+            failure: stepFailure,
+            error: stepError ?? topError ?? jobError,
+            failedAtMs: null,
+            found: true
+        };
+    }
+    const looseError = topError ?? jobError;
+    if (looseError) {
+        return {
+            failure: { ...NO_SITE, kind: "job" },
+            error: looseError,
+            failedAtMs: null,
+            found: true
+        };
+    }
+    return { failure: NO_SITE, error: null, failedAtMs: null, found: false };
+}
+// ---------------------------------------------------------------------------
+// Last LLM span at/before the failure
+// ---------------------------------------------------------------------------
+function spanAttr(span, key) {
+    return span.attributes ? span.attributes[key] : undefined;
+}
+/**
+ * Pick the last `llm.chat`/`llm.stream` span that started at or before the
+ * failure. When no failure time is known, the last LLM span in trace order is
+ * used. Ties on start time resolve to the later end time (and otherwise the
+ * later position in the array), so the most recent call wins deterministically.
+ */
+function selectLastLlmSpan(spans, failedAtMs) {
+    let best = null;
+    let bestStart = Number.NEGATIVE_INFINITY;
+    let bestEnd = Number.NEGATIVE_INFINITY;
+    for (const span of spans) {
+        if (!LLM_SPAN_NAMES.has(span.name))
+            continue;
+        const start = num(span.start_time_ms);
+        if (failedAtMs !== null && start !== null && start > failedAtMs)
+            continue;
+        const startKey = start ?? Number.NEGATIVE_INFINITY;
+        const endKey = num(span.end_time_ms) ?? startKey;
+        // `>=` so a later array position with an equal key still wins (trace order).
+        if (best === null || startKey > bestStart || (startKey === bestStart && endKey >= bestEnd)) {
+            best = span;
+            bestStart = startKey;
+            bestEnd = endKey;
+        }
+    }
+    return best;
+}
+function summarizeLlmSpan(span) {
+    const provider = firstString(spanAttr(span, "gen_ai.system"), spanAttr(span, "agent.provider")) ?? null;
+    const model = firstString(spanAttr(span, "gen_ai.request.model"), spanAttr(span, "gen_ai.response.model"), spanAttr(span, "agent.model")) ?? null;
+    const responseRaw = str(spanAttr(span, "llm.response.content"));
+    const start = num(span.start_time_ms);
+    const end = num(span.end_time_ms);
+    return {
+        name: span.name,
+        provider,
+        model,
+        inputTokens: num(spanAttr(span, "gen_ai.usage.input_tokens")),
+        outputTokens: num(spanAttr(span, "gen_ai.usage.output_tokens")),
+        totalTokens: num(spanAttr(span, "gen_ai.usage.total_tokens")),
+        costUsd: num(spanAttr(span, "gen_ai.usage.cost_usd")),
+        responsePreview: responseRaw ? truncate(responseRaw, RESPONSE_PREVIEW_MAX) : null,
+        durationMs: start !== null && end !== null ? end - start : null,
+        statusCode: span.status?.code ?? null
+    };
+}
+// ---------------------------------------------------------------------------
+// Memory snapshot
+// ---------------------------------------------------------------------------
+function summarizeMemory(memory) {
+    const keys = Object.keys(memory);
+    const shown = keys.slice(0, MEMORY_KEY_LIMIT);
+    const preview = {};
+    for (const k of shown) {
+        const v = memory[k];
+        let s;
+        if (isString(v))
+            s = v;
+        else if (v === null || v === undefined)
+            s = String(v);
+        else if (isObjectLike(v))
+            s = Array.isArray(v) ? `array(${v.length})` : "{…}";
+        else
+            s = String(v);
+        preview[k] = truncate(s, VALUE_PREVIEW_MAX);
+    }
+    return { keyCount: keys.length, keys: shown, preview };
+}
+// ---------------------------------------------------------------------------
+// Fix locus heuristic
+// ---------------------------------------------------------------------------
+/** Extract concrete file/prompt/model hints from an error string. */
+function extractHints(error, model) {
+    const hints = [];
+    if (!error)
+        return hints;
+    const lower = error.toLowerCase();
+    // File path references (./foo.ts, /abs/path, src/x.json).
+    const fileMatch = error.match(/(?:\.{0,2}\/)?[\w.-]+\/[\w./-]+\.\w+/);
+    if (fileMatch)
+        hints.push(`file: ${fileMatch[0]}`);
+    if (/\bprompt\b/.test(lower) || /system\s*prompt/.test(lower)) {
+        hints.push("prompt: review the node/agent prompt");
+    }
+    if (/\bmodel\b/.test(lower) || /\bcontext\s*length\b/.test(lower) || /\btoken/.test(lower)) {
+        hints.push(model ? `model: ${model}` : "model: check the model id/limits");
+    }
+    if (/\bapi[\s_-]?key\b/.test(lower) || /unauthorized|401|forbidden|403/.test(lower)) {
+        hints.push("auth: missing/invalid API key or permissions");
+    }
+    if (/timeout|timed out|etimedout/.test(lower)) {
+        hints.push("timeout: increase the run/request timeout or retry");
+    }
+    if (/rate.?limit|429/.test(lower)) {
+        hints.push("rate-limit: back off or lower concurrency");
+    }
+    return hints;
+}
+function buildFixLocus(failure, error, lastLlm) {
+    const model = lastLlm?.model ?? null;
+    const hints = extractHints(error, model);
+    const where = failure.kind === "node"
+        ? `node ${failure.nodeName ?? failure.nodeId ?? "?"}` +
+            (failure.nodeType ? ` (${failure.nodeType})` : "")
+        : failure.kind === "step"
+            ? `agent step ${failure.stepLabel ?? failure.stepId ?? "?"}`
+            : failure.kind === "job"
+                ? "the job (no node attributed)"
+                : "the run";
+    const errPart = error ? ` — ${truncate(error, 160)}` : "";
+    const summary = `Start at ${where}${errPart}`;
+    return {
+        summary,
+        nodeType: failure.nodeType,
+        model,
+        hints
+    };
+}
+// ---------------------------------------------------------------------------
+// diagnoseRun
+// ---------------------------------------------------------------------------
+/**
+ * Aggregate a failed run's job record, message stream, and trace spans into a
+ * single `DiagnosisReport`. Deterministic and side-effect-free.
+ */
+export function diagnoseRun(inputs) {
+    const { job, messages, spans, memory } = inputs;
+    const missing = [];
+    // ---- Failure site + error -------------------------------------------------
+    let findings = { failure: NO_SITE, error: null, failedAtMs: null, found: false };
+    if (messages && messages.length > 0) {
+        findings = scanMessages(messages);
+    }
+    else {
+        missing.push("messages");
+    }
+    // The job record can supply (or override-fill) the error and status even when
+    // the message stream named the failing node.
+    const jobStatus = job?.status ?? null;
+    const jobError = firstString(job?.error);
+    const jobFailed = jobStatus ? FAILED_JOB_STATUSES.has(jobStatus) : false;
+    let { failure, error } = findings;
+    let found = findings.found;
+    if (!found && (jobFailed || jobError)) {
+        failure = { ...NO_SITE, kind: "job" };
+        error = jobError;
+        found = true;
+    }
+    else if (found && !error && jobError) {
+        error = jobError;
+    }
+    if (!job)
+        missing.push("job");
+    // ---- Last LLM span --------------------------------------------------------
+    let lastLlm = null;
+    if (spans && spans.length > 0) {
+        const span = selectLastLlmSpan(spans, findings.failedAtMs);
+        if (span)
+            lastLlm = summarizeLlmSpan(span);
+        else
+            missing.push("llm-span");
+    }
+    else {
+        missing.push("trace");
+    }
+    // ---- Memory snapshot ------------------------------------------------------
+    let memorySnapshot = null;
+    if (memory && Object.keys(memory).length > 0) {
+        memorySnapshot = summarizeMemory(memory);
+    }
+    else {
+        missing.push("memory");
+    }
+    // ---- Verdict --------------------------------------------------------------
+    const ok = !found;
+    const fixLocus = ok ? null : buildFixLocus(failure, error, lastLlm);
+    return {
+        ok,
+        jobId: job?.id ?? null,
+        jobStatus,
+        failure: ok ? NO_SITE : failure,
+        error: ok ? null : error,
+        lastLlm,
+        memory: memorySnapshot,
+        fixLocus,
+        missing
+    };
+}
+// ---------------------------------------------------------------------------
+// Human-readable rendering
+// ---------------------------------------------------------------------------
+/**
+ * Render a `DiagnosisReport` as a plain-text block for the terminal. Kept here
+ * (not in the CLI command) so the formatting is unit-testable and dependency-
+ * free; the command adds color separately if desired.
+ */
+export function renderDiagnosis(report) {
+    const lines = [];
+    const id = report.jobId ?? "(unknown job)";
+    if (report.ok) {
+        lines.push(`✓ No failure detected for ${id}.`);
+        if (report.jobStatus)
+            lines.push(`  status: ${report.jobStatus}`);
+        if (report.missing.length > 0) {
+            lines.push(`  sources missing: ${report.missing.join(", ")}`);
+        }
+        return lines.join("\n");
+    }
+    lines.push(`✗ Diagnosis for ${id}${report.jobStatus ? ` (${report.jobStatus})` : ""}`);
+    const f = report.failure;
+    const site = f.kind === "node"
+        ? `node ${f.nodeName ?? f.nodeId ?? "?"}${f.nodeType ? ` [${f.nodeType}]` : ""}`
+        : f.kind === "step"
+            ? `step ${f.stepLabel ?? f.stepId ?? "?"}`
+            : f.kind === "job"
+                ? "job (no node attributed)"
+                : "unknown";
+    lines.push(`  failure: ${site}`);
+    lines.push(`  error:   ${report.error ?? "(none reported)"}`);
+    if (report.lastLlm) {
+        const l = report.lastLlm;
+        const tok = l.inputTokens !== null || l.outputTokens !== null
+            ? ` tokens=${l.inputTokens ?? "?"}/${l.outputTokens ?? "?"}`
+            : "";
+        lines.push(`  last LLM (${l.name}): ${l.provider ?? "?"}/${l.model ?? "?"}${tok}` +
+            (l.statusCode ? ` status=${l.statusCode}` : ""));
+        if (l.responsePreview)
+            lines.push(`    response: ${l.responsePreview}`);
+    }
+    else {
+        lines.push("  last LLM: (no trace span available)");
+    }
+    if (report.memory) {
+        lines.push(`  memory: ${report.memory.keyCount} key(s): ${report.memory.keys.join(", ")}`);
+    }
+    if (report.fixLocus) {
+        lines.push(`  → ${report.fixLocus.summary}`);
+        for (const hint of report.fixLocus.hints)
+            lines.push(`    • ${hint}`);
+    }
+    if (report.missing.length > 0) {
+        lines.push(`  sources missing: ${report.missing.join(", ")}`);
+    }
+    return lines.join("\n");
+}
+//# sourceMappingURL=diagnose.js.map

package/dist/diagnose.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"diagnose.js","sourceRoot":"","sources":["../src/diagnose.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,EACL,cAAc,EACd,YAAY,EACZ,QAAQ,EACT,MAAM,iBAAiB,CAAC;AAkIzB,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAC1D,sDAAsD;AACtD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;AACtE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC,CAAC;AAE3D,SAAS,GAAG,CAAC,CAAU;IACrB,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAChC,CAAC;AAED,SAAS,GAAG,CAAC,CAAU;IACrB,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACtC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAW;IACtC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9C,OAAO,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;AAC/F,CAAC;AAED,mDAAmD;AACnD,SAAS,WAAW,CAAC,GAAG,UAAqB;IAC3C,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAcD,MAAM,OAAO,GAAgB;IAC3B,IAAI,EAAE,SAAS;IACf,MAAM,EAAE,IAAI;IACZ,QAAQ,EAAE,IAAI;IACd,QAAQ,EAAE,IAAI;IACd,MAAM,EAAE,IAAI;IACZ,SAAS,EAAE,IAAI;CAChB,CAAC;AAEF;;;;;;GAMG;AACH,SAAS,YAAY,CACnB,QAAoE;IAEpE,IAAI,WAAW,GAAuB,IAAI,CAAC;IAC3C,IAAI,SAAS,GAAkB,IAAI,CAAC;IACpC,IAAI,WAAW,GAAuB,IAAI,CAAC;IAC3C,IAAI,SAAS,GAAkB,IAAI,CAAC;IACpC,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,QAAQ,GAAkB,IAAI,CAAC;IAEnC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,GAAG,CAAC;QAChB,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,aAAa,EAAE,CAAC;gBACnB,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC/B,IAAI,MAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;oBAC/D,WAAW,GAAG;wBACZ,IAAI,EAAE,MAAM;wBACZ,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC;wBACxB,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC;wBAC5B,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC;wBAC5B,MAAM,EAAE,IAAI;wBACZ,SAAS,EAAE,IAAI;qBAChB,CAAC;oBACF,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC;gBAC/C,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,aAAa,EAAE,CAAC;gBACnB,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,MAAM,IAAI,GAAI,GAAG,CAAC,IAAgC,IAAI,EAAE,CAAC;gBACzD,MAAM,IAAI,GAAI,GAAG,CAAC,IAAgC,IAAI,EAAE,CAAC;gBACzD,MAAM,MAAM,GACV,KAAK,KAAK,aAAa;oBACvB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ;oBAC7B,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC;gBAChC,IAAI,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;oBAC3B,WAAW,GAAG;wBACZ,IAAI,EAAE,MAAM;wBACZ,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC;wBACxB,QAAQ,EAAE,IAAI;wBACd,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC;wBAC5C,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC5B,SAAS,EAAE,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC;qBACjE,CAAC;oBACF,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;gBAClD,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,aAAa,EAAE,CAAC;gBACnB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACvC,IAAI,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;oBAC5B,MAAM,IAAI,GAAI,GAAG,CAAC,IAAgC,IAAI,EAAE,CAAC;oBACzD,WAAW,GAAG;wBACZ,IAAI,EAAE,MAAM;wBACZ,MAAM,EAAE,IAAI;wBACZ,QAAQ,EAAE,IAAI;wBACd,QAAQ,EAAE,IAAI;wBACd,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC5B,SAAS,EAAE,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC;qBACrD,CAAC;oBACF,SAAS,GAAG,OAAO,CAAC;gBACtB,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,YAAY,EAAE,CAAC;gBAClB,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC/B,IAAI,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC9C,QAAQ,KAAK,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBACnD,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,OAAO,EAAE,CAAC;gBACb,QAAQ,KAAK,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtC,MAAM;YACR,CAAC;YACD;gBACE,MAAM;QACV,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,wDAAwD;IACxD,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO;YACL,OAAO,EAAE,WAAW;YACpB,KAAK,EAAE,SAAS,IAAI,SAAS,IAAI,QAAQ,IAAI,QAAQ;YACrD,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,IAAI;SACZ,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO;YACL,OAAO,EAAE,WAAW;YACpB,KAAK,EAAE,SAAS,IAAI,QAAQ,IAAI,QAAQ;YACxC,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,IAAI;SACZ,CAAC;IACJ,CAAC;IACD,MAAM,UAAU,GAAG,QAAQ,IAAI,QAAQ,CAAC;IACxC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;YACpC,KAAK,EAAE,UAAU;YACjB,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,IAAI;SACZ,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC3E,CAAC;AAED,8EAA8E;AAC9E,sCAAsC;AACtC,8EAA8E;AAE9E,SAAS,QAAQ,CACf,IAAmB,EACnB,GAAW;IAEX,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC5D,CAAC;AAED;;;;;GAKG;AACH,SAAS,iBAAiB,CACxB,KAAmC,EACnC,UAAyB;IAEzB,IAAI,IAAI,GAAyB,IAAI,CAAC;IACtC,IAAI,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACzC,IAAI,OAAO,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAC7C,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,IAAI,UAAU,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,UAAU;YAAE,SAAS;QAC1E,MAAM,QAAQ,GAAG,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC;QACnD,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC;QACjD,6EAA6E;QAC7E,IAAI,IAAI,KAAK,IAAI,IAAI,QAAQ,GAAG,SAAS,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;YAC3F,IAAI,GAAG,IAAI,CAAC;YACZ,SAAS,GAAG,QAAQ,CAAC;YACrB,OAAO,GAAG,MAAM,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAmB;IAC3C,MAAM,QAAQ,GACZ,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC,IAAI,IAAI,CAAC;IACzF,MAAM,KAAK,GACT,WAAW,CACT,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC,EACtC,QAAQ,CAAC,IAAI,EAAE,uBAAuB,CAAC,EACvC,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC,CAC9B,IAAI,IAAI,CAAC;IACZ,MAAM,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAClC,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,QAAQ;QACR,KAAK;QACL,WAAW,EAAE,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,2BAA2B,CAAC,CAAC;QAC7D,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,4BAA4B,CAAC,CAAC;QAC/D,WAAW,EAAE,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,2BAA2B,CAAC,CAAC;QAC7D,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;QACrD,eAAe,EAAE,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI;QACjF,UAAU,EAAE,KAAK,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI;QAC/D,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,IAAI;KACtC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,SAAS,eAAe,CAAC,MAA+B;IACtD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAS,CAAC;QACd,IAAI,QAAQ,CAAC,CAAC,CAAC;YAAE,CAAC,GAAG,CAAC,CAAC;aAClB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;YAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;aACjD,IAAI,YAAY,CAAC,CAAC,CAAC;YAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;;YACzE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACnB,OAAO,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AACzD,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,qEAAqE;AACrE,SAAS,YAAY,CAAC,KAAoB,EAAE,KAAoB;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAElC,0DAA0D;IAC1D,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACtE,IAAI,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEnD,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9D,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3F,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACpF,KAAK,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,6BAA6B,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CACpB,OAAoB,EACpB,KAAoB,EACpB,OAA8B;IAE9B,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC;IACrC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAEzC,MAAM,KAAK,GACT,OAAO,CAAC,IAAI,KAAK,MAAM;QACrB,CAAC,CAAC,QAAQ,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG,EAAE;YACnD,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM;YACvB,CAAC,CAAC,cAAc,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG,EAAE;YAC5D,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,KAAK;gBACtB,CAAC,CAAC,8BAA8B;gBAChC,CAAC,CAAC,SAAS,CAAC;IAEpB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,MAAM,OAAO,GAAG,YAAY,KAAK,GAAG,OAAO,EAAE,CAAC;IAE9C,OAAO;QACL,OAAO;QACP,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,KAAK;QACL,KAAK;KACN,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,MAAsB;IAChD,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAChD,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,8EAA8E;IAC9E,IAAI,QAAQ,GAAoB,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAClG,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC3B,CAAC;IAED,8EAA8E;IAC9E,6CAA6C;IAC7C,MAAM,SAAS,GAAG,GAAG,EAAE,MAAM,IAAI,IAAI,CAAC;IACtC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAEzE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;IAE3B,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,IAAI,QAAQ,CAAC,EAAE,CAAC;QACtC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACtC,KAAK,GAAG,QAAQ,CAAC;QACjB,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;SAAM,IAAI,KAAK,IAAI,CAAC,KAAK,IAAI,QAAQ,EAAE,CAAC;QACvC,KAAK,GAAG,QAAQ,CAAC;IACnB,CAAC;IAED,IAAI,CAAC,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAE9B,8EAA8E;IAC9E,IAAI,OAAO,GAA0B,IAAI,CAAC;IAC1C,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC3D,IAAI,IAAI;YAAE,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;;YACtC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;IAED,8EAA8E;IAC9E,IAAI,cAAc,GAA0B,IAAI,CAAC;IACjD,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,cAAc,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAED,8EAA8E;IAC9E,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC;IAClB,MAAM,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAEpE,OAAO;QACL,EAAE;QACF,KAAK,EAAE,GAAG,EAAE,EAAE,IAAI,IAAI;QACtB,SAAS;QACT,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO;QAC/B,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;QACxB,OAAO;QACP,MAAM,EAAE,cAAc;QACtB,QAAQ;QACR,OAAO;KACR,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,2BAA2B;AAC3B,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAuB;IACrD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC;IAE3C,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC;QAC/C,IAAI,MAAM,CAAC,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QAClE,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,sBAAsB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEvF,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC;IACzB,MAAM,IAAI,GACR,CAAC,CAAC,IAAI,KAAK,MAAM;QACf,CAAC,CAAC,QAAQ,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAChF,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM;YACjB,CAAC,CAAC,QAAQ,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,EAAE;YAC1C,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK;gBAChB,CAAC,CAAC,0BAA0B;gBAC5B,CAAC,CAAC,SAAS,CAAC;IACpB,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,KAAK,IAAI,iBAAiB,EAAE,CAAC,CAAC;IAE9D,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC;QACzB,MAAM,GAAG,GACP,CAAC,CAAC,WAAW,KAAK,IAAI,IAAI,CAAC,CAAC,YAAY,KAAK,IAAI;YAC/C,CAAC,CAAC,WAAW,CAAC,CAAC,WAAW,IAAI,GAAG,IAAI,CAAC,CAAC,YAAY,IAAI,GAAG,EAAE;YAC5D,CAAC,CAAC,EAAE,CAAC;QACT,KAAK,CAAC,IAAI,CACR,eAAe,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,QAAQ,IAAI,GAAG,IAAI,CAAC,CAAC,KAAK,IAAI,GAAG,GAAG,GAAG,EAAE;YACpE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAClD,CAAC;QACF,IAAI,CAAC,CAAC,eAAe;YAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC1E,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IACtD,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,QAAQ,YAAY,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,OAAO,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7C,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,sBAAsB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/evals/graph-runner.d.ts

@@ -0,0 +1,8 @@
+import type { GraphRunner } from "@nodetool-ai/agents";
+/**
+ * Build the runner the eval harness calls once per planned graph. The registry
+ * is built once and reused across cases; each case gets a fresh session,
+ * context, and job id.
+ */
+export declare function createEvalGraphRunner(): GraphRunner;
+//# sourceMappingURL=graph-runner.d.ts.map

package/dist/evals/graph-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-runner.d.ts","sourceRoot":"","sources":["../../src/evals/graph-runner.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,WAAW,EAAkB,MAAM,qBAAqB,CAAC;AAQvE;;;;GAIG;AACH,wBAAgB,qBAAqB,IAAI,WAAW,CA4DnD"}

package/dist/evals/graph-runner.js

@@ -0,0 +1,136 @@
+/**
+ * Real graph runner for the `graph-e2e` eval suite.
+ *
+ * The suite lives in `@nodetool-ai/agents`, which has no execution dependency —
+ * it takes a `GraphRunner` from its caller. This is that caller's side: the
+ * planned graph runs through `ExecutionSession` exactly as `nodetool debug`
+ * runs one, and the terminal status plus every output the run surfaced come
+ * back in the shape the harness scores.
+ *
+ * Integration code (registries, runtime context, Python bridge) — exercised by
+ * running the suite, not by unit tests.
+ */
+import { getDefaultAssetsPath } from "@nodetool-ai/config";
+import { getSecret } from "@nodetool-ai/models";
+import { ExecutionSession, toRawGraphInput } from "@nodetool-ai/execution";
+import { ProcessingContext } from "@nodetool-ai/runtime";
+import { FileStorageAdapter } from "@nodetool-ai/storage";
+import { createGraphNodeTypeResolver } from "@nodetool-ai/node-sdk";
+import { buildFullRegistry } from "../node-registry.js";
+import { isString } from "../predicates.js";
+/**
+ * Build the runner the eval harness calls once per planned graph. The registry
+ * is built once and reused across cases; each case gets a fresh session,
+ * context, and job id.
+ */
+export function createEvalGraphRunner() {
+    const registry = buildFullRegistry();
+    return async ({ graph, params, timeoutMs, signal }) => {
+        const startedAt = Date.now();
+        const jobId = `eval-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+        const context = new ProcessingContext({
+            jobId,
+            workflowId: null,
+            userId: "1",
+            secretResolver: getSecret,
+            storage: new FileStorageAdapter(getDefaultAssetsPath()),
+            workspaceDir: null,
+            workspaceStorage: null
+        });
+        const session = await ExecutionSession.create({
+            graph: toRawGraphInput(graph),
+            registry,
+            // Registry alone hydrates node flags but not `propertyTypes`, and
+            // correlation analysis reads list-ness only from that map — without it
+            // every `list[...]` handle reads as non-list, so a stream arriving on
+            // one collapses to empty scope and the node runs once on the last
+            // value. A planner-built graph never carries the map, so an eval would
+            // score a graph the real runner executes differently.
+            resolveNodeType: createGraphNodeTypeResolver(registry).resolveNodeType,
+            jobId,
+            params,
+            context,
+            limits: { runTimeoutMs: timeoutMs }
+        });
+        const abort = () => session.cancel("aborted");
+        signal?.addEventListener("abort", abort, { once: true });
+        let result;
+        try {
+            // `session.result` never rejects — kernel failures resolve as
+            // `status: "failed"` rather than throwing.
+            result = await session.result;
+        }
+        finally {
+            signal?.removeEventListener("abort", abort);
+        }
+        const messages = result.messages ?? [];
+        const outputs = collectOutputs(messages, result.outputs);
+        const timedOut = session.cancelReason === "timeout";
+        return {
+            ok: result.status === "completed",
+            status: result.status,
+            error: result.error ??
+                (timedOut ? `run exceeded timeout (${timeoutMs}ms)` : undefined),
+            outputs,
+            nodeErrors: collectNodeErrors(messages),
+            durationMs: Date.now() - startedAt
+        };
+    };
+}
+/**
+ * Outputs come from the `output_update` stream, whose `output_name` is the
+ * output node's workflow-facing name — the name a case's expectations refer to.
+ * Streamed values arrive as chunks, so string chunks for one name concatenate
+ * and any other type keeps the last value. `RunResult.outputs` fills in
+ * anything the stream didn't carry.
+ */
+function collectOutputs(messages, runOutputs) {
+    const byName = new Map();
+    // `RunResult.outputs` keys an output node by its descriptor name, falling
+    // back to the node id — which is the same value the stream already reported
+    // under the node's workflow-facing name. Track the ids so the fallback merge
+    // doesn't add a second, id-keyed copy of an output we already have.
+    const seenNodeIds = new Set();
+    for (const msg of messages) {
+        if (msg.type !== "output_update")
+            continue;
+        const name = String(msg.output_name ?? msg.node_name ?? "output");
+        const nodeId = msg.node_id;
+        const previous = byName.get(name);
+        const value = previous &&
+            isString(previous.value) &&
+            isString(msg.value) &&
+            msg.disposition !== "replace"
+            ? previous.value + msg.value
+            : msg.value;
+        byName.set(name, { name, nodeId, value });
+        if (nodeId)
+            seenNodeIds.add(nodeId);
+    }
+    for (const [name, values] of Object.entries(runOutputs ?? {})) {
+        if (byName.has(name) || seenNodeIds.has(name) || values.length === 0)
+            continue;
+        byName.set(name, {
+            name,
+            value: values.length === 1 ? values[0] : values
+        });
+    }
+    return [...byName.values()];
+}
+function collectNodeErrors(messages) {
+    const errors = [];
+    for (const msg of messages) {
+        if (msg.type === "node_update" &&
+            (msg.status === "error" || msg.status === "failed")) {
+            errors.push({
+                nodeId: msg.node_id,
+                message: msg.error ?? String(msg.status)
+            });
+        }
+        else if (msg.type === "error") {
+            errors.push({ nodeId: null, message: msg.message });
+        }
+    }
+    return errors;
+}
+//# sourceMappingURL=graph-runner.js.map

package/dist/evals/graph-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-runner.js","sourceRoot":"","sources":["../../src/evals/graph-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAG3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,2BAA2B,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C;;;;GAIG;AACH,MAAM,UAAU,qBAAqB;IACnC,MAAM,QAAQ,GAAG,iBAAiB,EAAE,CAAC;IAErC,OAAO,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE;QACpD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAE7E,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC;YACpC,KAAK;YACL,UAAU,EAAE,IAAI;YAChB,MAAM,EAAE,GAAG;YACX,cAAc,EAAE,SAAS;YACzB,OAAO,EAAE,IAAI,kBAAkB,CAAC,oBAAoB,EAAE,CAAC;YACvD,YAAY,EAAE,IAAI;YAClB,gBAAgB,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC;YAC5C,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC;YAC7B,QAAQ;YACR,kEAAkE;YAClE,uEAAuE;YACvE,sEAAsE;YACtE,kEAAkE;YAClE,uEAAuE;YACvE,sDAAsD;YACtD,eAAe,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC,eAAe;YACtE,KAAK;YACL,MAAM;YACN,OAAO;YACP,MAAM,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE;SACpC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzD,IAAI,MAAM,CAAC;QACX,IAAI,CAAC;YACH,8DAA8D;YAC9D,2CAA2C;YAC3C,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC;QAChC,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,KAAK,SAAS,CAAC;QAEpD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,MAAM,KAAK,WAAW;YACjC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,KAAK,EACH,MAAM,CAAC,KAAK;gBACZ,CAAC,QAAQ,CAAC,CAAC,CAAC,yBAAyB,SAAS,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAClE,OAAO;YACP,UAAU,EAAE,iBAAiB,CAAC,QAAQ,CAAC;YACvC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CACrB,QAAsC,EACtC,UAAiD;IAEjD,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,0EAA0E;IAC1E,4EAA4E;IAC5E,6EAA6E;IAC7E,oEAAoE;IACpE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,GAAG,CAAC,IAAI,KAAK,eAAe;YAAE,SAAS;QAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAC;QAClE,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,KAAK,GACT,QAAQ;YACR,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;YACxB,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;YACnB,GAAG,CAAC,WAAW,KAAK,SAAS;YAC3B,CAAC,CAAC,QAAQ,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK;YAC5B,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;QAChB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1C,IAAI,MAAM;YAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;QAC9D,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAClE,SAAS;QACX,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE;YACf,IAAI;YACJ,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM;SAChD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,iBAAiB,CACxB,QAAsC;IAEtC,MAAM,MAAM,GAAiD,EAAE,CAAC;IAChE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,IACE,GAAG,CAAC,IAAI,KAAK,aAAa;YAC1B,CAAC,GAAG,CAAC,MAAM,KAAK,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,EACnD,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,MAAM,EAAE,GAAG,CAAC,OAAO;gBACnB,OAAO,EAAE,GAAG,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;aACzC,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}