@runtypelabs/sdk 5.8.1 → 5.9.0

package/dist/index.cjs

@@ -2283,12 +2283,16 @@ function judge(criteria, opts) {
   if (typeof criteria !== "string" || criteria.trim().length === 0) {
     throw new Error("judge() requires non-empty criteria");
   }
+  if (opts?.judgeFlowId !== void 0 && opts.judgeFlowId.trim().length === 0) {
+    throw new Error("judge() requires a non-empty judgeFlowId when one is provided");
+  }
   return gradeable({
     kind: "ai",
     criteria,
     ...opts?.preset ? { preset: opts.preset } : {},
     ...opts?.useExpected ? { useExpected: true } : {},
     ...opts?.model ? { model: opts.model } : {},
+    ...opts?.judgeFlowId ? { judgeFlowId: opts.judgeFlowId } : {},
     ...opts?.threshold !== void 0 ? { threshold: opts.threshold } : {}
   });
 }
@@ -6395,7 +6399,7 @@ var Runtype = class {
 
 // src/version.ts
 var FALLBACK_VERSION = "0.0.0";
-var SDK_VERSION = "5.8.1".length > 0 ? "5.8.1" : FALLBACK_VERSION;
+var SDK_VERSION = "5.9.0".length > 0 ? "5.9.0" : FALLBACK_VERSION;
 var RUNTYPE_CLIENT_KIND = "sdk";
 var SDK_USER_AGENT = `runtype-sdk/${SDK_VERSION} (typescript)`;
 

package/dist/index.d.cts

@@ -18404,6 +18404,8 @@ interface paths {
                     content: {
                         "application/json": {
                             data: {
+                                /** @description True when historical (R2 SQL) logs were unavailable and only recent hot-tier entries are included. Absent on healthy responses. */
+                                degraded?: boolean;
                                 entries: {
                                     [key: string]: unknown;
                                 }[];
@@ -18582,6 +18584,8 @@ interface paths {
                                 byType: {
                                     [key: string]: number;
                                 };
+                                /** @description True when historical (R2 SQL) counts were unavailable and stats cover only the recent hot-tier window. Absent on healthy responses. */
+                                degraded?: boolean;
                                 histogram: {
                                     bucket: string;
                                     count: number;
@@ -38717,6 +38721,12 @@ interface components {
                 caseId: string | null;
                 name: string;
                 outcomes: {
+                    /** @description AI graders only: transcript entries the verdict rests on — indices into the run's trace steps (message) or tool calls (tool_call). */
+                    evidence?: {
+                        index: number;
+                        /** @enum {string} */
+                        kind: "message" | "tool_call";
+                    }[];
                     graderIndex: number;
                     kind: string;
                     passed: boolean;
@@ -38727,6 +38737,11 @@ interface components {
                      * @enum {string}
                      */
                     severity?: "gate" | "soft";
+                    /**
+                     * @description AI graders only: the judge's binary verdict. insufficient_evidence means the judge declined to guess; it renders as a warning and never fails the case.
+                     * @enum {string}
+                     */
+                    verdict?: "pass" | "fail" | "insufficient_evidence";
                 }[];
                 /** @description Truncated snapshot of the output that was graded (2000 chars max). */
                 outputExcerpt: string;
@@ -39836,6 +39851,12 @@ interface components {
                 errored: boolean;
                 name: string;
                 outcomes: {
+                    /** @description AI graders only: transcript entries the verdict rests on — indices into the run's trace steps (message) or tool calls (tool_call). */
+                    evidence?: {
+                        index: number;
+                        /** @enum {string} */
+                        kind: "message" | "tool_call";
+                    }[];
                     graderIndex: number;
                     kind: string;
                     passed: boolean;
@@ -39846,6 +39867,11 @@ interface components {
                      * @enum {string}
                      */
                     severity?: "gate" | "soft";
+                    /**
+                     * @description AI graders only: the judge's binary verdict. insufficient_evidence means the judge declined to guess; it renders as a warning and never fails the case.
+                     * @enum {string}
+                     */
+                    verdict?: "pass" | "fail" | "insufficient_evidence";
                 }[];
                 outputExcerpt: string;
                 passed: boolean;
@@ -42396,6 +42422,8 @@ interface LogQueryResult {
         cursor: string | null;
         hasMore: boolean;
     };
+    /** True when historical (R2 SQL) logs were unavailable and only recent hot-tier entries are included. */
+    degraded?: boolean;
 }
 /** Logs are wrapped in a `{ success, data }` envelope by the API. */
 interface LogQueryResponse {
@@ -42414,6 +42442,8 @@ interface LogStatsResult {
         count: number;
         errors: number;
     }>;
+    /** True when historical (R2 SQL) counts were unavailable and stats cover only the recent hot-tier window. */
+    degraded?: boolean;
 }
 interface LogStatsResponse {
     success: boolean;
@@ -42621,8 +42651,17 @@ interface AIGrader {
     criteria: string;
     /** Reference-guided when the case has `expected`. */
     useExpected?: boolean;
-    /** Defaults to a cheap routed model at execution time. */
+    /** Defaults to a cheap routed model at execution time. Ignored when `judgeFlowId` is set. */
     model?: string;
+    /**
+     * Custom judge flow: your flow (by id) runs as the judge instead of the
+     * platform judge prompt. It receives `criteria`, `caseName`, `expected`, and
+     * `transcript` as inputs and must output the platform verdict JSON
+     * (`{"reasoning", "verdict": "pass"|"fail"|"insufficient_evidence", "evidence"}`).
+     * A miswired flow (missing, failing, or returning malformed output) resolves
+     * to an insufficient-evidence warning with the error, never a verdict.
+     */
+    judgeFlowId?: string;
     /** Pass cutoff on the 1-5 judge scale (default 4 server-side). Set via `.atLeast(n)`. */
     threshold?: number;
     /** Hard gate (default) vs soft. Set via `.gate()` / `.soft()`. */
@@ -42717,6 +42756,7 @@ declare function cost(maxUsd: number): Gradeable<CheckGrader>;
 declare function judge(criteria: string, opts?: {
     useExpected?: boolean;
     model?: string;
+    judgeFlowId?: string;
     threshold?: number;
     preset?: BuiltInGraderId;
 }): Gradeable<AIGrader>;
@@ -42852,10 +42892,24 @@ interface GraderOutcome {
     passed: boolean;
     /** The grader's severity (absent ⇒ gate). A soft miss only fails under `--strict`. */
     severity?: GraderSeverity;
-    /** 0..1 for scaled graders (AI graders normalize their 1-5 to 0..1). */
+    /** 0..1 for scaled graders (AI graders normalize their 1-5 to 0..1). Absent for `insufficient_evidence`. */
     score?: number;
-    /** AI-grader verdict, or a check's human-readable reason. */
+    /** AI-grader reasoning (critique-before-verdict), or a check's human-readable reason. */
     reasoning?: string;
+    /**
+     * AI graders only: the trajectory judge's binary verdict.
+     * `insufficient_evidence` means the judge declined to guess — it renders as
+     * a warning and never fails the case.
+     */
+    verdict?: 'pass' | 'fail' | 'insufficient_evidence';
+    /**
+     * AI graders only: transcript entries the verdict rests on — indices into
+     * the run's trace steps (`message`) or tool calls (`tool_call`).
+     */
+    evidence?: Array<{
+        kind: 'message' | 'tool_call';
+        index: number;
+    }>;
 }
 /** One case's run result: pass/fail plus each grader's outcome. */
 interface RunEvalCaseResult {

package/dist/index.d.ts

@@ -18404,6 +18404,8 @@ interface paths {
                     content: {
                         "application/json": {
                             data: {
+                                /** @description True when historical (R2 SQL) logs were unavailable and only recent hot-tier entries are included. Absent on healthy responses. */
+                                degraded?: boolean;
                                 entries: {
                                     [key: string]: unknown;
                                 }[];
@@ -18582,6 +18584,8 @@ interface paths {
                                 byType: {
                                     [key: string]: number;
                                 };
+                                /** @description True when historical (R2 SQL) counts were unavailable and stats cover only the recent hot-tier window. Absent on healthy responses. */
+                                degraded?: boolean;
                                 histogram: {
                                     bucket: string;
                                     count: number;
@@ -38717,6 +38721,12 @@ interface components {
                 caseId: string | null;
                 name: string;
                 outcomes: {
+                    /** @description AI graders only: transcript entries the verdict rests on — indices into the run's trace steps (message) or tool calls (tool_call). */
+                    evidence?: {
+                        index: number;
+                        /** @enum {string} */
+                        kind: "message" | "tool_call";
+                    }[];
                     graderIndex: number;
                     kind: string;
                     passed: boolean;
@@ -38727,6 +38737,11 @@ interface components {
                      * @enum {string}
                      */
                     severity?: "gate" | "soft";
+                    /**
+                     * @description AI graders only: the judge's binary verdict. insufficient_evidence means the judge declined to guess; it renders as a warning and never fails the case.
+                     * @enum {string}
+                     */
+                    verdict?: "pass" | "fail" | "insufficient_evidence";
                 }[];
                 /** @description Truncated snapshot of the output that was graded (2000 chars max). */
                 outputExcerpt: string;
@@ -39836,6 +39851,12 @@ interface components {
                 errored: boolean;
                 name: string;
                 outcomes: {
+                    /** @description AI graders only: transcript entries the verdict rests on — indices into the run's trace steps (message) or tool calls (tool_call). */
+                    evidence?: {
+                        index: number;
+                        /** @enum {string} */
+                        kind: "message" | "tool_call";
+                    }[];
                     graderIndex: number;
                     kind: string;
                     passed: boolean;
@@ -39846,6 +39867,11 @@ interface components {
                      * @enum {string}
                      */
                     severity?: "gate" | "soft";
+                    /**
+                     * @description AI graders only: the judge's binary verdict. insufficient_evidence means the judge declined to guess; it renders as a warning and never fails the case.
+                     * @enum {string}
+                     */
+                    verdict?: "pass" | "fail" | "insufficient_evidence";
                 }[];
                 outputExcerpt: string;
                 passed: boolean;
@@ -42396,6 +42422,8 @@ interface LogQueryResult {
         cursor: string | null;
         hasMore: boolean;
     };
+    /** True when historical (R2 SQL) logs were unavailable and only recent hot-tier entries are included. */
+    degraded?: boolean;
 }
 /** Logs are wrapped in a `{ success, data }` envelope by the API. */
 interface LogQueryResponse {
@@ -42414,6 +42442,8 @@ interface LogStatsResult {
         count: number;
         errors: number;
     }>;
+    /** True when historical (R2 SQL) counts were unavailable and stats cover only the recent hot-tier window. */
+    degraded?: boolean;
 }
 interface LogStatsResponse {
     success: boolean;
@@ -42621,8 +42651,17 @@ interface AIGrader {
     criteria: string;
     /** Reference-guided when the case has `expected`. */
     useExpected?: boolean;
-    /** Defaults to a cheap routed model at execution time. */
+    /** Defaults to a cheap routed model at execution time. Ignored when `judgeFlowId` is set. */
     model?: string;
+    /**
+     * Custom judge flow: your flow (by id) runs as the judge instead of the
+     * platform judge prompt. It receives `criteria`, `caseName`, `expected`, and
+     * `transcript` as inputs and must output the platform verdict JSON
+     * (`{"reasoning", "verdict": "pass"|"fail"|"insufficient_evidence", "evidence"}`).
+     * A miswired flow (missing, failing, or returning malformed output) resolves
+     * to an insufficient-evidence warning with the error, never a verdict.
+     */
+    judgeFlowId?: string;
     /** Pass cutoff on the 1-5 judge scale (default 4 server-side). Set via `.atLeast(n)`. */
     threshold?: number;
     /** Hard gate (default) vs soft. Set via `.gate()` / `.soft()`. */
@@ -42717,6 +42756,7 @@ declare function cost(maxUsd: number): Gradeable<CheckGrader>;
 declare function judge(criteria: string, opts?: {
     useExpected?: boolean;
     model?: string;
+    judgeFlowId?: string;
     threshold?: number;
     preset?: BuiltInGraderId;
 }): Gradeable<AIGrader>;
@@ -42852,10 +42892,24 @@ interface GraderOutcome {
     passed: boolean;
     /** The grader's severity (absent ⇒ gate). A soft miss only fails under `--strict`. */
     severity?: GraderSeverity;
-    /** 0..1 for scaled graders (AI graders normalize their 1-5 to 0..1). */
+    /** 0..1 for scaled graders (AI graders normalize their 1-5 to 0..1). Absent for `insufficient_evidence`. */
     score?: number;
-    /** AI-grader verdict, or a check's human-readable reason. */
+    /** AI-grader reasoning (critique-before-verdict), or a check's human-readable reason. */
     reasoning?: string;
+    /**
+     * AI graders only: the trajectory judge's binary verdict.
+     * `insufficient_evidence` means the judge declined to guess — it renders as
+     * a warning and never fails the case.
+     */
+    verdict?: 'pass' | 'fail' | 'insufficient_evidence';
+    /**
+     * AI graders only: transcript entries the verdict rests on — indices into
+     * the run's trace steps (`message`) or tool calls (`tool_call`).
+     */
+    evidence?: Array<{
+        kind: 'message' | 'tool_call';
+        index: number;
+    }>;
 }
 /** One case's run result: pass/fail plus each grader's outcome. */
 interface RunEvalCaseResult {

package/dist/index.mjs

@@ -2096,12 +2096,16 @@ function judge(criteria, opts) {
   if (typeof criteria !== "string" || criteria.trim().length === 0) {
     throw new Error("judge() requires non-empty criteria");
   }
+  if (opts?.judgeFlowId !== void 0 && opts.judgeFlowId.trim().length === 0) {
+    throw new Error("judge() requires a non-empty judgeFlowId when one is provided");
+  }
   return gradeable({
     kind: "ai",
     criteria,
     ...opts?.preset ? { preset: opts.preset } : {},
     ...opts?.useExpected ? { useExpected: true } : {},
     ...opts?.model ? { model: opts.model } : {},
+    ...opts?.judgeFlowId ? { judgeFlowId: opts.judgeFlowId } : {},
     ...opts?.threshold !== void 0 ? { threshold: opts.threshold } : {}
   });
 }
@@ -6208,7 +6212,7 @@ var Runtype = class {
 
 // src/version.ts
 var FALLBACK_VERSION = "0.0.0";
-var SDK_VERSION = "5.8.1".length > 0 ? "5.8.1" : FALLBACK_VERSION;
+var SDK_VERSION = "5.9.0".length > 0 ? "5.9.0" : FALLBACK_VERSION;
 var RUNTYPE_CLIENT_KIND = "sdk";
 var SDK_USER_AGENT = `runtype-sdk/${SDK_VERSION} (typescript)`;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runtypelabs/sdk",
-  "version": "5.8.1",
+  "version": "5.9.0",
   "type": "module",
   "description": "TypeScript SDK for the Runtype API with fluent methods. Use it to quickly realize AI products, agents, and workflows.",
   "main": "dist/index.cjs",