@runtypelabs/sdk 5.3.1 → 5.4.0

package/dist/index.cjs

@@ -99,12 +99,14 @@ __export(index_exports, {
   buildSendViewOffloadMarker: () => buildSendViewOffloadMarker,
   compileWorkflowConfig: () => compileWorkflowConfig,
   computeAgentContentHash: () => computeAgentContentHash,
+  computeEvalContentHash: () => computeEvalContentHash,
   computeFlowContentHash: () => computeFlowContentHash,
   computeFpoContentHash: () => computeFpoContentHash,
   computeProductContentHash: () => computeProductContentHash,
   computeSkillContentHash: () => computeSkillContentHash,
   computeSurfaceContentHash: () => computeSurfaceContentHash,
   computeToolContentHash: () => computeToolContentHash,
+  contains: () => contains,
   createAgentEventTranslator: () => createAgentEventTranslator,
   createClient: () => createClient,
   createExternalTool: () => createExternalTool,
@@ -112,6 +114,7 @@ __export(index_exports, {
   defaultWorkflow: () => defaultWorkflow,
   defaultWorkflowConfig: () => defaultWorkflowConfig,
   defineAgent: () => defineAgent,
+  defineEval: () => defineEval,
   defineFlow: () => defineFlow,
   defineFpo: () => defineFpo,
   definePlaybook: () => definePlaybook,
@@ -121,6 +124,7 @@ __export(index_exports, {
   defineTool: () => defineTool,
   deployWorkflow: () => deployWorkflow,
   ensureDefaultWorkflowHooks: () => ensureDefaultWorkflowHooks,
+  ensureEval: () => ensureEval,
   ensureFpo: () => ensureFpo,
   evaluateGeneratedRuntimeToolProposal: () => evaluateGeneratedRuntimeToolProposal,
   extractDeclaredToolResultChars: () => extractDeclaredToolResultChars,
@@ -133,7 +137,14 @@ __export(index_exports, {
   isPreservationSensitiveTask: () => isPreservationSensitiveTask,
   isUnifiedEventType: () => isUnifiedEventType,
   isWorkflowHookRef: () => isWorkflowHookRef,
+  jsonField: () => jsonField,
+  judge: () => judge,
+  judges: () => judges,
+  latency: () => latency,
+  length: () => length,
   listWorkflowHooks: () => listWorkflowHooks,
+  matchesExpected: () => matchesExpected,
+  noError: () => noError,
   normalizeAgentDefinition: () => normalizeAgentDefinition,
   normalizeCandidatePath: () => normalizeCandidatePath,
   normalizeFpoDefinition: () => normalizeFpoDefinition,
@@ -141,20 +152,25 @@ __export(index_exports, {
   normalizeSkillDefinition: () => normalizeSkillDefinition,
   normalizeSurfaceDefinition: () => normalizeSurfaceDefinition,
   normalizeToolDefinition: () => normalizeToolDefinition,
+  notContains: () => notContains,
   parseFinalBuffer: () => parseFinalBuffer,
   parseLedgerArtifactRelativePath: () => parseLedgerArtifactRelativePath,
   parseOffloadedOutputId: () => parseOffloadedOutputId,
   parseSSEChunk: () => parseSSEChunk,
   processStream: () => processStream,
+  pullEval: () => pullEval,
   pullFpo: () => pullFpo,
+  regex: () => regex,
   registerWorkflowHook: () => registerWorkflowHook,
   resolveStallStopAfter: () => resolveStallStopAfter,
   resolveWorkflowHook: () => resolveWorkflowHook,
+  runEvalSuite: () => runEvalSuite,
   sanitizeTaskSlug: () => sanitizeTaskSlug,
   shouldInjectEmptySessionNudge: () => shouldInjectEmptySessionNudge,
   shouldRequestModelEscalation: () => shouldRequestModelEscalation,
   streamEvents: () => streamEvents,
   unregisterWorkflowHook: () => unregisterWorkflowHook,
+  validJson: () => validJson,
   withUnifiedEvents: () => withUnifiedEvents
 });
 module.exports = __toCommonJS(index_exports);
@@ -3437,6 +3453,300 @@ var BatchesNamespace = class {
   }
 };
 
+// src/evals-ensure.ts
+var CHECK_GRADER_KINDS = /* @__PURE__ */ new Set([
+  "contains",
+  "not_contains",
+  "matches_expected",
+  "regex",
+  "valid_json",
+  "json_field",
+  "length",
+  "latency",
+  "no_error"
+]);
+function contains(value, opts) {
+  return { kind: "contains", value, ...opts?.caseSensitive ? { caseSensitive: true } : {} };
+}
+function notContains(value, opts) {
+  return { kind: "not_contains", value, ...opts?.caseSensitive ? { caseSensitive: true } : {} };
+}
+function matchesExpected() {
+  return { kind: "matches_expected" };
+}
+function regex(pattern, flags) {
+  return { kind: "regex", pattern, ...flags ? { flags } : {} };
+}
+function validJson() {
+  return { kind: "valid_json" };
+}
+function jsonField(path, opts) {
+  return {
+    kind: "json_field",
+    path,
+    ...opts && "equals" in opts && opts.equals !== void 0 ? { equals: opts.equals } : {},
+    ...opts && typeof opts.exists === "boolean" ? { exists: opts.exists } : {}
+  };
+}
+function length(opts) {
+  if (!opts || opts.minChars === void 0 && opts.maxChars === void 0) {
+    throw new Error("length() requires at least one of minChars or maxChars");
+  }
+  return {
+    kind: "length",
+    ...opts.minChars !== void 0 ? { minChars: opts.minChars } : {},
+    ...opts.maxChars !== void 0 ? { maxChars: opts.maxChars } : {}
+  };
+}
+function latency(maxMs) {
+  if (!Number.isFinite(maxMs) || maxMs <= 0) {
+    throw new Error("latency() requires a positive maxMs");
+  }
+  return { kind: "latency", maxMs };
+}
+function noError() {
+  return { kind: "no_error" };
+}
+function judge(criteria, opts) {
+  if (typeof criteria !== "string" || criteria.trim().length === 0) {
+    throw new Error("judge() requires non-empty criteria");
+  }
+  return {
+    kind: "ai",
+    criteria,
+    ...opts?.preset ? { preset: opts.preset } : {},
+    ...opts?.useExpected ? { useExpected: true } : {},
+    ...opts?.model ? { model: opts.model } : {},
+    ...opts?.threshold !== void 0 ? { threshold: opts.threshold } : {}
+  };
+}
+var judges = {
+  answersQuestion: () => judge(
+    "The response directly addresses what the user asked, without dodging or answering a different question.",
+    { preset: "answersQuestion" }
+  ),
+  matchesExpected: () => judge(
+    "The response conveys the same facts and conclusion as the expected answer. Wording may differ.",
+    { preset: "matchesExpected", useExpected: true }
+  ),
+  followsInstructions: () => judge(
+    "The response obeys every instruction in the system prompt (format, tone, constraints, refusals).",
+    { preset: "followsInstructions" }
+  ),
+  grounded: () => judge(
+    "Every factual claim in the response is supported by the provided context or the expected answer. Flag anything invented.",
+    { preset: "grounded" }
+  ),
+  rightTone: (voice = "{describe the voice you want}") => judge(`The response matches this voice: ${voice}.`, { preset: "rightTone" }),
+  safeToSend: () => judge(
+    "The response contains nothing embarrassing to show a customer: no leaked internals, no hostile tone, no policy violations.",
+    { preset: "safeToSend" }
+  )
+};
+var DEFINE_EVAL_TOP_LEVEL_KEYS = /* @__PURE__ */ new Set([
+  "name",
+  "target",
+  "graders",
+  "cases",
+  "virtual"
+]);
+var DEFINE_EVAL_CASE_KEYS = /* @__PURE__ */ new Set(["name", "input", "expected", "expect"]);
+function isPlainObject2(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function normalizeTarget(target) {
+  if (!isPlainObject2(target)) {
+    throw new Error('defineEval requires a "target" object: { flow: name } or { agent: name }');
+  }
+  const hasFlow = typeof target.flow === "string" && target.flow.length > 0;
+  const hasAgent = typeof target.agent === "string" && target.agent.length > 0;
+  if (hasFlow === hasAgent) {
+    throw new Error(
+      'defineEval "target" must name exactly one of flow or agent: { flow: "name" } XOR { agent: "name" }'
+    );
+  }
+  const extraKeys = Object.keys(target).filter((k) => k !== "flow" && k !== "agent");
+  if (extraKeys.length > 0) {
+    throw new Error(`defineEval "target" has unknown field(s): ${extraKeys.join(", ")}`);
+  }
+  return hasFlow ? { flow: target.flow } : { agent: target.agent };
+}
+function validateGrader(grader, where) {
+  if (!isPlainObject2(grader) || typeof grader.kind !== "string") {
+    throw new Error(`defineEval: ${where} must be a grader object with a string "kind"`);
+  }
+  if (grader.kind === "ai") {
+    if (typeof grader.criteria !== "string" || grader.criteria.trim().length === 0) {
+      throw new Error(`defineEval: ${where} is an AI grader and requires non-empty "criteria"`);
+    }
+    return grader;
+  }
+  if (!CHECK_GRADER_KINDS.has(grader.kind)) {
+    throw new Error(
+      `defineEval: ${where} has unknown grader kind "${grader.kind}". Known kinds: ${[...CHECK_GRADER_KINDS].join(", ")}, ai. (Trace graders such as called_tool/step_order are not available yet.)`
+    );
+  }
+  return grader;
+}
+function normalizeCaseInput(input, where) {
+  if (input === void 0) return {};
+  if (!isPlainObject2(input)) {
+    throw new Error(`defineEval: ${where} "input" must be an object`);
+  }
+  const out = {};
+  if (input.variables !== void 0) {
+    if (!isPlainObject2(input.variables)) {
+      throw new Error(`defineEval: ${where} "input.variables" must be an object`);
+    }
+    out.variables = input.variables;
+  }
+  if (input.messages !== void 0) {
+    if (!Array.isArray(input.messages)) {
+      throw new Error(`defineEval: ${where} "input.messages" must be an array`);
+    }
+    out.messages = input.messages.map((m, i) => {
+      if (!isPlainObject2(m) || typeof m.role !== "string" || typeof m.content !== "string") {
+        throw new Error(`defineEval: ${where} "input.messages[${i}]" must be { role, content }`);
+      }
+      return { role: m.role, content: m.content };
+    });
+  }
+  return out;
+}
+function defineEval(input) {
+  if (!input || typeof input !== "object") {
+    throw new Error("defineEval requires a definition object");
+  }
+  const unknownKeys = Object.keys(input).filter((k) => !DEFINE_EVAL_TOP_LEVEL_KEYS.has(k));
+  if (unknownKeys.length > 0) {
+    throw new Error(
+      `defineEval: unknown field(s): ${unknownKeys.join(", ")}. Allowed fields are target, graders, cases, virtual.`
+    );
+  }
+  const target = normalizeTarget(input.target);
+  if (input.name !== void 0 && (typeof input.name !== "string" || input.name.length === 0)) {
+    throw new Error('defineEval "name" must be a non-empty string when provided');
+  }
+  const name = input.name ?? ("flow" in target ? `flow:${target.flow}` : `agent:${target.agent}`);
+  const suiteGraders = (input.graders ?? []).map((g, i) => validateGrader(g, `graders[${i}]`));
+  if (!Array.isArray(input.cases) || input.cases.length === 0) {
+    throw new Error('defineEval requires a non-empty "cases" array');
+  }
+  const seenNames = /* @__PURE__ */ new Set();
+  const cases = input.cases.map((c, index) => {
+    if (!isPlainObject2(c)) {
+      throw new Error(`defineEval: cases[${index}] must be an object`);
+    }
+    if (typeof c.name !== "string" || c.name.length === 0) {
+      throw new Error(`defineEval: cases[${index}] requires a non-empty string "name"`);
+    }
+    if (seenNames.has(c.name)) {
+      throw new Error(`defineEval: duplicate case name "${c.name}" (case names are the identity)`);
+    }
+    seenNames.add(c.name);
+    const unknownCaseKeys = Object.keys(c).filter((k) => !DEFINE_EVAL_CASE_KEYS.has(k));
+    if (unknownCaseKeys.length > 0) {
+      throw new Error(
+        `defineEval: cases[${index}] ("${c.name}") has unknown field(s): ${unknownCaseKeys.join(
+          ", "
+        )}. Allowed case fields are name, input, expected, expect.`
+      );
+    }
+    const caseGraders = (c.expect ?? []).map(
+      (g, i) => validateGrader(g, `cases[${index}].expect[${i}]`)
+    );
+    const expect = [...suiteGraders, ...caseGraders];
+    if (expect.length === 0) {
+      throw new Error(
+        `defineEval: cases[${index}] ("${c.name}") has no graders. Add suite-level "graders" or case-level "expect" so there is something to score.`
+      );
+    }
+    if (c.expected !== void 0 && !isPlainObject2(c.expected)) {
+      throw new Error(`defineEval: cases[${index}] ("${c.name}") "expected" must be an object`);
+    }
+    return {
+      name: c.name,
+      input: normalizeCaseInput(c.input, `cases[${index}] ("${c.name}")`),
+      ...c.expected !== void 0 ? { expected: c.expected } : {},
+      expect
+    };
+  });
+  return { name, target, cases, virtual: input.virtual === true };
+}
+function normalizeForHash(value) {
+  if (Array.isArray(value)) return value.map(normalizeForHash);
+  if (isPlainObject2(value)) {
+    const out = {};
+    for (const key of Object.keys(value).sort()) {
+      const v = value[key];
+      if (v === void 0) continue;
+      out[key] = normalizeForHash(v);
+    }
+    return out;
+  }
+  return value;
+}
+async function computeEvalContentHash(definition) {
+  const canonical = {
+    target: normalizeForHash(definition.target),
+    virtual: definition.virtual,
+    cases: [...definition.cases].sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0).map((c) => ({
+      name: c.name,
+      input: normalizeForHash(c.input),
+      ...c.expected !== void 0 ? { expected: normalizeForHash(c.expected) } : {},
+      // Grader order preserved on purpose (it maps to the result index).
+      expect: c.expect.map((g) => normalizeForHash(g))
+    }))
+  };
+  const serialized = JSON.stringify(canonical);
+  const encoded = new TextEncoder().encode(serialized);
+  const hashBuffer = await crypto.subtle.digest("SHA-256", encoded);
+  return Array.from(new Uint8Array(hashBuffer)).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+var serverHashMemo2 = /* @__PURE__ */ new WeakMap();
+function memoFor2(client) {
+  let memo = serverHashMemo2.get(client);
+  if (!memo) {
+    memo = /* @__PURE__ */ new Map();
+    serverHashMemo2.set(client, memo);
+  }
+  return memo;
+}
+async function ensureEval(client, definition) {
+  if (definition.virtual) {
+    throw new Error(
+      "Cannot ensure a virtual eval: virtual evals are ephemeral (nothing is persisted to converge). Remove `virtual: true` to converge a durable suite, or run it directly."
+    );
+  }
+  const memo = memoFor2(client);
+  const localHash = await computeEvalContentHash(definition);
+  const memoKey = `${definition.name} ${localHash}`;
+  const contentHash = memo.get(memoKey) ?? localHash;
+  const probe = await client.post(
+    "/eval/ensure",
+    { name: definition.name, contentHash }
+  );
+  if (probe.result !== "definitionRequired") {
+    memo.set(memoKey, probe.contentHash);
+    return probe;
+  }
+  const converged = await client.post(
+    "/eval/ensure",
+    { name: definition.name, definition }
+  );
+  if (converged.result === "definitionRequired") {
+    throw new Error("Server reported definitionRequired for a full-definition request");
+  }
+  memo.set(memoKey, converged.contentHash);
+  return converged;
+}
+async function pullEval(client, name) {
+  return client.get("/eval/pull", { name });
+}
+async function runEvalSuite(client, input) {
+  return client.post("/eval/run", input);
+}
+
 // src/evals-namespace.ts
 var EvalRunner = class {
   constructor(getClient, config) {
@@ -3581,6 +3891,49 @@ var EvalsNamespace = class {
   run(config) {
     return new EvalRunner(this.getClient, config);
   }
+  /**
+   * Idempotently converge a `defineEval` suite definition onto the platform —
+   * the deploy-time, non-executing converge for code-colocated evals. Hash-first:
+   * the steady state is one tiny probe request. Upserts the eval suite + replaces
+   * its cases; never runs the eval.
+   *
+   * @example
+   * ```typescript
+   * const suite = defineEval({
+   *   target: { flow: 'support-triage' },
+   *   graders: [noError()],
+   *   cases: [{ name: 'billing', input: { variables: { message: 'I was double charged' } }, expect: [contains('finance')] }],
+   * })
+   * const result = await Runtype.evals.ensure(suite)
+   * ```
+   */
+  async ensure(definition) {
+    return ensureEval(this.getClient(), definition);
+  }
+  /**
+   * Pull the canonical definition + provenance for an eval suite by name — the
+   * absorb-drift direction of the ensure protocol.
+   */
+  async pull(name) {
+    return pullEval(this.getClient(), name);
+  }
+  /**
+   * Run an eval suite synchronously and return the suite score + per-case grader
+   * outcomes — the executing counterpart of `ensure`, powering the `runtype
+   * eval` CI gate. Run a saved suite by id (`{ suiteId }`, the post-`ensure`
+   * path) or an inline definition without persisting (`{ definition }`, the
+   * virtual path).
+   *
+   * @example
+   * ```typescript
+   * const { suiteId } = await Runtype.evals.ensure(suite)
+   * const result = await Runtype.evals.runSuite({ suiteId })
+   * if (!result.passed) process.exit(1)
+   * ```
+   */
+  async runSuite(input) {
+    return runEvalSuite(this.getClient(), input);
+  }
   /**
    * Get evaluation status by ID
    *
@@ -3699,14 +4052,14 @@ var PromptsNamespace = class {
 };
 
 // src/skills-ensure.ts
-function isPlainObject2(value) {
+function isPlainObject3(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 function normalizeValue(value) {
   if (Array.isArray(value)) {
     return value.map((item) => normalizeValue(item));
   }
-  if (isPlainObject2(value)) {
+  if (isPlainObject3(value)) {
     const normalized = {};
     for (const key of Object.keys(value).sort()) {
       const entry = value[key];
@@ -3718,15 +4071,15 @@ function normalizeValue(value) {
   return value;
 }
 function normalizeSkillDefinition(definition) {
-  const manifest = isPlainObject2(definition.manifest) ? definition.manifest : {};
-  const rawFrontmatter = isPlainObject2(manifest.frontmatter) ? manifest.frontmatter : {};
+  const manifest = isPlainObject3(definition.manifest) ? definition.manifest : {};
+  const rawFrontmatter = isPlainObject3(manifest.frontmatter) ? manifest.frontmatter : {};
   const frontmatterWithoutName = {};
   for (const key of Object.keys(rawFrontmatter)) {
     if (key === "name") continue;
     frontmatterWithoutName[key] = rawFrontmatter[key];
   }
   const frontmatter = normalizeValue(frontmatterWithoutName);
-  const runtype = isPlainObject2(manifest.runtype) ? normalizeValue(manifest.runtype) : {};
+  const runtype = isPlainObject3(manifest.runtype) ? normalizeValue(manifest.runtype) : {};
   const body = typeof manifest.body === "string" ? manifest.body : "";
   return { frontmatter, runtype, body };
 }
@@ -3744,7 +4097,7 @@ function defineSkill(input) {
   if (typeof input.name !== "string" || input.name.length === 0) {
     throw new Error('defineSkill requires a non-empty string "name"');
   }
-  if (!isPlainObject2(input.manifest)) {
+  if (!isPlainObject3(input.manifest)) {
     throw new Error('defineSkill requires a "manifest" object ({ frontmatter, runtype, body })');
   }
   const unknownKeys = Object.keys(input).filter((key) => !DEFINE_SKILL_TOP_LEVEL_KEYS.has(key));
@@ -3754,7 +4107,7 @@ function defineSkill(input) {
     );
   }
   const frontmatter = input.manifest.frontmatter;
-  if (!isPlainObject2(frontmatter) || typeof frontmatter.name !== "string") {
+  if (!isPlainObject3(frontmatter) || typeof frontmatter.name !== "string") {
     throw new Error("defineSkill: manifest.frontmatter.name is required");
   }
   if (frontmatter.name !== input.name) {
@@ -3795,19 +4148,19 @@ function parseRequestError2(err) {
 }
 function toConflictError2(err) {
   const { status, body } = parseRequestError2(err);
-  if (status !== 409 || !isPlainObject2(body)) return null;
+  if (status !== 409 || !isPlainObject3(body)) return null;
   const code = body.code;
   if (code !== "external_modification" && code !== "remote_changed") return null;
   return new SkillEnsureConflictError(
     body
   );
 }
-var serverHashMemo2 = /* @__PURE__ */ new WeakMap();
-function memoFor2(client) {
-  let memo = serverHashMemo2.get(client);
+var serverHashMemo3 = /* @__PURE__ */ new WeakMap();
+function memoFor3(client) {
+  let memo = serverHashMemo3.get(client);
   if (!memo) {
     memo = /* @__PURE__ */ new Map();
-    serverHashMemo2.set(client, memo);
+    serverHashMemo3.set(client, memo);
   }
   return memo;
 }
@@ -3848,7 +4201,7 @@ async function ensureSkill(client, definition, options = {}) {
     }
     return plan;
   }
-  const memo = memoFor2(client);
+  const memo = memoFor3(client);
   const localHash = await computeSkillContentHash(definition);
   const memoKey = `${definition.name} ${localHash}`;
   const contentHash = memo.get(memoKey) ?? localHash;
@@ -4143,14 +4496,14 @@ var AGENT_CONFIG_KEYS = [
   "tenancyStrategy"
 ];
 var AGENT_CONFIG_KEY_LIST = [...AGENT_CONFIG_KEYS].sort();
-function isPlainObject3(value) {
+function isPlainObject4(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 function normalizeValue2(value) {
   if (Array.isArray(value)) {
     return value.map((item) => normalizeValue2(item));
   }
-  if (isPlainObject3(value)) {
+  if (isPlainObject4(value)) {
     const normalized = {};
     for (const key of Object.keys(value).sort()) {
       const entry = value[key];
@@ -4163,7 +4516,7 @@ function normalizeValue2(value) {
 }
 function normalizeAgentDefinition(definition) {
   const config = {};
-  const rawConfig = isPlainObject3(definition.config) ? definition.config : {};
+  const rawConfig = isPlainObject4(definition.config) ? definition.config : {};
   for (const key of AGENT_CONFIG_KEY_LIST) {
     const value = rawConfig[key];
     if (value === void 0 || value === null) continue;
@@ -4185,7 +4538,7 @@ async function computeAgentContentHash(definition) {
 var DEFINE_TOP_LEVEL_KEYS = /* @__PURE__ */ new Set(["name", "description", "icon", ...AGENT_CONFIG_KEYS]);
 function collectNonPortableToolRefs(config) {
   const tools = config.tools;
-  if (!isPlainObject3(tools)) return [];
+  if (!isPlainObject4(tools)) return [];
   const found = [];
   const isAccountScoped = (ref) => typeof ref === "string" && ref.startsWith("tool_");
   const scanArray = (value, path) => {
@@ -4195,7 +4548,7 @@ function collectNonPortableToolRefs(config) {
     });
   };
   const scanKeys = (value, path) => {
-    if (!isPlainObject3(value)) return;
+    if (!isPlainObject4(value)) return;
     for (const key of Object.keys(value)) {
       if (isAccountScoped(key)) found.push(`${path}.${key}`);
     }
@@ -4203,16 +4556,16 @@ function collectNonPortableToolRefs(config) {
   scanArray(tools.toolIds, "tools.toolIds");
   scanKeys(tools.toolConfigs, "tools.toolConfigs");
   scanKeys(tools.perToolLimits, "tools.perToolLimits");
-  if (isPlainObject3(tools.approval)) scanArray(tools.approval.require, "tools.approval.require");
-  if (isPlainObject3(tools.subagentConfig)) {
+  if (isPlainObject4(tools.approval)) scanArray(tools.approval.require, "tools.approval.require");
+  if (isPlainObject4(tools.subagentConfig)) {
     scanArray(tools.subagentConfig.toolPool, "tools.subagentConfig.toolPool");
   }
-  if (isPlainObject3(tools.codeModeConfig)) {
+  if (isPlainObject4(tools.codeModeConfig)) {
     scanArray(tools.codeModeConfig.toolPool, "tools.codeModeConfig.toolPool");
   }
   if (Array.isArray(tools.runtimeTools)) {
     tools.runtimeTools.forEach((runtimeTool, i) => {
-      if (!isPlainObject3(runtimeTool) || !isPlainObject3(runtimeTool.config)) return;
+      if (!isPlainObject4(runtimeTool) || !isPlainObject4(runtimeTool.config)) return;
       const base = `tools.runtimeTools[${i}].config`;
       const rtConfig = runtimeTool.config;
       if (runtimeTool.toolType === "subagent" && typeof rtConfig.agentId === "string" && rtConfig.agentId.startsWith("agent_")) {
@@ -4286,19 +4639,19 @@ function parseRequestError3(err) {
 }
 function toConflictError3(err) {
   const { status, body } = parseRequestError3(err);
-  if (status !== 409 || !isPlainObject3(body)) return null;
+  if (status !== 409 || !isPlainObject4(body)) return null;
   const code = body.code;
   if (code !== "external_modification" && code !== "remote_changed") return null;
   return new AgentEnsureConflictError(
     body
   );
 }
-var serverHashMemo3 = /* @__PURE__ */ new WeakMap();
-function memoFor3(client) {
-  let memo = serverHashMemo3.get(client);
+var serverHashMemo4 = /* @__PURE__ */ new WeakMap();
+function memoFor4(client) {
+  let memo = serverHashMemo4.get(client);
   if (!memo) {
     memo = /* @__PURE__ */ new Map();
-    serverHashMemo3.set(client, memo);
+    serverHashMemo4.set(client, memo);
   }
   return memo;
 }
@@ -4335,7 +4688,7 @@ var AgentsNamespace = class {
       }
       return plan;
     }
-    const memo = memoFor3(client);
+    const memo = memoFor4(client);
     const localHash = await computeAgentContentHash({
       ...definition,
       config: definition.config
@@ -4388,14 +4741,14 @@ var AgentsNamespace = class {
 };
 
 // src/tools-ensure.ts
-function isPlainObject4(value) {
+function isPlainObject5(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 function normalizeValue3(value) {
   if (Array.isArray(value)) {
     return value.map((item) => normalizeValue3(item));
   }
-  if (isPlainObject4(value)) {
+  if (isPlainObject5(value)) {
     const normalized = {};
     for (const key of Object.keys(value).sort()) {
       const entry = value[key];
@@ -4407,8 +4760,8 @@ function normalizeValue3(value) {
   return value;
 }
 function normalizeToolDefinition(definition) {
-  const parametersSchema = isPlainObject4(definition.parametersSchema) ? normalizeValue3(definition.parametersSchema) : {};
-  const config = isPlainObject4(definition.config) ? normalizeValue3(definition.config) : {};
+  const parametersSchema = isPlainObject5(definition.parametersSchema) ? normalizeValue3(definition.parametersSchema) : {};
+  const config = isPlainObject5(definition.config) ? normalizeValue3(definition.config) : {};
   return {
     toolType: definition.toolType,
     ...definition.description ? { description: definition.description } : {},
@@ -4453,10 +4806,10 @@ function defineTool(input) {
       `defineTool requires "toolType" to be one of: ${[...TOOL_DEFINITION_TYPES].join(", ")}`
     );
   }
-  if (!isPlainObject4(input.parametersSchema)) {
+  if (!isPlainObject5(input.parametersSchema)) {
     throw new Error('defineTool requires a "parametersSchema" object (a JSON Schema)');
   }
-  if (!isPlainObject4(input.config)) {
+  if (!isPlainObject5(input.config)) {
     throw new Error('defineTool requires a "config" object');
   }
   const unknownKeys = Object.keys(input).filter((key) => !DEFINE_TOOL_TOP_LEVEL_KEYS.has(key));
@@ -4504,19 +4857,19 @@ function parseRequestError4(err) {
 }
 function toConflictError4(err) {
   const { status, body } = parseRequestError4(err);
-  if (status !== 409 || !isPlainObject4(body)) return null;
+  if (status !== 409 || !isPlainObject5(body)) return null;
   const code = body.code;
   if (code !== "external_modification" && code !== "remote_changed") return null;
   return new ToolEnsureConflictError(
     body
   );
 }
-var serverHashMemo4 = /* @__PURE__ */ new WeakMap();
-function memoFor4(client) {
-  let memo = serverHashMemo4.get(client);
+var serverHashMemo5 = /* @__PURE__ */ new WeakMap();
+function memoFor5(client) {
+  let memo = serverHashMemo5.get(client);
   if (!memo) {
     memo = /* @__PURE__ */ new Map();
-    serverHashMemo4.set(client, memo);
+    serverHashMemo5.set(client, memo);
   }
   return memo;
 }
@@ -4556,7 +4909,7 @@ async function ensureTool(client, definition, options = {}) {
     }
     return plan;
   }
-  const memo = memoFor4(client);
+  const memo = memoFor5(client);
   const localHash = await computeToolContentHash(definition);
   const memoKey = `${definition.name} ${localHash}`;
   const contentHash = memo.get(memoKey) ?? localHash;
@@ -4624,14 +4977,14 @@ var ToolsNamespace = class {
 };
 
 // src/products-ensure.ts
-function isPlainObject5(value) {
+function isPlainObject6(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 function normalizeValue4(value) {
   if (Array.isArray(value)) {
     return value.map((item) => normalizeValue4(item));
   }
-  if (isPlainObject5(value)) {
+  if (isPlainObject6(value)) {
     const normalized = {};
     for (const key of Object.keys(value).sort()) {
       const entry = value[key];
@@ -4643,7 +4996,7 @@ function normalizeValue4(value) {
   return value;
 }
 function normalizeProductDefinition(definition) {
-  const spec = isPlainObject5(definition.spec) ? normalizeValue4(definition.spec) : {};
+  const spec = isPlainObject6(definition.spec) ? normalizeValue4(definition.spec) : {};
   return {
     ...definition.description ? { description: definition.description } : {},
     ...definition.icon ? { icon: definition.icon } : {},
@@ -4670,7 +5023,7 @@ function defineProduct(input) {
   if (input.icon != null && typeof input.icon !== "string") {
     throw new Error('defineProduct "icon" must be a string when provided');
   }
-  if (input.spec != null && !isPlainObject5(input.spec)) {
+  if (input.spec != null && !isPlainObject6(input.spec)) {
     throw new Error('defineProduct "spec" must be an object when provided');
   }
   const unknownKeys = Object.keys(input).filter((key) => !DEFINE_PRODUCT_TOP_LEVEL_KEYS.has(key));
@@ -4717,19 +5070,19 @@ function parseRequestError5(err) {
 }
 function toConflictError5(err) {
   const { status, body } = parseRequestError5(err);
-  if (status !== 409 || !isPlainObject5(body)) return null;
+  if (status !== 409 || !isPlainObject6(body)) return null;
   const code = body.code;
   if (code !== "external_modification" && code !== "remote_changed") return null;
   return new ProductEnsureConflictError(
     body
   );
 }
-var serverHashMemo5 = /* @__PURE__ */ new WeakMap();
-function memoFor5(client) {
-  let memo = serverHashMemo5.get(client);
+var serverHashMemo6 = /* @__PURE__ */ new WeakMap();
+function memoFor6(client) {
+  let memo = serverHashMemo6.get(client);
   if (!memo) {
     memo = /* @__PURE__ */ new Map();
-    serverHashMemo5.set(client, memo);
+    serverHashMemo6.set(client, memo);
   }
   return memo;
 }
@@ -4769,7 +5122,7 @@ async function ensureProduct(client, definition, options = {}) {
     }
     return plan;
   }
-  const memo = memoFor5(client);
+  const memo = memoFor6(client);
   const localHash = await computeProductContentHash(definition);
   const memoKey = `${definition.name} ${localHash}`;
   const contentHash = memo.get(memoKey) ?? localHash;
@@ -4798,14 +5151,14 @@ async function pullProduct(client, name) {
 }
 
 // src/products-ensure-fpo.ts
-function isPlainObject6(value) {
+function isPlainObject7(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 function normalizeValue5(value) {
   if (Array.isArray(value)) {
     return value.map((item) => normalizeValue5(item));
   }
-  if (isPlainObject6(value)) {
+  if (isPlainObject7(value)) {
     const normalized = {};
     for (const key of Object.keys(value).sort()) {
       const entry = value[key];
@@ -4817,7 +5170,7 @@ function normalizeValue5(value) {
   return value;
 }
 function normalizeFpoDefinition(fpo) {
-  const productInput = isPlainObject6(fpo.product) ? fpo.product : {};
+  const productInput = isPlainObject7(fpo.product) ? fpo.product : {};
   const { name: _identityName, ...productRest } = productInput;
   const product = normalizeValue5(productRest);
   return {
@@ -4838,10 +5191,10 @@ async function computeFpoContentHash(fpo) {
   return Array.from(new Uint8Array(hashBuffer)).map((b) => b.toString(16).padStart(2, "0")).join("");
 }
 function defineFpo(fpo) {
-  if (!isPlainObject6(fpo)) {
+  if (!isPlainObject7(fpo)) {
     throw new Error("defineFpo requires an FPO object");
   }
-  const product = isPlainObject6(fpo.product) ? fpo.product : void 0;
+  const product = isPlainObject7(fpo.product) ? fpo.product : void 0;
   if (!product || typeof product.name !== "string" || product.name.length === 0) {
     throw new Error('defineFpo requires a non-empty "product.name" (the converge identity)');
   }
@@ -4937,14 +5290,14 @@ var ProductsNamespace = class {
 };
 
 // src/surfaces-ensure.ts
-function isPlainObject7(value) {
+function isPlainObject8(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 function normalizeValue6(value) {
   if (Array.isArray(value)) {
     return value.map((item) => normalizeValue6(item));
   }
-  if (isPlainObject7(value)) {
+  if (isPlainObject8(value)) {
     const normalized = {};
     for (const key of Object.keys(value).sort()) {
       const entry = value[key];
@@ -4956,7 +5309,7 @@ function normalizeValue6(value) {
   return value;
 }
 function normalizeSurfaceDefinition(definition) {
-  const behavior = isPlainObject7(definition.behavior) ? normalizeValue6(definition.behavior) : { type: definition.type };
+  const behavior = isPlainObject8(definition.behavior) ? normalizeValue6(definition.behavior) : { type: definition.type };
   return {
     type: definition.type,
     behavior,
@@ -5009,13 +5362,13 @@ function defineSurface(input) {
       `defineSurface requires "type" to be one of: ${[...SURFACE_DEFINITION_TYPES].join(", ")}`
     );
   }
-  if (input.behavior !== void 0 && !isPlainObject7(input.behavior)) {
+  if (input.behavior !== void 0 && !isPlainObject8(input.behavior)) {
     throw new Error('defineSurface "behavior" must be an object when provided');
   }
-  if (input.inbound !== void 0 && !isPlainObject7(input.inbound)) {
+  if (input.inbound !== void 0 && !isPlainObject8(input.inbound)) {
     throw new Error('defineSurface "inbound" must be an object when provided');
   }
-  if (input.outbound !== void 0 && !isPlainObject7(input.outbound)) {
+  if (input.outbound !== void 0 && !isPlainObject8(input.outbound)) {
     throw new Error('defineSurface "outbound" must be an object when provided');
   }
   if (input.status !== void 0 && !["draft", "active", "paused"].includes(input.status)) {
@@ -5071,19 +5424,19 @@ function parseRequestError6(err) {
 }
 function toConflictError6(err) {
   const { status, body } = parseRequestError6(err);
-  if (status !== 409 || !isPlainObject7(body)) return null;
+  if (status !== 409 || !isPlainObject8(body)) return null;
   const code = body.code;
   if (code !== "external_modification" && code !== "remote_changed") return null;
   return new SurfaceEnsureConflictError(
     body
   );
 }
-var serverHashMemo6 = /* @__PURE__ */ new WeakMap();
-function memoFor6(client) {
-  let memo = serverHashMemo6.get(client);
+var serverHashMemo7 = /* @__PURE__ */ new WeakMap();
+function memoFor7(client) {
+  let memo = serverHashMemo7.get(client);
   if (!memo) {
     memo = /* @__PURE__ */ new Map();
-    serverHashMemo6.set(client, memo);
+    serverHashMemo7.set(client, memo);
   }
   return memo;
 }
@@ -5123,7 +5476,7 @@ async function ensureSurface(client, productId, definition, options = {}) {
     }
     return plan;
   }
-  const memo = memoFor6(client);
+  const memo = memoFor7(client);
   const localHash = await computeSurfaceContentHash(definition);
   const memoKey = `${productId} ${definition.name} ${localHash}`;
   const contentHash = memo.get(memoKey) ?? localHash;
@@ -5672,7 +6025,7 @@ var Runtype = class {
 
 // src/version.ts
 var FALLBACK_VERSION = "0.0.0";
-var SDK_VERSION = "5.3.1".length > 0 ? "5.3.1" : FALLBACK_VERSION;
+var SDK_VERSION = "5.4.0".length > 0 ? "5.4.0" : FALLBACK_VERSION;
 var RUNTYPE_CLIENT_KIND = "sdk";
 var SDK_USER_AGENT = `runtype-sdk/${SDK_VERSION} (typescript)`;
 
@@ -13385,12 +13738,14 @@ var STEP_TYPE_TO_METHOD = {
   buildSendViewOffloadMarker,
   compileWorkflowConfig,
   computeAgentContentHash,
+  computeEvalContentHash,
   computeFlowContentHash,
   computeFpoContentHash,
   computeProductContentHash,
   computeSkillContentHash,
   computeSurfaceContentHash,
   computeToolContentHash,
+  contains,
   createAgentEventTranslator,
   createClient,
   createExternalTool,
@@ -13398,6 +13753,7 @@ var STEP_TYPE_TO_METHOD = {
   defaultWorkflow,
   defaultWorkflowConfig,
   defineAgent,
+  defineEval,
   defineFlow,
   defineFpo,
   definePlaybook,
@@ -13407,6 +13763,7 @@ var STEP_TYPE_TO_METHOD = {
   defineTool,
   deployWorkflow,
   ensureDefaultWorkflowHooks,
+  ensureEval,
   ensureFpo,
   evaluateGeneratedRuntimeToolProposal,
   extractDeclaredToolResultChars,
@@ -13419,7 +13776,14 @@ var STEP_TYPE_TO_METHOD = {
   isPreservationSensitiveTask,
   isUnifiedEventType,
   isWorkflowHookRef,
+  jsonField,
+  judge,
+  judges,
+  latency,
+  length,
   listWorkflowHooks,
+  matchesExpected,
+  noError,
   normalizeAgentDefinition,
   normalizeCandidatePath,
   normalizeFpoDefinition,
@@ -13427,19 +13791,24 @@ var STEP_TYPE_TO_METHOD = {
   normalizeSkillDefinition,
   normalizeSurfaceDefinition,
   normalizeToolDefinition,
+  notContains,
   parseFinalBuffer,
   parseLedgerArtifactRelativePath,
   parseOffloadedOutputId,
   parseSSEChunk,
   processStream,
+  pullEval,
   pullFpo,
+  regex,
   registerWorkflowHook,
   resolveStallStopAfter,
   resolveWorkflowHook,
+  runEvalSuite,
   sanitizeTaskSlug,
   shouldInjectEmptySessionNudge,
   shouldRequestModelEscalation,
   streamEvents,
   unregisterWorkflowHook,
+  validJson,
   withUnifiedEvents
 });