@foldrun/core 2.3.0 → 2.4.0

package/dist/src/evals.js

@@ -30,7 +30,7 @@
 // Deterministic checks run first and cost nothing. The judge only runs if
 // they pass — no point paying a model to grade output already known to be
 // wrong.
-import { accountModelEnv } from "./model-env.js";
+import { accountModelEnv, withModelEnv } from "./model-env.js";
 import { sdkEnv } from "./step-exec.js";
 import fs from "node:fs";
 import path from "node:path";
@@ -382,7 +382,7 @@ async function judge(rubric, output, model, effort, env) {
     }
     return text.trim();
 }
-async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => sdkEnv(undefined)) {
+async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: sdkEnv(undefined) })) {
     const hay = output.toLowerCase();
     switch (assertion.type) {
         case "contains": {
@@ -417,14 +417,14 @@ async function checkAssertion(assertion, output, agentDir, model, effort, judgeE
         case "judge": {
             // The judge runs on the account's own model key, never quietly on the
             // platform's: a customer's eval is not the operator's bill.
-            let env;
+            let m;
             try {
-                env = judgeEnv();
+                m = judgeEnv();
             }
             catch (err) {
                 return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
             }
-            const verdict = await judge(assertion.value, output, model, effort, env);
+            const verdict = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
             const passed = /^\s*PASS\b/i.test(verdict);
             return { assertion, passed, detail: verdict.slice(0, 300) || "judge returned nothing" };
         }
@@ -495,13 +495,7 @@ export async function runEval(tenant, workspace, info) {
             for (const a of cheap)
                 assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort));
             if (assertions.every((r) => r.passed)) {
-                const judgeEnv = () => {
-                    const m = accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
-                    if (m.format !== "anthropic") {
-                        throw new Error(`this account's provider speaks ${m.format}; a judge needs an Anthropic-format provider`);
-                    }
-                    return m.env;
-                };
+                const judgeEnv = () => accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
                 for (const a of judges)
                     assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort, judgeEnv));
             }

package/dist/src/model-env.d.ts

@@ -1,3 +1,4 @@
+import { type TranslatorSpec } from "./translator.ts";
 /** Refused for want of a usable key; `status` is what an HTTP layer answers. */
 export declare class ModelKeyError extends Error {
     readonly status: number;
@@ -7,11 +8,22 @@ export declare const NO_MODEL_KEY: string;
 /**
  * `platform` is a function: the caller's own way to build the platform's env
  * (a step's, or the SDK's host env), called only when this account may use
- * it. `format` says what the endpoint speaks — a caller with no translator
- * refuses anything but anthropic.
+ * it. `format` says what the endpoint speaks; `translator` is set when it is
+ * not Anthropic's, and `withModelEnv` puts the step's translator in front of
+ * it, so a judge or a draft works on any key a run works on (FOL-16).
  */
 export declare function accountModelEnv(tenant: string, workspace: string, platformEnv: () => Record<string, string | undefined>): {
     env: Record<string, string | undefined>;
     supply: "provider" | "platform";
     format: string;
+    translator: TranslatorSpec | null;
 };
+/**
+ * Run `fn` with the model env, through a translator on this host when the
+ * provider speaks Chat Completions or Responses — the same one a step uses
+ * (runner.ts), started for this call and closed after it.
+ */
+export declare function withModelEnv<T>(m: {
+    env: Record<string, string | undefined>;
+    translator?: TranslatorSpec | null;
+}, fn: (env: Record<string, string | undefined>) => Promise<T>): Promise<T>;

package/dist/src/model-env.js

@@ -10,6 +10,8 @@ import { readAgentsMd } from "./runner.js";
 import { getSecret } from "./secrets.js";
 import { platform } from "./platform.js";
 import { accountModelBlock, isClaudeLoginToken } from "./model-credential.js";
+import { providerPreset } from "./providers.js";
+import { startTranslator, translatorSpecFor } from "./translator.js";
 /** Refused for want of a usable key; `status` is what an HTTP layer answers. */
 export class ModelKeyError extends Error {
     status;
@@ -23,8 +25,9 @@ export const NO_MODEL_KEY = "no model key: this account runs on its own API key,
 /**
  * `platform` is a function: the caller's own way to build the platform's env
  * (a step's, or the SDK's host env), called only when this account may use
- * it. `format` says what the endpoint speaks — a caller with no translator
- * refuses anything but anthropic.
+ * it. `format` says what the endpoint speaks; `translator` is set when it is
+ * not Anthropic's, and `withModelEnv` puts the step's translator in front of
+ * it, so a judge or a draft works on any key a run works on (FOL-16).
  */
 export function accountModelEnv(tenant, workspace, platformEnv) {
     const raw = readAgentsMd(workspaceDir(tenant, workspace))?.data?.provider ??
@@ -34,7 +37,7 @@ export function accountModelEnv(tenant, workspace, platformEnv) {
     if (!spec?.baseUrl) {
         if (!platform.platformKeyAllowed(tenant))
             throw new ModelKeyError(NO_MODEL_KEY);
-        return { env: platformEnv(), supply: "platform", format: "anthropic" };
+        return { env: platformEnv(), supply: "platform", format: "anthropic", translator: null };
     }
     const sub = (text) => text.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => getSecret(tenant, name, workspace)?.value ?? whole);
     const token = sub(spec.token);
@@ -43,15 +46,37 @@ export function accountModelEnv(tenant, workspace, platformEnv) {
     if (!platform.platformKeyAllowed(tenant) && isClaudeLoginToken(token)) {
         throw new ModelKeyError("model key: a Claude login token (sk-ant-oat…) cannot be used here — use an API key from your provider", 422);
     }
+    const headers = Object.fromEntries(Object.entries(spec.headers).map(([k, v]) => [k, sub(v)]));
+    const preset = providerPreset(spec.name);
     return {
-        env: providerEnvFor({
+        env: providerEnvFor({ baseUrl: spec.baseUrl, token, auth: spec.auth, models: spec.models, headers }),
+        supply: "provider",
+        format: spec.format,
+        translator: translatorSpecFor({
+            format: spec.format,
             baseUrl: spec.baseUrl,
             token,
-            auth: spec.auth,
-            models: spec.models,
-            headers: Object.fromEntries(Object.entries(spec.headers).map(([k, v]) => [k, sub(v)])),
+            headers,
+            params: spec.params,
+            name: spec.name,
+            maxTokensParam: preset?.maxTokensParam,
+            reasoningEffort: preset?.reasoningEffort,
         }),
-        supply: "provider",
-        format: spec.format,
     };
 }
+/**
+ * Run `fn` with the model env, through a translator on this host when the
+ * provider speaks Chat Completions or Responses — the same one a step uses
+ * (runner.ts), started for this call and closed after it.
+ */
+export async function withModelEnv(m, fn) {
+    if (!m.translator)
+        return fn(m.env);
+    const t = await startTranslator(m.translator);
+    try {
+        return await fn({ ...m.env, ...t.env });
+    }
+    finally {
+        await t.close();
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@foldrun/core",
-  "version": "2.3.0",
+  "version": "2.4.0",
   "description": "Agents are just folders. The format, the runner, the checks.",
   "license": "SEE LICENSE IN LICENSE",
   "type": "module",

package/src/evals.ts

@@ -31,7 +31,8 @@
 // they pass — no point paying a model to grade output already known to be
 // wrong.
 
-import { accountModelEnv } from "./model-env.ts";
+import { accountModelEnv, withModelEnv } from "./model-env.ts";
+import type { TranslatorSpec } from "./translator.ts";
 import { sdkEnv } from "./step-exec.ts";
 import fs from "node:fs";
 import path from "node:path";
@@ -515,7 +516,7 @@ async function checkAssertion(
   agentDir: string,
   model: string,
   effort: Effort | null,
-  judgeEnv: () => Record<string, string | undefined> = () => sdkEnv(undefined),
+  judgeEnv: () => { env: Record<string, string | undefined>; translator?: TranslatorSpec | null } = () => ({ env: sdkEnv(undefined) }),
 ): Promise<AssertionResult> {
   const hay = output.toLowerCase();
   switch (assertion.type) {
@@ -550,13 +551,13 @@ async function checkAssertion(
     case "judge": {
       // The judge runs on the account's own model key, never quietly on the
       // platform's: a customer's eval is not the operator's bill.
-      let env: Record<string, string | undefined>;
+      let m: ReturnType<typeof judgeEnv>;
       try {
-        env = judgeEnv();
+        m = judgeEnv();
       } catch (err) {
         return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
       }
-      const verdict = await judge(assertion.value, output, model, effort, env);
+      const verdict = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
       const passed = /^\s*PASS\b/i.test(verdict);
       return { assertion, passed, detail: verdict.slice(0, 300) || "judge returned nothing" };
     }
@@ -639,13 +640,7 @@ export async function runEval(
       const judges = testCase.expect.filter((a) => a.type === "judge");
       for (const a of cheap) assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort));
       if (assertions.every((r) => r.passed)) {
-        const judgeEnv = () => {
-          const m = accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
-          if (m.format !== "anthropic") {
-            throw new Error(`this account's provider speaks ${m.format}; a judge needs an Anthropic-format provider`);
-          }
-          return m.env;
-        };
+        const judgeEnv = () => accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
         for (const a of judges) assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort, judgeEnv));
       } else {
         for (const a of judges) {

package/src/model-env.ts

@@ -11,6 +11,8 @@ import { readAgentsMd } from "./runner.ts";
 import { getSecret } from "./secrets.ts";
 import { platform } from "./platform.ts";
 import { accountModelBlock, isClaudeLoginToken } from "./model-credential.ts";
+import { providerPreset } from "./providers.ts";
+import { startTranslator, translatorSpecFor, type TranslatorSpec } from "./translator.ts";
 
 /** Refused for want of a usable key; `status` is what an HTTP layer answers. */
 export class ModelKeyError extends Error {
@@ -28,14 +30,20 @@ export const NO_MODEL_KEY =
 /**
  * `platform` is a function: the caller's own way to build the platform's env
  * (a step's, or the SDK's host env), called only when this account may use
- * it. `format` says what the endpoint speaks — a caller with no translator
- * refuses anything but anthropic.
+ * it. `format` says what the endpoint speaks; `translator` is set when it is
+ * not Anthropic's, and `withModelEnv` puts the step's translator in front of
+ * it, so a judge or a draft works on any key a run works on (FOL-16).
  */
 export function accountModelEnv(
   tenant: string,
   workspace: string,
   platformEnv: () => Record<string, string | undefined>,
-): { env: Record<string, string | undefined>; supply: "provider" | "platform"; format: string } {
+): {
+  env: Record<string, string | undefined>;
+  supply: "provider" | "platform";
+  format: string;
+  translator: TranslatorSpec | null;
+} {
   const raw =
     (readAgentsMd(workspaceDir(tenant, workspace))?.data?.provider as Record<string, unknown> | undefined) ??
     (readAgentsMd(accountDir(tenant))?.data?.provider as Record<string, unknown> | undefined) ??
@@ -43,7 +51,7 @@ export function accountModelEnv(
   const spec = parseProvider(raw);
   if (!spec?.baseUrl) {
     if (!platform.platformKeyAllowed(tenant)) throw new ModelKeyError(NO_MODEL_KEY);
-    return { env: platformEnv(), supply: "platform", format: "anthropic" };
+    return { env: platformEnv(), supply: "platform", format: "anthropic", translator: null };
   }
   const sub = (text: string) =>
     text.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => getSecret(tenant, name, workspace)?.value ?? whole);
@@ -52,15 +60,39 @@ export function accountModelEnv(
   if (!platform.platformKeyAllowed(tenant) && isClaudeLoginToken(token)) {
     throw new ModelKeyError("model key: a Claude login token (sk-ant-oat…) cannot be used here — use an API key from your provider", 422);
   }
+  const headers = Object.fromEntries(Object.entries(spec.headers).map(([k, v]) => [k, sub(v)]));
+  const preset = providerPreset(spec.name);
   return {
-    env: providerEnvFor({
+    env: providerEnvFor({ baseUrl: spec.baseUrl, token, auth: spec.auth, models: spec.models, headers }),
+    supply: "provider",
+    format: spec.format,
+    translator: translatorSpecFor({
+      format: spec.format,
       baseUrl: spec.baseUrl,
       token,
-      auth: spec.auth,
-      models: spec.models,
-      headers: Object.fromEntries(Object.entries(spec.headers).map(([k, v]) => [k, sub(v)])),
+      headers,
+      params: spec.params,
+      name: spec.name,
+      maxTokensParam: preset?.maxTokensParam,
+      reasoningEffort: preset?.reasoningEffort,
     }),
-    supply: "provider",
-    format: spec.format,
   };
 }
+
+/**
+ * Run `fn` with the model env, through a translator on this host when the
+ * provider speaks Chat Completions or Responses — the same one a step uses
+ * (runner.ts), started for this call and closed after it.
+ */
+export async function withModelEnv<T>(
+  m: { env: Record<string, string | undefined>; translator?: TranslatorSpec | null },
+  fn: (env: Record<string, string | undefined>) => Promise<T>,
+): Promise<T> {
+  if (!m.translator) return fn(m.env);
+  const t = await startTranslator(m.translator);
+  try {
+    return await fn({ ...m.env, ...t.env });
+  } finally {
+    await t.close();
+  }
+}