mohdel 3.7.0 → 3.8.1

package/README.md

@@ -472,6 +472,7 @@ QWEN_API_SK=sk-...
 XIAOMI_API_SK=...
 META_API_SK=...
 COHERE_API_SK=...
+TYPESAFE_API_SK=...
 MOHDEL_LOCAL_API_SK=...
 ```
 
@@ -515,6 +516,7 @@ What each provider supports through mohdel's unified interface:
 | OpenRouter | Yes | Yes | Yes | No | Varies | Meta-provider; `providerOptions.openrouter` for routing prefs |
 | Local | Yes | Yes | Yes | No | No | Any OpenAI-compatible server; endpoint is the catalog entry's `baseURL` |
 | Cohere | n/a | n/a | n/a | n/a | n/a | Embeddings only: no chat models reach mohdel through it |
+| TypeSafe | n/a | n/a | n/a | n/a | n/a | Evaluation only (`evaluate()`): typed questions answered with probabilities |
 | Novita | Yes | Yes | Yes | No | Yes (`reasoning_content`) | Prices in the model list; text via the shared chat-completions path, separate image adapter |
 
 Adapter capability ≠ model capability — whether a given model accepts images, tools, or thinking effort depends on the model spec in `curated.json`. The adapter passes through what the envelope supplies; the provider rejects unsupported combos.

package/config/curated.schema.json

@@ -87,10 +87,11 @@
           "enum": [
             "model",
             "image",
-            "transcription"
+            "transcription",
+            "evaluation"
           ],
           "default": "model",
-          "description": "'model' for chat/completion, 'image' for image generation, 'transcription' for speech-to-text."
+          "description": "'model' for chat/completion, 'image' for image generation, 'transcription' for speech-to-text, 'evaluation' for typed-question answering (evaluate())."
         },
         "label": {
           "type": "string",
@@ -550,6 +551,11 @@
           "type": "string",
           "description": "Symbolic role sent when the caller names none. Required for providers that make the parameter mandatory (Cohere v3+)."
         },
+        "evaluationTypes": {
+          "type": "array",
+          "items": { "type": "string", "enum": ["binary", "choice", "score"] },
+          "description": "Question types an evaluation model answers. A question of another type fails before dispatch. Without it, every type is sent and the provider decides."
+        },
         "rpmLimit": {
           "type": "integer",
           "minimum": 1,

package/js/client/call_evaluation.js

@@ -0,0 +1,47 @@
+/**
+ * Send an EvaluateEnvelope to thin-gate's `POST /v1/evaluate`. One-shot: a
+ * single JSON response body.
+ *
+ * @module client/call_evaluation
+ */
+
+import { requestUnix } from './transport.js'
+import { readAll, parseErrorBody } from './response.js'
+import { MohdelError } from '#core'
+
+/**
+ * @param {import('#core/evaluation.js').EvaluateEnvelope} envelope
+ * @param {object} options
+ * @param {string} options.socketPath
+ * @param {AbortSignal} [options.signal]
+ * @param {string} [options.path]  HTTP path; defaults to '/v1/evaluate'
+ * @param {Record<string, string>} [options.headers]  sent with the request, for a router in front of the gate;
+ *   `content-type`, `content-length`, `transfer-encoding`, `connection` and `host` are the transport's
+ * @returns {Promise<import('#core/evaluation.js').EvaluateResult>}
+ */
+export async function callEvaluation (envelope, { socketPath, signal, path = '/v1/evaluate', headers }) {
+  const res = await requestUnix({
+    socketPath,
+    path,
+    method: 'POST',
+    body: envelope,
+    signal,
+    headers
+  })
+
+  const body = await readAll(res)
+
+  if (res.statusCode !== 200) {
+    throw MohdelError.fromJSON(parseErrorBody(body, res.statusCode ?? 0))
+  }
+
+  try {
+    return JSON.parse(body)
+  } catch (e) {
+    throw new MohdelError(`gate returned an unparseable evaluate body: ${body.slice(0, 200)}`, {
+      type: 'PROTOCOL_INVALID_RESPONSE',
+      severity: 'error',
+      retryable: false
+    })
+  }
+}

package/js/client/index.js

@@ -18,4 +18,5 @@ export { coalesce } from './coalesce.js'
 export { callImage } from './call_image.js'
 export { callTranscription } from './call_transcription.js'
 export { callEmbedding } from './call_embedding.js'
+export { callEvaluation } from './call_evaluation.js'
 export { resolveGateBinary } from './gate-binary.js'

package/js/core/evaluation.js

@@ -0,0 +1,105 @@
+/**
+ * Evaluation envelope and result: one `state`, a map of typed questions, one
+ * typed answer per question.
+ *
+ * @module core/evaluation
+ */
+
+/**
+ * @typedef {object} EvaluateEnvelope
+ * @property {string} callId
+ * @property {string} authId
+ * @property {import('./envelope.js').Auth} auth
+ * @property {string} [traceparent]
+ * @property {string} [baggage]
+ * @property {string} model
+ * @property {string | object | any[]} state
+ * @property {Record<string, Question>} questions
+ *   Answers come back under the same ids.
+ */
+
+/**
+ * @typedef {BinaryQuestion | ChoiceQuestion | ScoreQuestion} Question
+ *
+ * `instructions` and every criterion description may be a string, an object
+ * or an array.
+ */
+
+/**
+ * @typedef {object} BinaryQuestion
+ * @property {'binary'} type
+ * @property {any} instructions
+ * @property {{yes: any, no: any}} [criteria]
+ */
+
+/**
+ * @typedef {object} ChoiceQuestion
+ * @property {'choice'} type
+ * @property {any} instructions
+ * @property {Record<string, any>} criteria
+ *   Option name to its description, or null. At least two options.
+ */
+
+/**
+ * @typedef {object} ScoreQuestion
+ * @property {'score'} type
+ * @property {any} instructions
+ * @property {any[]} criteria
+ *   Level descriptions, lowest first; level `i` is worth `i`. At least two.
+ */
+
+/**
+ * @typedef {BinaryAnswer | ChoiceAnswer | ScoreAnswer} Answer
+ */
+
+/**
+ * @typedef {object} BinaryAnswer
+ * @property {'binary'} type
+ * @property {number} probability
+ *   Of yes.
+ */
+
+/**
+ * @typedef {object} ChoiceAnswer
+ * @property {'choice'} type
+ * @property {string} choice
+ * @property {Record<string, number>} probabilities
+ * @property {number} [confidence]
+ *   Absent when the provider reports none.
+ */
+
+/**
+ * @typedef {object} ScoreAnswer
+ * @property {'score'} type
+ * @property {number} score
+ *   Probability-weighted level; can fall between two.
+ * @property {number[]} probabilities
+ *   One per level, in `criteria` order.
+ * @property {number} [confidence]
+ */
+
+/**
+ * @typedef {object} EvaluateResult
+ * @property {'completed'} status
+ * @property {Record<string, Answer>} answers
+ * @property {string} upstreamModel
+ *   The provider's id for the version that answered, which differs from the
+ *   entry's `model` when that is an alias.
+ * @property {number} inputTokens
+ * @property {number} outputTokens
+ * @property {number} cost
+ * @property {{start: string, first: string, end: string}} timestamps
+ */
+
+export const EVALUATE_ENVELOPE_FIELDS = Object.freeze([
+  'callId',
+  'authId',
+  'auth',
+  'traceparent',
+  'baggage',
+  'model',
+  'state',
+  'questions'
+])
+
+export const QUESTION_TYPES = Object.freeze(['binary', 'choice', 'score'])

package/js/factory/bridge.js

@@ -24,6 +24,7 @@ import { run } from '../session/run.js'
 import { runImage } from '../session/run_image.js'
 import { runTranscription } from '../session/run_transcription.js'
 import { runEmbedding } from '../session/run_embedding.js'
+import { runEvaluation } from '../session/run_evaluation.js'
 import { markTrustedMedia } from '../session/adapters/_media.js'
 import { MohdelError, validateIds } from '#core'
 import { createRealtimeDeltaBuffer } from '../../src/lib/utils.js'
@@ -224,6 +225,44 @@ export async function runAnswerEmbedding ({ provider, model, modelKey, configura
   return out.result
 }
 
+/**
+ * Run an `evaluate()` call through the /session runtime.
+ *
+ * @param {object} args
+ * @param {string} args.provider
+ * @param {string} args.model
+ * @param {string} [args.modelKey]
+ * @param {any} args.configuration
+ * @param {import('#core/evaluation.js').EvaluateEnvelope['state']} args.state
+ * @param {import('#core/evaluation.js').EvaluateEnvelope['questions']} args.questions
+ * @param {any} [args.options]             `callId` / `authId` only.
+ * @param {any} [args.spec]
+ * @param {BridgeDeps} [deps]
+ * @returns {Promise<import('#core/evaluation.js').EvaluateResult>}
+ */
+export async function runAnswerEvaluation ({ provider, model, modelKey, configuration, state, questions, options = {}, spec }, deps = {}) {
+  const callId = options.callId || newCallId()
+  const authId = options.authId || 'local'
+  assertValidIds(callId, authId, `${provider}/${model}`)
+
+  const envelope = {
+    callId,
+    authId,
+    auth: configToAuth(configuration),
+    model: `${provider}/${model}`,
+    state,
+    questions
+  }
+
+  const out = await runEvaluation(envelope, {
+    ...deps,
+    ...(modelKey ? { modelKey } : {}),
+    ...(spec ? { spec } : {})
+  })
+  if (!out.ok) throw MohdelError.fromJSON(out.error, { provider, model })
+  return out.result
+}
+
 /**
  * @param {object} args
  * @param {string} args.modelKey      Mohdel catalog key `<provider>/<bare>`. The

package/js/session/adapters/_chat_completions.js

@@ -262,6 +262,11 @@ async function * runStreaming (envelope, client, args, config, start, deps) {
     return
   }
 
+  if (finishReason === null) {
+    yield { type: 'error', error: classifyProviderError(new Error(`${config.provider} stream ended before completion`), envelope.auth?.key, { provider: config.provider }) }
+    return
+  }
+
   const collectedToolCalls = Object.values(toolCallAccum).map(tc => ({
     id: tc.id,
     function: { name: tc.name, arguments: tc.arguments }

package/js/session/adapters/_errors.js

@@ -469,9 +469,11 @@ export function typedError (message, type, retryable, detail) {
  * @param {number} status
  * @param {string} message
  * @param {string} [detail]
+ * @param {string} [key]  masked out of `detail`
  * @returns {Error & {typed: import('#core/errors.js').TypedError}}
  */
-export function fromHttpStatus (status, message, detail) {
+export function fromHttpStatus (status, message, detail, key) {
   const typed = classifyProviderError({ status })
-  return typedError(typed.message, typed.type, typed.retryable, detail ? `${message}: ${detail}` : message)
+  const scrubbed = scrubKey(detail, key)
+  return typedError(typed.message, typed.type, typed.retryable, scrubbed ? `${message}: ${scrubbed}` : message)
 }

package/js/session/adapters/_pricing.js

@@ -146,6 +146,16 @@ export function embeddingCostFor (envelope, spec, usage) {
   return unmetered(envelope.model) ? 0 : computeEmbeddingCost(spec, usage)
 }
 
+/**
+ * @param {{model: string}} envelope
+ * @param {any} spec
+ * @param {{inputTokens: number, outputTokens: number}} usage
+ * @returns {number}
+ */
+export function evaluationCostFor (envelope, spec, usage) {
+  return unmetered(envelope.model) ? 0 : computeCost(spec, usage)
+}
+
 /**
  * Cost of a transcription call.
  *

package/js/session/adapters/embedding/cohere.js

@@ -61,7 +61,7 @@ export async function cohereEmbedding (envelope, deps = {}) {
 
   if (!res.ok) {
     const detail = await res.text().catch(() => '')
-    throw fromHttpStatus(res.status, detail, envelope.auth?.key)
+    throw fromHttpStatus(res.status, 'embedding request failed', detail.slice(0, 500), envelope.auth?.key)
   }
 
   const payload = await res.json()

package/js/session/adapters/embedding/gemini.js

@@ -59,7 +59,7 @@ export async function geminiEmbedding (envelope, deps = {}) {
 
   if (!res.ok) {
     const detail = await res.text().catch(() => '')
-    throw fromHttpStatus(res.status, detail, envelope.auth?.key)
+    throw fromHttpStatus(res.status, 'embedding request failed', detail.slice(0, 500), envelope.auth?.key)
   }
 
   const payload = await res.json()

package/js/session/adapters/embedding/openai_compatible.js

@@ -56,7 +56,7 @@ export function createEmbeddingAdapter ({ baseURL, dimensionsField = 'dimensions
 
     if (!res.ok) {
       const detail = await res.text().catch(() => '')
-      throw fromHttpStatus(res.status, detail, envelope.auth?.key)
+      throw fromHttpStatus(res.status, 'embedding request failed', detail.slice(0, 500), envelope.auth?.key)
     }
 
     const payload = await res.json()

package/js/session/adapters/evaluation/_shared.js

@@ -0,0 +1,73 @@
+/**
+ * Pre-dispatch checks every evaluation adapter runs. They reject what the
+ * gate's serde types reject, so a malformed envelope fails the same way
+ * in-process.
+ *
+ * @module session/adapters/evaluation/shared
+ */
+
+import { typedError } from '../_errors.js'
+import { QUESTION_TYPES } from '#core/evaluation.js'
+
+const QUESTION_KEYS = new Set(['type', 'instructions', 'criteria'])
+const BINARY_CRITERIA_KEYS = ['yes', 'no']
+
+/**
+ * @param {import('#core/evaluation.js').EvaluateEnvelope} envelope
+ * @param {any} spec
+ */
+export function checkEvaluation (envelope, spec) {
+  const { state, questions } = envelope
+  if (typeof state !== 'string' && !isObject(state) && !Array.isArray(state)) {
+    throw invalid('state must be a string, an object or an array')
+  }
+  if (!isObject(questions) || Object.keys(questions).length === 0) {
+    throw invalid('questions must be a non-empty object')
+  }
+  for (const [id, q] of Object.entries(questions)) {
+    checkQuestion(id, q, spec)
+  }
+}
+
+function checkQuestion (id, q, spec) {
+  if (!isObject(q)) throw invalid(`question '${id}' must be an object`)
+  const extra = Object.keys(q).filter(k => !QUESTION_KEYS.has(k))
+  if (extra.length) throw invalid(`question '${id}' has unknown field '${extra.join("', '")}'`)
+  if (!QUESTION_TYPES.includes(q.type)) {
+    throw invalid(`question '${id}' type must be one of ${QUESTION_TYPES.join(', ')}`)
+  }
+  if (Array.isArray(spec?.evaluationTypes) && !spec.evaluationTypes.includes(q.type)) {
+    throw typedError(
+      `question '${id}' is '${q.type}', which this model does not answer; it answers ${spec.evaluationTypes.join(', ')}`,
+      'EVALUATE_QUESTION_TYPE_UNSUPPORTED',
+      false
+    )
+  }
+  if (q.instructions === undefined || q.instructions === null) {
+    throw invalid(`question '${id}' requires instructions`)
+  }
+
+  const c = q.criteria
+  if (q.type === 'binary') {
+    if (c === undefined) return
+    if (!isObject(c) || Object.keys(c).length !== 2 || !BINARY_CRITERIA_KEYS.every(k => c[k] !== undefined && c[k] !== null)) {
+      throw invalid(`question '${id}' criteria must have exactly 'yes' and 'no'`)
+    }
+  } else if (q.type === 'choice') {
+    if (!isObject(c) || Object.keys(c).length < 2) {
+      throw invalid(`question '${id}' criteria must name at least two options`)
+    }
+  } else if (!Array.isArray(c) || c.length < 2) {
+    throw invalid(`question '${id}' criteria must list at least two levels`)
+  }
+}
+
+/** @param {unknown} v */
+function isObject (v) {
+  return typeof v === 'object' && v !== null && !Array.isArray(v)
+}
+
+/** @param {string} message */
+function invalid (message) {
+  return typedError(message, 'EVALUATE_INPUT_INVALID', false)
+}

package/js/session/adapters/evaluation/index.js

@@ -0,0 +1,22 @@
+/**
+ * Evaluation-adapter registry.
+ *
+ * @module session/adapters/evaluation
+ */
+
+import { typesafeEvaluation } from './typesafe.js'
+
+const EVALUATION_ADAPTERS = {
+  typesafe: typesafeEvaluation
+}
+
+export const EVALUATION_PROVIDERS = Object.freeze(Object.keys(EVALUATION_ADAPTERS))
+
+/**
+ * @param {string} provider
+ */
+export function getEvaluationAdapter (provider) {
+  const adapter = EVALUATION_ADAPTERS[provider]
+  if (!adapter) throw new Error(`no evaluation adapter for provider: ${provider}`)
+  return adapter
+}

package/js/session/adapters/evaluation/typesafe.js

@@ -0,0 +1,118 @@
+/**
+ * TypeSafe evaluation adapter (`POST /v1/systemone`). TypeSafe calls a binary
+ * question `noul` and keys binary criteria `true` / `false`; score
+ * probabilities come back keyed by level index as strings.
+ *
+ * @module session/adapters/evaluation/typesafe
+ */
+
+import { getSpec } from '../_catalog.js'
+import { classifyProviderError, fromHttpStatus, typedError } from '../_errors.js'
+import { evaluationCostFor } from '../_pricing.js'
+import { catalogKey, bareOf } from '#core/model-id.js'
+import { checkEvaluation } from './_shared.js'
+
+const BASE_URL = 'https://api.typesafe.ai/v1'
+
+export async function typesafeEvaluation (envelope, deps = {}) {
+  const fetchFn = deps.fetch ?? globalThis.fetch
+  const spec = deps.spec ?? getSpec(catalogKey(envelope.model)) ?? {}
+  const start = String(process.hrtime.bigint())
+
+  checkEvaluation(envelope, spec)
+
+  const body = {
+    model: spec.model ?? bareOf(envelope.model),
+    state: envelope.state,
+    questions: Object.fromEntries(
+      Object.entries(envelope.questions).map(([id, q]) => [id, toQuestion(q)])
+    )
+  }
+
+  let res
+  try {
+    res = await fetchFn(`${BASE_URL}/systemone`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${envelope.auth.key}`
+      },
+      body: JSON.stringify(body)
+    })
+  } catch (e) {
+    throw typedError(classifyProviderError(e, envelope.auth?.key).message, 'NET_ERROR', true)
+  }
+
+  if (!res.ok) {
+    const detail = await res.text().catch(() => '')
+    throw fromHttpStatus(res.status, 'evaluation request failed', detail.slice(0, 500), envelope.auth?.key)
+  }
+
+  const payload = await res.json()
+  const usage = payload?.usage
+  if (typeof payload?.model !== 'string' || !isCount(usage?.input_tokens) || !isCount(usage?.output_tokens)) {
+    throw mismatch('response lacks model or token usage')
+  }
+
+  const answers = {}
+  for (const [id, q] of Object.entries(envelope.questions)) {
+    answers[id] = fromAnswer(id, q, payload.answers?.[id])
+  }
+
+  const inputTokens = usage.input_tokens
+  const outputTokens = usage.output_tokens
+  const end = String(process.hrtime.bigint())
+
+  return {
+    status: 'completed',
+    answers,
+    upstreamModel: payload.model,
+    inputTokens,
+    outputTokens,
+    cost: evaluationCostFor(envelope, spec, { inputTokens, outputTokens }),
+    timestamps: { start, first: end, end }
+  }
+}
+
+function toQuestion (q) {
+  if (q.type !== 'binary') return q
+  const out = { type: 'noul', instructions: q.instructions }
+  if (q.criteria) out.criteria = { true: q.criteria.yes, false: q.criteria.no }
+  return out
+}
+
+function fromAnswer (id, q, a) {
+  if (q.type === 'binary') {
+    if (a?.type !== 'noul' || !isProbability(a.noul)) throw mismatch(`no binary answer for '${id}'`)
+    return { type: 'binary', probability: a.noul }
+  }
+  if (a?.type !== q.type || !isProbability(a.confidence)) throw mismatch(`no ${q.type} answer for '${id}'`)
+
+  if (q.type === 'choice') {
+    const options = Object.keys(q.criteria)
+    if (typeof a.choice !== 'string' || !options.includes(a.choice) ||
+        !options.every(o => isProbability(a.probabilities?.[o]))) {
+      throw mismatch(`malformed choice answer for '${id}'`)
+    }
+    const probabilities = Object.fromEntries(options.map(o => [o, a.probabilities[o]]))
+    return { type: 'choice', choice: a.choice, probabilities, confidence: a.confidence }
+  }
+
+  const probabilities = q.criteria.map((_, i) => a.probabilities?.[String(i)])
+  if (typeof a.score !== 'number' || !probabilities.every(isProbability)) {
+    throw mismatch(`malformed score answer for '${id}'`)
+  }
+  return { type: 'score', score: a.score, probabilities, confidence: a.confidence }
+}
+
+function isProbability (v) {
+  return typeof v === 'number' && v >= 0 && v <= 1
+}
+
+function isCount (v) {
+  return Number.isInteger(v) && v >= 0
+}
+
+function mismatch (message) {
+  return typedError(message, 'EVALUATE_RESULT_MISMATCH', false)
+}

package/js/session/adapters/openai.js

@@ -167,12 +167,12 @@ export async function * openai (envelope, deps = {}) {
           }
           break
 
-        case 'response.failed':
-          if (providerOf(envelope.model) === 'chatgpt') {
-            yield { type: 'error', error: classifyProviderError(new Error('ChatGPT response failed'), envelope.auth?.key, { provider: 'chatgpt' }) }
-            return
-          }
-          break
+        case 'response.failed': {
+          const provider = providerOf(envelope.model)
+          const failure = Object.assign(new Error(`${provider} response failed`), { error: event.response?.error })
+          yield { type: 'error', error: classifyProviderError(failure, envelope.auth?.key, { provider }) }
+          return
+        }
 
         default:
           break
@@ -194,8 +194,9 @@ export async function * openai (envelope, deps = {}) {
   }
 
   const end = String(process.hrtime.bigint())
-  if (providerOf(envelope.model) === 'chatgpt' && !terminalResponse) {
-    yield { type: 'error', error: classifyProviderError(new Error('ChatGPT stream ended before completion'), envelope.auth?.key, { provider: 'chatgpt' }) }
+  if (!terminalResponse) {
+    const provider = providerOf(envelope.model)
+    yield { type: 'error', error: classifyProviderError(new Error(`${provider} stream ended before completion`), envelope.auth?.key, { provider }) }
     return
   }
   // OpenAI Responses reports `output_tokens` INCLUDING reasoning

package/js/session/driver.js

@@ -19,6 +19,7 @@ import { run } from './run.js'
 import { runImage } from './run_image.js'
 import { runTranscription } from './run_transcription.js'
 import { runEmbedding } from './run_embedding.js'
+import { runEvaluation } from './run_evaluation.js'
 import { runInfo } from './run_info.js'
 import { setCatalog } from './adapters/_catalog.js'
 
@@ -233,6 +234,14 @@ export async function drive (stdin, stdout) {
         } else {
           await writeLine({ type: 'error', error: out.error })
         }
+      } else if (envelope.op === 'evaluate') {
+        const { op: _op, ...evEnv } = envelope
+        const out = await runEvaluation(evEnv)
+        if (out.ok) {
+          await writeLine({ type: 'evaluate_done', result: out.result })
+        } else {
+          await writeLine({ type: 'error', error: out.error })
+        }
       } else if (envelope.op === 'info') {
         const out = await runInfo(envelope)
         if (out.ok) {

package/js/session/run.js

@@ -23,6 +23,7 @@
  */
 
 import { ADAPTER_NAMES, isImageProvider } from './adapters/_registry.js'
+import { EVALUATION_PROVIDERS } from './adapters/evaluation/index.js'
 import { getSpec } from './adapters/_catalog.js'
 import { getProviderLimits } from './adapters/_providers.js'
 import { hasSpeed, mergeSpeed, speedHasOwnQuota, speedNames } from './adapters/_speed.js'
@@ -119,6 +120,13 @@ export async function * run (envelope, {
       yield err
       return
     }
+    if (EVALUATION_PROVIDERS.includes(provider)) {
+      const detail = `provider '${provider}' supports evaluation only; use evaluate(...) instead`
+      log.warn({ provider }, '[mohdel:answer] evaluation-only provider via answer')
+      endSpanError(span, new Error(detail))
+      yield errorEvent(detail, 'PROVIDER_TEXT_NOT_SUPPORTED')
+      return
+    }
     const err = errorEvent(messageOf(e), 'SESSION_UNKNOWN_PROVIDER')
     log.warn({ err: e, provider }, '[mohdel:answer] unknown provider')
     endSpanError(span, e)

package/js/session/run_evaluation.js

@@ -0,0 +1,84 @@
+/**
+ * Evaluation runtime. Resolves the adapter for the envelope's provider and
+ * returns either a result or a typed error, never throwing. Rate limits as in
+ * `run_embedding.js`, minus the input count.
+ *
+ * @module session/run_evaluation
+ */
+
+import { getEvaluationAdapter } from './adapters/evaluation/index.js'
+import { classifyProviderError } from './adapters/_errors.js'
+import { getProviderLimits } from './adapters/_providers.js'
+import * as defaultLimiter from './_rate_limiter.js'
+import { providerOf } from '#core/model-id.js'
+
+/**
+ * @param {import('#core/evaluation.js').EvaluateEnvelope} envelope
+ * @param {{
+ *   resolveAdapter?: (provider: string) => any,
+ *   resolveProviderLimits?: (provider: string) => any,
+ *   limiter?: any,
+ *   sleep?: (ms: number) => Promise<void>,
+ *   modelKey?: string,
+ *   spec?: any
+ * }} [options]
+ * @returns {Promise<
+ *   | {ok: true, result: import('#core/evaluation.js').EvaluateResult}
+ *   | {ok: false, error: import('#core/errors.js').TypedError}
+ * >}
+ */
+export async function runEvaluation (envelope, {
+  resolveAdapter = getEvaluationAdapter,
+  resolveProviderLimits = getProviderLimits,
+  limiter = defaultLimiter,
+  sleep = defaultSleep,
+  modelKey = envelope.model,
+  spec
+} = {}) {
+  const provider = providerOf(envelope.model)
+
+  let adapter
+  try {
+    adapter = resolveAdapter(provider)
+  } catch (e) {
+    return {
+      ok: false,
+      error: {
+        message: messageOf(e),
+        severity: 'error',
+        retryable: false,
+        type: 'SESSION_UNKNOWN_PROVIDER'
+      }
+    }
+  }
+
+  const providerCfg = resolveProviderLimits(provider) || {}
+  const rpmLimit = spec?.rpmLimit ?? providerCfg.rpmLimit
+  const tpmLimit = spec?.tpmLimit ?? providerCfg.tpmLimit
+  const bucketKey = spec?.rateLimitScope === 'model' ? modelKey : provider
+
+  if (rpmLimit != null || tpmLimit != null) {
+    const delay = limiter.check(bucketKey, { rpmLimit, tpmLimit })
+    if (delay > 0) await sleep(delay)
+    limiter.recordRequest(bucketKey)
+  }
+
+  try {
+    const result = await adapter(envelope, spec ? { spec } : {})
+    if (tpmLimit != null) limiter.recordTokens(bucketKey, result.inputTokens + result.outputTokens)
+    return { ok: true, result }
+  } catch (e) {
+    const typed = /** @type {any} */(e).typed || classifyProviderError(e, envelope.auth?.key)
+    return { ok: false, error: typed }
+  }
+}
+
+/** @param {unknown} e */
+function messageOf (e) {
+  return e instanceof Error ? e.message : String(e)
+}
+
+/** @param {number} ms */
+function defaultSleep (ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mohdel",
-  "version": "3.7.0",
+  "version": "3.8.1",
   "license": "MIT",
   "author": {
     "name": "Christophe Le Bars",
@@ -144,7 +144,7 @@
     "@opentelemetry/exporter-trace-otlp-grpc": "^0.222.0",
     "@opentelemetry/sdk-node": "^0.222.0",
     "chalk": "^6.0.1",
-    "mohdel-thin-gate-linux-x64-gnu": "3.7.0"
+    "mohdel-thin-gate-linux-x64-gnu": "3.8.1"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.129.0",

package/src/lib/creators.js

@@ -65,6 +65,12 @@ const creators = {
     logo: 'cohere.svg',
     description: 'Cohere builds retrieval-focused models: embeddings and rerankers aimed at enterprise search rather than chat.'
   },
+  typesafe: {
+    prefixes: ['jev'],
+    label: 'TypeSafe',
+    logo: 'typesafe.svg',
+    description: 'TypeSafe trains decision models that answer typed questions with calibrated probabilities instead of generating text.'
+  },
   nomic: {
     prefixes: ['nomic-embed'],
     label: 'Nomic',

package/src/lib/index.js

@@ -16,7 +16,7 @@ import {
 import { createRateLimiter } from '../../js/session/_rate_limiter.js'
 import { createCooldownTracker } from '../../js/session/_cooldown.js'
 import { setCatalog } from '../../js/session/adapters/_catalog.js'
-import { runAnswer, runAnswerEmbedding, runAnswerImage, runAnswerTranscription } from '../../js/factory/bridge.js'
+import { runAnswer, runAnswerEmbedding, runAnswerEvaluation, runAnswerImage, runAnswerTranscription } from '../../js/factory/bridge.js'
 import { startSpan, endSpanOk, endSpanError } from './tracing.js'
 import { isValidTag } from './schema.js'
 import { silent } from './logger.js'
@@ -751,6 +751,22 @@ const createModelProxy = (resolvedModelId, modelSpec, handlers, aliasOutputEffor
         }
       }
 
+      if (prop === 'evaluate') {
+        return async (state, questions, options = {}) => {
+          const { configuration } = await getRuntime()
+          return runAnswerEvaluation({
+            provider: modelSpec.provider,
+            model: modelSpec.model ?? resolvedModelId.split('/').pop(),
+            modelKey: resolvedModelId,
+            configuration,
+            state,
+            questions,
+            options,
+            spec: modelSpec
+          }, { limiter: rateLimiter, resolveProviderLimits })
+        }
+      }
+
       if (prop === 'setRateLimit') {
         return async ({ rpm, tpm, inpm } = {}) => {
           const curatedCache = getCuratedCacheSnapshot()

package/src/lib/provider-info.js

@@ -100,6 +100,13 @@ const PROVIDER_INFO = {
     hint: 'Create an API key in the Cohere dashboard under API Keys',
     free: true
   },
+  typesafe: {
+    label: 'TypeSafe',
+    description: 'Calibrated yes/no, choice and score answers about a state. No chat models through mohdel.',
+    url: 'https://console.typesafe.ai/keys',
+    hint: 'Create an API key in the TypeSafe console under Keys',
+    free: false
+  },
   qwen: {
     label: 'Qwen Cloud',
     description: 'Qwen — reasoning, coding, long context. Free quota for new users.',

package/src/lib/providers.js

@@ -144,6 +144,20 @@ const providers = {
       rateLimits: 'https://docs.cohere.com/docs/rate-limits'
     }
   },
+  typesafe: {
+    billing: { kind: 'metered' },
+    sdk: 'typesafe',
+    api: 'evaluation',
+    catalog: false,
+    apiKeyEnv: 'TYPESAFE_API_SK',
+    baseURL: 'https://api.typesafe.ai/v1',
+    createConfiguration: apiKey => ({ apiKey }),
+    references: {
+      pricing: 'https://docs.typesafe.ai/models',
+      models: 'https://docs.typesafe.ai/models',
+      rateLimits: 'https://docs.typesafe.ai/models'
+    }
+  },
   mistral: {
     billing: { kind: 'metered' },
     sdk: 'openai',

package/src/lib/schema.js

@@ -65,6 +65,7 @@ const fieldDefs = {
   maxInputTokens: { type: 'number' },
   inputTypes: { type: 'object' },
   defaultInputType: { type: 'string' },
+  evaluationTypes: { type: 'array', itemType: 'string' },
   deprecated: { type: 'string' },
   suspended: { type: 'string' },
   rpmLimit: { type: 'number' },