mohdel 3.6.1 → 3.8.0

package/README.md

@@ -472,6 +472,7 @@ QWEN_API_SK=sk-...
 XIAOMI_API_SK=...
 META_API_SK=...
 COHERE_API_SK=...
+TYPESAFE_API_SK=...
 MOHDEL_LOCAL_API_SK=...
 ```
 
@@ -515,6 +516,7 @@ What each provider supports through mohdel's unified interface:
 | OpenRouter | Yes | Yes | Yes | No | Varies | Meta-provider; `providerOptions.openrouter` for routing prefs |
 | Local | Yes | Yes | Yes | No | No | Any OpenAI-compatible server; endpoint is the catalog entry's `baseURL` |
 | Cohere | n/a | n/a | n/a | n/a | n/a | Embeddings only: no chat models reach mohdel through it |
+| TypeSafe | n/a | n/a | n/a | n/a | n/a | Evaluation only (`evaluate()`): typed questions answered with probabilities |
 | Novita | Yes | Yes | Yes | No | Yes (`reasoning_content`) | Prices in the model list; text via the shared chat-completions path, separate image adapter |
 
 Adapter capability ≠ model capability — whether a given model accepts images, tools, or thinking effort depends on the model spec in `curated.json`. The adapter passes through what the envelope supplies; the provider rejects unsupported combos.

package/config/curated.schema.json

@@ -87,10 +87,11 @@
           "enum": [
             "model",
             "image",
-            "transcription"
+            "transcription",
+            "evaluation"
           ],
           "default": "model",
-          "description": "'model' for chat/completion, 'image' for image generation, 'transcription' for speech-to-text."
+          "description": "'model' for chat/completion, 'image' for image generation, 'transcription' for speech-to-text, 'evaluation' for typed-question answering (evaluate())."
         },
         "label": {
           "type": "string",
@@ -550,6 +551,11 @@
           "type": "string",
           "description": "Symbolic role sent when the caller names none. Required for providers that make the parameter mandatory (Cohere v3+)."
         },
+        "evaluationTypes": {
+          "type": "array",
+          "items": { "type": "string", "enum": ["binary", "choice", "score"] },
+          "description": "Question types an evaluation model answers. A question of another type fails before dispatch. Without it, every type is sent and the provider decides."
+        },
         "rpmLimit": {
           "type": "integer",
           "minimum": 1,

package/js/chatgpt/index.js

@@ -4,6 +4,7 @@ import { hostname } from 'node:os'
 import { createRemoteJWKSet, jwtVerify } from 'jose'
 import { defaultDirectory, readStore, withStore } from './store.js'
 import { discoverModels, visibleModels } from './models.js'
+import providers from '../../src/lib/providers.js'
 
 const ISSUER = 'https://auth.openai.com'
 const RESOURCE = 'https://api.openai.com/v1'
@@ -15,7 +16,7 @@ const random = () => randomBytes(32).toString('base64url')
 const REFRESH_MARGIN = 30000
 // Every registration made before names were stored was sent this hint.
 const UNNAMED_REGISTRATION = 'Mohdel'
-export const usageURL = 'https://chatgpt.com/settings/usage'
+export const usageURL = providers.chatgpt.billing.usage
 
 async function jsonRequest (fetcher, url, options = {}) {
   let response

package/js/client/call_evaluation.js

@@ -0,0 +1,47 @@
+/**
+ * Send an EvaluateEnvelope to thin-gate's `POST /v1/evaluate`. One-shot: a
+ * single JSON response body.
+ *
+ * @module client/call_evaluation
+ */
+
+import { requestUnix } from './transport.js'
+import { readAll, parseErrorBody } from './response.js'
+import { MohdelError } from '#core'
+
+/**
+ * @param {import('#core/evaluation.js').EvaluateEnvelope} envelope
+ * @param {object} options
+ * @param {string} options.socketPath
+ * @param {AbortSignal} [options.signal]
+ * @param {string} [options.path]  HTTP path; defaults to '/v1/evaluate'
+ * @param {Record<string, string>} [options.headers]  sent with the request, for a router in front of the gate;
+ *   `content-type`, `content-length`, `transfer-encoding`, `connection` and `host` are the transport's
+ * @returns {Promise<import('#core/evaluation.js').EvaluateResult>}
+ */
+export async function callEvaluation (envelope, { socketPath, signal, path = '/v1/evaluate', headers }) {
+  const res = await requestUnix({
+    socketPath,
+    path,
+    method: 'POST',
+    body: envelope,
+    signal,
+    headers
+  })
+
+  const body = await readAll(res)
+
+  if (res.statusCode !== 200) {
+    throw MohdelError.fromJSON(parseErrorBody(body, res.statusCode ?? 0))
+  }
+
+  try {
+    return JSON.parse(body)
+  } catch (e) {
+    throw new MohdelError(`gate returned an unparseable evaluate body: ${body.slice(0, 200)}`, {
+      type: 'PROTOCOL_INVALID_RESPONSE',
+      severity: 'error',
+      retryable: false
+    })
+  }
+}

package/js/client/index.js

@@ -18,4 +18,5 @@ export { coalesce } from './coalesce.js'
 export { callImage } from './call_image.js'
 export { callTranscription } from './call_transcription.js'
 export { callEmbedding } from './call_embedding.js'
+export { callEvaluation } from './call_evaluation.js'
 export { resolveGateBinary } from './gate-binary.js'

package/js/core/evaluation.js

@@ -0,0 +1,105 @@
+/**
+ * Evaluation envelope and result: one `state`, a map of typed questions, one
+ * typed answer per question.
+ *
+ * @module core/evaluation
+ */
+
+/**
+ * @typedef {object} EvaluateEnvelope
+ * @property {string} callId
+ * @property {string} authId
+ * @property {import('./envelope.js').Auth} auth
+ * @property {string} [traceparent]
+ * @property {string} [baggage]
+ * @property {string} model
+ * @property {string | object | any[]} state
+ * @property {Record<string, Question>} questions
+ *   Answers come back under the same ids.
+ */
+
+/**
+ * @typedef {BinaryQuestion | ChoiceQuestion | ScoreQuestion} Question
+ *
+ * `instructions` and every criterion description may be a string, an object
+ * or an array.
+ */
+
+/**
+ * @typedef {object} BinaryQuestion
+ * @property {'binary'} type
+ * @property {any} instructions
+ * @property {{yes: any, no: any}} [criteria]
+ */
+
+/**
+ * @typedef {object} ChoiceQuestion
+ * @property {'choice'} type
+ * @property {any} instructions
+ * @property {Record<string, any>} criteria
+ *   Option name to its description, or null. At least two options.
+ */
+
+/**
+ * @typedef {object} ScoreQuestion
+ * @property {'score'} type
+ * @property {any} instructions
+ * @property {any[]} criteria
+ *   Level descriptions, lowest first; level `i` is worth `i`. At least two.
+ */
+
+/**
+ * @typedef {BinaryAnswer | ChoiceAnswer | ScoreAnswer} Answer
+ */
+
+/**
+ * @typedef {object} BinaryAnswer
+ * @property {'binary'} type
+ * @property {number} probability
+ *   Of yes.
+ */
+
+/**
+ * @typedef {object} ChoiceAnswer
+ * @property {'choice'} type
+ * @property {string} choice
+ * @property {Record<string, number>} probabilities
+ * @property {number} [confidence]
+ *   Absent when the provider reports none.
+ */
+
+/**
+ * @typedef {object} ScoreAnswer
+ * @property {'score'} type
+ * @property {number} score
+ *   Probability-weighted level; can fall between two.
+ * @property {number[]} probabilities
+ *   One per level, in `criteria` order.
+ * @property {number} [confidence]
+ */
+
+/**
+ * @typedef {object} EvaluateResult
+ * @property {'completed'} status
+ * @property {Record<string, Answer>} answers
+ * @property {string} upstreamModel
+ *   The provider's id for the version that answered, which differs from the
+ *   entry's `model` when that is an alias.
+ * @property {number} inputTokens
+ * @property {number} outputTokens
+ * @property {number} cost
+ * @property {{start: string, first: string, end: string}} timestamps
+ */
+
+export const EVALUATE_ENVELOPE_FIELDS = Object.freeze([
+  'callId',
+  'authId',
+  'auth',
+  'traceparent',
+  'baggage',
+  'model',
+  'state',
+  'questions'
+])
+
+export const QUESTION_TYPES = Object.freeze(['binary', 'choice', 'score'])

package/js/factory/bridge.js

@@ -24,6 +24,7 @@ import { run } from '../session/run.js'
 import { runImage } from '../session/run_image.js'
 import { runTranscription } from '../session/run_transcription.js'
 import { runEmbedding } from '../session/run_embedding.js'
+import { runEvaluation } from '../session/run_evaluation.js'
 import { markTrustedMedia } from '../session/adapters/_media.js'
 import { MohdelError, validateIds } from '#core'
 import { createRealtimeDeltaBuffer } from '../../src/lib/utils.js'
@@ -224,6 +225,44 @@ export async function runAnswerEmbedding ({ provider, model, modelKey, configura
   return out.result
 }
 
+/**
+ * Run an `evaluate()` call through the /session runtime.
+ *
+ * @param {object} args
+ * @param {string} args.provider
+ * @param {string} args.model
+ * @param {string} [args.modelKey]
+ * @param {any} args.configuration
+ * @param {import('#core/evaluation.js').EvaluateEnvelope['state']} args.state
+ * @param {import('#core/evaluation.js').EvaluateEnvelope['questions']} args.questions
+ * @param {any} [args.options]             `callId` / `authId` only.
+ * @param {any} [args.spec]
+ * @param {BridgeDeps} [deps]
+ * @returns {Promise<import('#core/evaluation.js').EvaluateResult>}
+ */
+export async function runAnswerEvaluation ({ provider, model, modelKey, configuration, state, questions, options = {}, spec }, deps = {}) {
+  const callId = options.callId || newCallId()
+  const authId = options.authId || 'local'
+  assertValidIds(callId, authId, `${provider}/${model}`)
+
+  const envelope = {
+    callId,
+    authId,
+    auth: configToAuth(configuration),
+    model: `${provider}/${model}`,
+    state,
+    questions
+  }
+
+  const out = await runEvaluation(envelope, {
+    ...deps,
+    ...(modelKey ? { modelKey } : {}),
+    ...(spec ? { spec } : {})
+  })
+  if (!out.ok) throw MohdelError.fromJSON(out.error, { provider, model })
+  return out.result
+}
+
 /**
  * @param {object} args
  * @param {string} args.modelKey      Mohdel catalog key `<provider>/<bare>`. The

package/js/session/adapters/_errors.js

@@ -469,9 +469,11 @@ export function typedError (message, type, retryable, detail) {
  * @param {number} status
  * @param {string} message
  * @param {string} [detail]
+ * @param {string} [key]  masked out of `detail`
  * @returns {Error & {typed: import('#core/errors.js').TypedError}}
  */
-export function fromHttpStatus (status, message, detail) {
+export function fromHttpStatus (status, message, detail, key) {
   const typed = classifyProviderError({ status })
-  return typedError(typed.message, typed.type, typed.retryable, detail ? `${message}: ${detail}` : message)
+  const scrubbed = scrubKey(detail, key)
+  return typedError(typed.message, typed.type, typed.retryable, scrubbed ? `${message}: ${scrubbed}` : message)
 }

package/js/session/adapters/_pricing.js

@@ -10,6 +10,16 @@
  */
 
 import { setCatalog, specFor } from './_catalog.js'
+import { providerOf } from '#core/model-id.js'
+import providers from '../../../src/lib/providers.js'
+
+// `plan` and `capacity` calls have no per-call API invoice, so `cost`, which is
+// API USD, is 0 whatever the catalog prices. `fake` and `echo` are adapters,
+// not providers: they carry no declaration and price from the catalog.
+function unmetered (model) {
+  const def = providers[providerOf(model)]
+  return def !== undefined && def.billing.kind !== 'metered'
+}
 
 /**
  * Pure cost computation from spec + usage.
@@ -112,11 +122,40 @@ function resolveTier (price, tokens) {
  * @returns {number}
  */
 export function costFor (envelope, usage) {
-  // Plan usage has no per-call API invoice; it still consumes the plan allowance.
-  if (envelope.model.startsWith('chatgpt/')) return 0
+  if (unmetered(envelope.model)) return 0
   return computeCost(specFor(envelope), usage)
 }
 
+/**
+ * @param {{model: string}} envelope
+ * @param {any} spec
+ * @param {{durationSeconds?: number | null, inputTokens?: number, outputTokens?: number}} usage
+ * @returns {number}
+ */
+export function transcriptionCostFor (envelope, spec, usage) {
+  return unmetered(envelope.model) ? 0 : computeTranscriptionCost(spec, usage)
+}
+
+/**
+ * @param {{model: string}} envelope
+ * @param {any} spec
+ * @param {{inputTokens?: number}} usage
+ * @returns {number}
+ */
+export function embeddingCostFor (envelope, spec, usage) {
+  return unmetered(envelope.model) ? 0 : computeEmbeddingCost(spec, usage)
+}
+
+/**
+ * @param {{model: string}} envelope
+ * @param {any} spec
+ * @param {{inputTokens: number, outputTokens: number}} usage
+ * @returns {number}
+ */
+export function evaluationCostFor (envelope, spec, usage) {
+  return unmetered(envelope.model) ? 0 : computeCost(spec, usage)
+}
+
 /**
  * Cost of a transcription call.
  *

package/js/session/adapters/embedding/cohere.js

@@ -12,7 +12,7 @@
 
 import { getSpec } from '../_catalog.js'
 import { classifyProviderError, fromHttpStatus, typedError } from '../_errors.js'
-import { computeEmbeddingCost } from '../_pricing.js'
+import { embeddingCostFor } from '../_pricing.js'
 import { catalogKey, bareOf } from '#core/model-id.js'
 import { checkBatch, checkDimensions, resolveInputType, widthOf } from './_shared.js'
 
@@ -61,7 +61,7 @@ export async function cohereEmbedding (envelope, deps = {}) {
 
   if (!res.ok) {
     const detail = await res.text().catch(() => '')
-    throw fromHttpStatus(res.status, detail, envelope.auth?.key)
+    throw fromHttpStatus(res.status, 'embedding request failed', detail.slice(0, 500), envelope.auth?.key)
   }
 
   const payload = await res.json()
@@ -85,7 +85,7 @@ export async function cohereEmbedding (envelope, deps = {}) {
     dimensions: widthOf(vectors),
     inputType,
     inputTokens,
-    cost: computeEmbeddingCost(spec, { inputTokens }),
+    cost: embeddingCostFor(envelope, spec, { inputTokens }),
     timestamps: { start, first: end, end }
   }
 }

package/js/session/adapters/embedding/gemini.js

@@ -13,7 +13,7 @@
 
 import { getSpec } from '../_catalog.js'
 import { classifyProviderError, fromHttpStatus, typedError } from '../_errors.js'
-import { computeEmbeddingCost } from '../_pricing.js'
+import { embeddingCostFor } from '../_pricing.js'
 import { catalogKey, bareOf } from '#core/model-id.js'
 import { checkBatch, checkDimensions, resolveInputType, widthOf } from './_shared.js'
 
@@ -59,7 +59,7 @@ export async function geminiEmbedding (envelope, deps = {}) {
 
   if (!res.ok) {
     const detail = await res.text().catch(() => '')
-    throw fromHttpStatus(res.status, detail, envelope.auth?.key)
+    throw fromHttpStatus(res.status, 'embedding request failed', detail.slice(0, 500), envelope.auth?.key)
   }
 
   const payload = await res.json()
@@ -86,7 +86,7 @@ export async function geminiEmbedding (envelope, deps = {}) {
     dimensions: widthOf(vectors),
     inputType: taskType,
     inputTokens,
-    cost: computeEmbeddingCost(spec, { inputTokens }),
+    cost: embeddingCostFor(envelope, spec, { inputTokens }),
     timestamps: { start, first: end, end }
   }
 }

package/js/session/adapters/embedding/openai_compatible.js

@@ -11,7 +11,7 @@
 
 import { getSpec } from '../_catalog.js'
 import { classifyProviderError, fromHttpStatus, typedError } from '../_errors.js'
-import { computeEmbeddingCost } from '../_pricing.js'
+import { embeddingCostFor } from '../_pricing.js'
 import { catalogKey, bareOf } from '#core/model-id.js'
 import { checkBatch, checkDimensions, resolveInputType, widthOf } from './_shared.js'
 
@@ -56,7 +56,7 @@ export function createEmbeddingAdapter ({ baseURL, dimensionsField = 'dimensions
 
     if (!res.ok) {
       const detail = await res.text().catch(() => '')
-      throw fromHttpStatus(res.status, detail, envelope.auth?.key)
+      throw fromHttpStatus(res.status, 'embedding request failed', detail.slice(0, 500), envelope.auth?.key)
     }
 
     const payload = await res.json()
@@ -85,7 +85,7 @@ export function createEmbeddingAdapter ({ baseURL, dimensionsField = 'dimensions
       dimensions: widthOf(vectors),
       inputType,
       inputTokens,
-      cost: computeEmbeddingCost(spec, { inputTokens }),
+      cost: embeddingCostFor(envelope, spec, { inputTokens }),
       timestamps: { start, first: end, end }
     }
   }

package/js/session/adapters/evaluation/_shared.js

@@ -0,0 +1,73 @@
+/**
+ * Pre-dispatch checks every evaluation adapter runs. They reject what the
+ * gate's serde types reject, so a malformed envelope fails the same way
+ * in-process.
+ *
+ * @module session/adapters/evaluation/shared
+ */
+
+import { typedError } from '../_errors.js'
+import { QUESTION_TYPES } from '#core/evaluation.js'
+
+const QUESTION_KEYS = new Set(['type', 'instructions', 'criteria'])
+const BINARY_CRITERIA_KEYS = ['yes', 'no']
+
+/**
+ * @param {import('#core/evaluation.js').EvaluateEnvelope} envelope
+ * @param {any} spec
+ */
+export function checkEvaluation (envelope, spec) {
+  const { state, questions } = envelope
+  if (typeof state !== 'string' && !isObject(state) && !Array.isArray(state)) {
+    throw invalid('state must be a string, an object or an array')
+  }
+  if (!isObject(questions) || Object.keys(questions).length === 0) {
+    throw invalid('questions must be a non-empty object')
+  }
+  for (const [id, q] of Object.entries(questions)) {
+    checkQuestion(id, q, spec)
+  }
+}
+
+function checkQuestion (id, q, spec) {
+  if (!isObject(q)) throw invalid(`question '${id}' must be an object`)
+  const extra = Object.keys(q).filter(k => !QUESTION_KEYS.has(k))
+  if (extra.length) throw invalid(`question '${id}' has unknown field '${extra.join("', '")}'`)
+  if (!QUESTION_TYPES.includes(q.type)) {
+    throw invalid(`question '${id}' type must be one of ${QUESTION_TYPES.join(', ')}`)
+  }
+  if (Array.isArray(spec?.evaluationTypes) && !spec.evaluationTypes.includes(q.type)) {
+    throw typedError(
+      `question '${id}' is '${q.type}', which this model does not answer; it answers ${spec.evaluationTypes.join(', ')}`,
+      'EVALUATE_QUESTION_TYPE_UNSUPPORTED',
+      false
+    )
+  }
+  if (q.instructions === undefined || q.instructions === null) {
+    throw invalid(`question '${id}' requires instructions`)
+  }
+
+  const c = q.criteria
+  if (q.type === 'binary') {
+    if (c === undefined) return
+    if (!isObject(c) || Object.keys(c).length !== 2 || !BINARY_CRITERIA_KEYS.every(k => c[k] !== undefined && c[k] !== null)) {
+      throw invalid(`question '${id}' criteria must have exactly 'yes' and 'no'`)
+    }
+  } else if (q.type === 'choice') {
+    if (!isObject(c) || Object.keys(c).length < 2) {
+      throw invalid(`question '${id}' criteria must name at least two options`)
+    }
+  } else if (!Array.isArray(c) || c.length < 2) {
+    throw invalid(`question '${id}' criteria must list at least two levels`)
+  }
+}
+
+/** @param {unknown} v */
+function isObject (v) {
+  return typeof v === 'object' && v !== null && !Array.isArray(v)
+}
+
+/** @param {string} message */
+function invalid (message) {
+  return typedError(message, 'EVALUATE_INPUT_INVALID', false)
+}

package/js/session/adapters/evaluation/index.js

@@ -0,0 +1,22 @@
+/**
+ * Evaluation-adapter registry.
+ *
+ * @module session/adapters/evaluation
+ */
+
+import { typesafeEvaluation } from './typesafe.js'
+
+const EVALUATION_ADAPTERS = {
+  typesafe: typesafeEvaluation
+}
+
+export const EVALUATION_PROVIDERS = Object.freeze(Object.keys(EVALUATION_ADAPTERS))
+
+/**
+ * @param {string} provider
+ */
+export function getEvaluationAdapter (provider) {
+  const adapter = EVALUATION_ADAPTERS[provider]
+  if (!adapter) throw new Error(`no evaluation adapter for provider: ${provider}`)
+  return adapter
+}

package/js/session/adapters/evaluation/typesafe.js

@@ -0,0 +1,118 @@
+/**
+ * TypeSafe evaluation adapter (`POST /v1/systemone`). TypeSafe calls a binary
+ * question `noul` and keys binary criteria `true` / `false`; score
+ * probabilities come back keyed by level index as strings.
+ *
+ * @module session/adapters/evaluation/typesafe
+ */
+
+import { getSpec } from '../_catalog.js'
+import { classifyProviderError, fromHttpStatus, typedError } from '../_errors.js'
+import { evaluationCostFor } from '../_pricing.js'
+import { catalogKey, bareOf } from '#core/model-id.js'
+import { checkEvaluation } from './_shared.js'
+
+const BASE_URL = 'https://api.typesafe.ai/v1'
+
+export async function typesafeEvaluation (envelope, deps = {}) {
+  const fetchFn = deps.fetch ?? globalThis.fetch
+  const spec = deps.spec ?? getSpec(catalogKey(envelope.model)) ?? {}
+  const start = String(process.hrtime.bigint())
+
+  checkEvaluation(envelope, spec)
+
+  const body = {
+    model: spec.model ?? bareOf(envelope.model),
+    state: envelope.state,
+    questions: Object.fromEntries(
+      Object.entries(envelope.questions).map(([id, q]) => [id, toQuestion(q)])
+    )
+  }
+
+  let res
+  try {
+    res = await fetchFn(`${BASE_URL}/systemone`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${envelope.auth.key}`
+      },
+      body: JSON.stringify(body)
+    })
+  } catch (e) {
+    throw typedError(classifyProviderError(e, envelope.auth?.key).message, 'NET_ERROR', true)
+  }
+
+  if (!res.ok) {
+    const detail = await res.text().catch(() => '')
+    throw fromHttpStatus(res.status, 'evaluation request failed', detail.slice(0, 500), envelope.auth?.key)
+  }
+
+  const payload = await res.json()
+  const usage = payload?.usage
+  if (typeof payload?.model !== 'string' || !isCount(usage?.input_tokens) || !isCount(usage?.output_tokens)) {
+    throw mismatch('response lacks model or token usage')
+  }
+
+  const answers = {}
+  for (const [id, q] of Object.entries(envelope.questions)) {
+    answers[id] = fromAnswer(id, q, payload.answers?.[id])
+  }
+
+  const inputTokens = usage.input_tokens
+  const outputTokens = usage.output_tokens
+  const end = String(process.hrtime.bigint())
+
+  return {
+    status: 'completed',
+    answers,
+    upstreamModel: payload.model,
+    inputTokens,
+    outputTokens,
+    cost: evaluationCostFor(envelope, spec, { inputTokens, outputTokens }),
+    timestamps: { start, first: end, end }
+  }
+}
+
+function toQuestion (q) {
+  if (q.type !== 'binary') return q
+  const out = { type: 'noul', instructions: q.instructions }
+  if (q.criteria) out.criteria = { true: q.criteria.yes, false: q.criteria.no }
+  return out
+}
+
+function fromAnswer (id, q, a) {
+  if (q.type === 'binary') {
+    if (a?.type !== 'noul' || !isProbability(a.noul)) throw mismatch(`no binary answer for '${id}'`)
+    return { type: 'binary', probability: a.noul }
+  }
+  if (a?.type !== q.type || !isProbability(a.confidence)) throw mismatch(`no ${q.type} answer for '${id}'`)
+
+  if (q.type === 'choice') {
+    const options = Object.keys(q.criteria)
+    if (typeof a.choice !== 'string' || !options.includes(a.choice) ||
+        !options.every(o => isProbability(a.probabilities?.[o]))) {
+      throw mismatch(`malformed choice answer for '${id}'`)
+    }
+    const probabilities = Object.fromEntries(options.map(o => [o, a.probabilities[o]]))
+    return { type: 'choice', choice: a.choice, probabilities, confidence: a.confidence }
+  }
+
+  const probabilities = q.criteria.map((_, i) => a.probabilities?.[String(i)])
+  if (typeof a.score !== 'number' || !probabilities.every(isProbability)) {
+    throw mismatch(`malformed score answer for '${id}'`)
+  }
+  return { type: 'score', score: a.score, probabilities, confidence: a.confidence }
+}
+
+function isProbability (v) {
+  return typeof v === 'number' && v >= 0 && v <= 1
+}
+
+function isCount (v) {
+  return Number.isInteger(v) && v >= 0
+}
+
+function mismatch (message) {
+  return typedError(message, 'EVALUATE_RESULT_MISMATCH', false)
+}

package/js/session/adapters/transcription/openai_compatible.js

@@ -23,7 +23,7 @@ import { basename } from 'node:path'
 import { getSpec } from '../_catalog.js'
 import { classifyProviderError, fromHttpStatus, typedError } from '../_errors.js'
 import { dataUriPayload, isTrustedMedia, mediaScheme, readLocalMedia } from '../_media.js'
-import { computeTranscriptionCost } from '../_pricing.js'
+import { transcriptionCostFor } from '../_pricing.js'
 import { catalogKey, bareOf } from '#core/model-id.js'
 
 /**
@@ -67,7 +67,7 @@ export function createTranscriptionAdapter ({ baseURL, responseFormat }) {
     const body = await res.json()
     const durationSeconds = extractDuration(body)
     const tokens = extractTokens(body)
-    const cost = computeTranscriptionCost(spec, { durationSeconds, ...tokens })
+    const cost = transcriptionCostFor(envelope, spec, { durationSeconds, ...tokens })
 
     const end = String(process.hrtime.bigint())
     return {

package/js/session/driver.js

@@ -19,6 +19,7 @@ import { run } from './run.js'
 import { runImage } from './run_image.js'
 import { runTranscription } from './run_transcription.js'
 import { runEmbedding } from './run_embedding.js'
+import { runEvaluation } from './run_evaluation.js'
 import { runInfo } from './run_info.js'
 import { setCatalog } from './adapters/_catalog.js'
 
@@ -233,6 +234,14 @@ export async function drive (stdin, stdout) {
         } else {
           await writeLine({ type: 'error', error: out.error })
         }
+      } else if (envelope.op === 'evaluate') {
+        const { op: _op, ...evEnv } = envelope
+        const out = await runEvaluation(evEnv)
+        if (out.ok) {
+          await writeLine({ type: 'evaluate_done', result: out.result })
+        } else {
+          await writeLine({ type: 'error', error: out.error })
+        }
       } else if (envelope.op === 'info') {
         const out = await runInfo(envelope)
         if (out.ok) {

package/js/session/run.js

@@ -23,6 +23,7 @@
  */
 
 import { ADAPTER_NAMES, isImageProvider } from './adapters/_registry.js'
+import { EVALUATION_PROVIDERS } from './adapters/evaluation/index.js'
 import { getSpec } from './adapters/_catalog.js'
 import { getProviderLimits } from './adapters/_providers.js'
 import { hasSpeed, mergeSpeed, speedHasOwnQuota, speedNames } from './adapters/_speed.js'
@@ -119,6 +120,13 @@ export async function * run (envelope, {
       yield err
       return
     }
+    if (EVALUATION_PROVIDERS.includes(provider)) {
+      const detail = `provider '${provider}' supports evaluation only; use evaluate(...) instead`
+      log.warn({ provider }, '[mohdel:answer] evaluation-only provider via answer')
+      endSpanError(span, new Error(detail))
+      yield errorEvent(detail, 'PROVIDER_TEXT_NOT_SUPPORTED')
+      return
+    }
     const err = errorEvent(messageOf(e), 'SESSION_UNKNOWN_PROVIDER')
     log.warn({ err: e, provider }, '[mohdel:answer] unknown provider')
     endSpanError(span, e)

package/js/session/run_evaluation.js

@@ -0,0 +1,84 @@
+/**
+ * Evaluation runtime. Resolves the adapter for the envelope's provider and
+ * returns either a result or a typed error, never throwing. Rate limits as in
+ * `run_embedding.js`, minus the input count.
+ *
+ * @module session/run_evaluation
+ */
+
+import { getEvaluationAdapter } from './adapters/evaluation/index.js'
+import { classifyProviderError } from './adapters/_errors.js'
+import { getProviderLimits } from './adapters/_providers.js'
+import * as defaultLimiter from './_rate_limiter.js'
+import { providerOf } from '#core/model-id.js'
+
+/**
+ * @param {import('#core/evaluation.js').EvaluateEnvelope} envelope
+ * @param {{
+ *   resolveAdapter?: (provider: string) => any,
+ *   resolveProviderLimits?: (provider: string) => any,
+ *   limiter?: any,
+ *   sleep?: (ms: number) => Promise<void>,
+ *   modelKey?: string,
+ *   spec?: any
+ * }} [options]
+ * @returns {Promise<
+ *   | {ok: true, result: import('#core/evaluation.js').EvaluateResult}
+ *   | {ok: false, error: import('#core/errors.js').TypedError}
+ * >}
+ */
+export async function runEvaluation (envelope, {
+  resolveAdapter = getEvaluationAdapter,
+  resolveProviderLimits = getProviderLimits,
+  limiter = defaultLimiter,
+  sleep = defaultSleep,
+  modelKey = envelope.model,
+  spec
+} = {}) {
+  const provider = providerOf(envelope.model)
+
+  let adapter
+  try {
+    adapter = resolveAdapter(provider)
+  } catch (e) {
+    return {
+      ok: false,
+      error: {
+        message: messageOf(e),
+        severity: 'error',
+        retryable: false,
+        type: 'SESSION_UNKNOWN_PROVIDER'
+      }
+    }
+  }
+
+  const providerCfg = resolveProviderLimits(provider) || {}
+  const rpmLimit = spec?.rpmLimit ?? providerCfg.rpmLimit
+  const tpmLimit = spec?.tpmLimit ?? providerCfg.tpmLimit
+  const bucketKey = spec?.rateLimitScope === 'model' ? modelKey : provider
+
+  if (rpmLimit != null || tpmLimit != null) {
+    const delay = limiter.check(bucketKey, { rpmLimit, tpmLimit })
+    if (delay > 0) await sleep(delay)
+    limiter.recordRequest(bucketKey)
+  }
+
+  try {
+    const result = await adapter(envelope, spec ? { spec } : {})
+    if (tpmLimit != null) limiter.recordTokens(bucketKey, result.inputTokens + result.outputTokens)
+    return { ok: true, result }
+  } catch (e) {
+    const typed = /** @type {any} */(e).typed || classifyProviderError(e, envelope.auth?.key)
+    return { ok: false, error: typed }
+  }
+}
+
+/** @param {unknown} e */
+function messageOf (e) {
+  return e instanceof Error ? e.message : String(e)
+}
+
+/** @param {number} ms */
+function defaultSleep (ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mohdel",
-  "version": "3.6.1",
+  "version": "3.8.0",
   "license": "MIT",
   "author": {
     "name": "Christophe Le Bars",
@@ -144,7 +144,7 @@
     "@opentelemetry/exporter-trace-otlp-grpc": "^0.222.0",
     "@opentelemetry/sdk-node": "^0.222.0",
     "chalk": "^6.0.1",
-    "mohdel-thin-gate-linux-x64-gnu": "3.6.1"
+    "mohdel-thin-gate-linux-x64-gnu": "3.8.0"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.129.0",

package/src/cli/ask.js

@@ -1,5 +1,6 @@
 import mohdel, { silent } from '../lib/index.js'
 import { getConfig, loadDefaultEnv } from '../lib/common.js'
+import providerDefs, { billingOf } from '../lib/providers.js'
 
 const noop = () => {}
 
@@ -47,8 +48,10 @@ export const hintsForError = (err, modelId) => {
     else hints.push('→ run:  mo                          # interactive provider/key setup')
   }
 
-  if (provider === 'chatgpt' && /RATE_LIMIT|QUOTA_EXHAUSTED|429|usage limit/i.test(`${err?.type || ''} ${both}`)) {
-    hints.push('→ manage ChatGPT plan usage: https://chatgpt.com/settings/usage')
+  // The id is what the user typed, so its provider may not exist.
+  const usage = providerDefs[provider]?.billing.usage
+  if (usage && /RATE_LIMIT|QUOTA_EXHAUSTED|429|usage limit/i.test(`${err?.type || ''} ${both}`)) {
+    hints.push(`→ manage ${providerDefs[provider].billing.label} usage: ${usage}`)
   }
 
   if (/deprecated/i.test(both) && /replacement/i.test(both)) {
@@ -223,7 +226,8 @@ Examples:
     if (tokens.inputTokens) summary.push(`${tokens.inputTokens} in`)
     if (tokens.outputTokens) summary.push(`${tokens.outputTokens} out`)
     if (tokens.thinkingTokens) summary.push(`${tokens.thinkingTokens} think`)
-    if (model.id.startsWith('chatgpt/')) summary.push('Using ChatGPT plan — manage usage: https://chatgpt.com/settings/usage')
+    const billing = billingOf(model.id)
+    if (billing.kind !== 'metered') summary.push(`not metered · ${billing.label}${billing.usage ? ` — manage usage: ${billing.usage}` : ''}`)
     else if (tokens.cost != null) summary.push(`$${tokens.cost.toFixed(4)}`)
     if (tokens.speed) {
       const served = tokens.servedSpeed

package/src/cli/model.js

@@ -731,11 +731,11 @@ Alibaba's Qwen. Routing follows the provider — see "mo provider --help".`)
 
 // The provider API returns ids, never prices — curated entries land unpriced.
 function printCurateNext (providerName) {
-  if (providerName === 'chatgpt') {
-    console.log('ChatGPT plan models are ready. Use mo ask chatgpt/<model-slug> "your prompt". Plan usage is not API billing.')
+  const def = providerDefs[providerName]
+  if (def.billing.kind !== 'metered') {
+    console.log(`${providerName} models are ready. Use mo ask ${providerName}/<model-id> "your prompt". Calls are not metered: they draw on your ${def.billing.label}.`)
     return
   }
-  const def = providerDefs[providerName]
   if (def?.pricesFromApi) {
     console.log(`\n${meta(`${providerName} publishes prices in its model list — the entries are complete.`)}`)
     console.log(`${meta('Check them:')} mo ls  ${meta('│')}  mo check`)
@@ -779,6 +779,9 @@ const SINGLE_DIMENSION_PRICES = [
 ]
 
 function formatPrice (info) {
+  // An entry naming an unknown provider is `mo check`'s to report; it lists with its prices.
+  const billing = providerDefs[info.provider]?.billing
+  if (billing && billing.kind !== 'metered') return meta(`not metered · ${billing.label}`)
   const inp = resolvePrice(info.inputPrice)
   const out = resolvePrice(info.outputPrice)
   if (inp || out) return price(`$${inp}`) + meta('/') + price(`$${out}`)

package/src/lib/catalog-review.js

@@ -73,6 +73,9 @@ export const reviewEntry = (key, spec, catalog, { strict = false, local = null }
   if (spec.provider && spec.provider !== keyProvider) {
     errors.push(`${key}: spec.provider '${spec.provider}' doesn't match key prefix '${keyProvider}'`)
   }
+  if (providerConfig?.billing.kind === 'capacity' && Object.keys(spec).some(k => k.endsWith('Price'))) {
+    warnings.push(`${key}: prices are never charged — ${providerConfig.billing.label} calls report cost 0`)
+  }
   if (providerConfig && spec.sdk && spec.sdk !== providerConfig.sdk) {
     errors.push(`${key}: spec.sdk '${spec.sdk}' doesn't match provider sdk '${providerConfig.sdk}'`)
   }

package/src/lib/creators.js

@@ -65,6 +65,12 @@ const creators = {
     logo: 'cohere.svg',
     description: 'Cohere builds retrieval-focused models: embeddings and rerankers aimed at enterprise search rather than chat.'
   },
+  typesafe: {
+    prefixes: ['jev'],
+    label: 'TypeSafe',
+    logo: 'typesafe.svg',
+    description: 'TypeSafe trains decision models that answer typed questions with calibrated probabilities instead of generating text.'
+  },
   nomic: {
     prefixes: ['nomic-embed'],
     label: 'Nomic',

package/src/lib/index.js

@@ -16,7 +16,7 @@ import {
 import { createRateLimiter } from '../../js/session/_rate_limiter.js'
 import { createCooldownTracker } from '../../js/session/_cooldown.js'
 import { setCatalog } from '../../js/session/adapters/_catalog.js'
-import { runAnswer, runAnswerEmbedding, runAnswerImage, runAnswerTranscription } from '../../js/factory/bridge.js'
+import { runAnswer, runAnswerEmbedding, runAnswerEvaluation, runAnswerImage, runAnswerTranscription } from '../../js/factory/bridge.js'
 import { startSpan, endSpanOk, endSpanError } from './tracing.js'
 import { isValidTag } from './schema.js'
 import { silent } from './logger.js'
@@ -751,6 +751,22 @@ const createModelProxy = (resolvedModelId, modelSpec, handlers, aliasOutputEffor
         }
       }
 
+      if (prop === 'evaluate') {
+        return async (state, questions, options = {}) => {
+          const { configuration } = await getRuntime()
+          return runAnswerEvaluation({
+            provider: modelSpec.provider,
+            model: modelSpec.model ?? resolvedModelId.split('/').pop(),
+            modelKey: resolvedModelId,
+            configuration,
+            state,
+            questions,
+            options,
+            spec: modelSpec
+          }, { limiter: rateLimiter, resolveProviderLimits })
+        }
+      }
+
       if (prop === 'setRateLimit') {
         return async ({ rpm, tpm, inpm } = {}) => {
           const curatedCache = getCuratedCacheSnapshot()

package/src/lib/provider-info.js

@@ -100,6 +100,13 @@ const PROVIDER_INFO = {
     hint: 'Create an API key in the Cohere dashboard under API Keys',
     free: true
   },
+  typesafe: {
+    label: 'TypeSafe',
+    description: 'Calibrated yes/no, choice and score answers about a state. No chat models through mohdel.',
+    url: 'https://console.typesafe.ai/keys',
+    hint: 'Create an API key in the TypeSafe console under Keys',
+    free: false
+  },
   qwen: {
     label: 'Qwen Cloud',
     description: 'Qwen — reasoning, coding, long context. Free quota for new users.',

package/src/lib/providers.js

@@ -1,3 +1,5 @@
+import { providerOf } from '#core/model-id.js'
+
 const LOCAL_API_KEY_ENV = 'MOHDEL_LOCAL_API_SK'
 
 // `contextSemantics` and `outputCapStrategy` are published facts about a
@@ -6,8 +8,13 @@ const LOCAL_API_KEY_ENV = 'MOHDEL_LOCAL_API_SK'
 // own provider requests does not have to rediscover the behaviour one 400 at a
 // time. Entries may override `outputCapStrategy` per model. See
 // ARCHITECTURE.md > "The output budget is capped to the model's ceiling".
+// `billing` is how a provider's calls are paid for. `metered`: API money,
+// reported as `cost`. `plan`: a share of a subscription allowance, seen at
+// `usage`. `capacity`: hardware run or rented at a flat rate. `cost` is 0 for
+// the last two.
 const providers = {
   anthropic: {
+    billing: { kind: 'metered' },
     sdk: 'anthropic',
     apiKeyEnv: 'ANTHROPIC_API_SK',
     createConfiguration: apiKey => ({ apiKey }),
@@ -20,6 +27,7 @@ const providers = {
     outputCapStrategy: 'error'
   },
   cerebras: {
+    billing: { kind: 'metered' },
     sdk: 'cerebras',
     apiKeyEnv: 'CEREBRAS_API_SK',
     createConfiguration: apiKey => ({ apiKey }),
@@ -32,6 +40,7 @@ const providers = {
     outputCapStrategy: 'accept'
   },
   chatgpt: {
+    billing: { kind: 'plan', label: 'ChatGPT plan', usage: 'https://chatgpt.com/settings/usage' },
     sdk: 'openai',
     catalogClient: 'chatgpt',
     refreshConfiguration: true,
@@ -47,6 +56,7 @@ const providers = {
     outputCapStrategy: 'accept'
   },
   deepseek: {
+    billing: { kind: 'metered' },
     sdk: 'openai',
     api: 'chatCompletions',
     apiKeyEnv: 'DEEPSEEK_API_SK',
@@ -61,6 +71,7 @@ const providers = {
     outputCapStrategy: 'accept'
   },
   fireworks: {
+    billing: { kind: 'metered' },
     sdk: 'fireworks',
     apiKeyEnv: 'FIREWORKS_API_SK',
     baseURL: 'https://api.fireworks.ai/inference/v1',
@@ -74,6 +85,7 @@ const providers = {
     outputCapStrategy: 'accept'
   },
   gemini: {
+    billing: { kind: 'metered' },
     sdk: 'gemini',
     apiKeyEnv: 'GEMINI_API_SK',
     createConfiguration: apiKey => ({ apiKey }),
@@ -86,6 +98,7 @@ const providers = {
     outputCapStrategy: 'accept'
   },
   groq: {
+    billing: { kind: 'metered' },
     sdk: 'groq',
     apiKeyEnv: 'GROQ_API_SK',
     createConfiguration: apiKey => ({ apiKey }),
@@ -96,6 +109,7 @@ const providers = {
     }
   },
   local: {
+    billing: { kind: 'capacity', label: 'local server' },
     sdk: 'openai',
     api: 'chatCompletions',
     catalog: false,
@@ -105,6 +119,7 @@ const providers = {
     outputCapStrategy: 'accept'
   },
   meta: {
+    billing: { kind: 'metered' },
     sdk: 'openai',
     apiKeyEnv: 'META_API_SK',
     baseURL: 'https://api.meta.ai/v1',
@@ -117,6 +132,7 @@ const providers = {
     outputCapStrategy: 'accept'
   },
   cohere: {
+    billing: { kind: 'metered' },
     sdk: 'cohere',
     api: 'embeddings',
     apiKeyEnv: 'COHERE_API_SK',
@@ -128,7 +144,22 @@ const providers = {
       rateLimits: 'https://docs.cohere.com/docs/rate-limits'
     }
   },
+  typesafe: {
+    billing: { kind: 'metered' },
+    sdk: 'typesafe',
+    api: 'evaluation',
+    catalog: false,
+    apiKeyEnv: 'TYPESAFE_API_SK',
+    baseURL: 'https://api.typesafe.ai/v1',
+    createConfiguration: apiKey => ({ apiKey }),
+    references: {
+      pricing: 'https://docs.typesafe.ai/models',
+      models: 'https://docs.typesafe.ai/models',
+      rateLimits: 'https://docs.typesafe.ai/models'
+    }
+  },
   mistral: {
+    billing: { kind: 'metered' },
     sdk: 'openai',
     api: 'chatCompletions',
     apiKeyEnv: 'MISTRAL_API_SK',
@@ -141,6 +172,7 @@ const providers = {
     }
   },
   novita: {
+    billing: { kind: 'metered' },
     sdk: 'openai',
     api: 'chatCompletions',
     imageHandler: 'novita',
@@ -157,6 +189,7 @@ const providers = {
     outputCapStrategy: 'error'
   },
   openai: {
+    billing: { kind: 'metered' },
     sdk: 'openai',
     apiKeyEnv: 'OPENAI_API_SK',
     createConfiguration: apiKey => ({ apiKey }),
@@ -169,6 +202,7 @@ const providers = {
     outputCapStrategy: 'accept'
   },
   openrouter: {
+    billing: { kind: 'metered' },
     sdk: 'openrouter',
     apiKeyEnv: 'OPENROUTER_API_SK',
     baseURL: 'https://openrouter.ai/api/v1',
@@ -184,6 +218,7 @@ const providers = {
     }
   },
   qwen: {
+    billing: { kind: 'metered' },
     sdk: 'openai',
     api: 'chatCompletions',
     apiKeyEnv: 'QWEN_API_SK',
@@ -198,6 +233,7 @@ const providers = {
     outputCapStrategy: 'accept'
   },
   xai: {
+    billing: { kind: 'metered' },
     sdk: 'openai',
     apiKeyEnv: 'XAI_API_SK',
     baseURL: 'https://api.x.ai/v1',
@@ -211,6 +247,7 @@ const providers = {
     outputCapStrategy: 'accept'
   },
   xiaomi: {
+    billing: { kind: 'metered' },
     sdk: 'openai',
     api: 'chatCompletions',
     apiKeyEnv: 'XIAOMI_API_SK',
@@ -223,4 +260,16 @@ const providers = {
 
 Object.freeze(providers)
 
+/**
+ * How calls to `modelId`'s provider are paid for.
+ * @param {string} modelId
+ * @returns {{kind: 'metered' | 'plan' | 'capacity', label?: string, usage?: string}}
+ */
+export function billingOf (modelId) {
+  const provider = providerOf(modelId)
+  const def = providers[provider]
+  if (!def) throw new Error(`Unknown provider '${provider}' in model id '${modelId}'`)
+  return { ...def.billing }
+}
+
 export default providers

package/src/lib/schema.js

@@ -65,6 +65,7 @@ const fieldDefs = {
   maxInputTokens: { type: 'number' },
   inputTypes: { type: 'object' },
   defaultInputType: { type: 'string' },
+  evaluationTypes: { type: 'array', itemType: 'string' },
   deprecated: { type: 'string' },
   suspended: { type: 'string' },
   rpmLimit: { type: 'number' },