openmates 0.12.0 → 0.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -986,14 +986,14 @@ var OpenMatesWsClient = class {
986
986
  });
987
987
  }
988
988
  async open(timeoutMs = 1e4) {
989
- await new Promise((resolve5, reject) => {
989
+ await new Promise((resolve6, reject) => {
990
990
  const timeout = setTimeout(
991
991
  () => reject(new Error("WebSocket open timeout")),
992
992
  timeoutMs
993
993
  );
994
994
  this.socket.once("open", () => {
995
995
  clearTimeout(timeout);
996
- resolve5();
996
+ resolve6();
997
997
  });
998
998
  this.socket.once("error", (error) => {
999
999
  clearTimeout(timeout);
@@ -1022,15 +1022,15 @@ var OpenMatesWsClient = class {
1022
1022
  this.socket.send(JSON.stringify({ type, payload }));
1023
1023
  }
1024
1024
  sendAsync(type, payload) {
1025
- return new Promise((resolve5, reject) => {
1025
+ return new Promise((resolve6, reject) => {
1026
1026
  this.socket.send(JSON.stringify({ type, payload }), (error) => {
1027
1027
  if (error) reject(error);
1028
- else resolve5();
1028
+ else resolve6();
1029
1029
  });
1030
1030
  });
1031
1031
  }
1032
1032
  waitForMessage(expectedType, predicate, timeoutMs = 2e4) {
1033
- return new Promise((resolve5, reject) => {
1033
+ return new Promise((resolve6, reject) => {
1034
1034
  const onMessage = (rawData) => {
1035
1035
  try {
1036
1036
  const parsed = JSON.parse(rawData.toString());
@@ -1041,7 +1041,7 @@ var OpenMatesWsClient = class {
1041
1041
  return;
1042
1042
  }
1043
1043
  cleanup();
1044
- resolve5(parsed);
1044
+ resolve6(parsed);
1045
1045
  } catch {
1046
1046
  }
1047
1047
  };
@@ -1074,14 +1074,14 @@ var OpenMatesWsClient = class {
1074
1074
  * Used by ensureSynced to consume the full phased-sync event stream.
1075
1075
  */
1076
1076
  collectMessages(terminatorType, timeoutMs = 9e4) {
1077
- return new Promise((resolve5, reject) => {
1077
+ return new Promise((resolve6, reject) => {
1078
1078
  const collected = [];
1079
1079
  const onMessage = (rawData) => {
1080
1080
  try {
1081
1081
  const parsed = JSON.parse(rawData.toString());
1082
1082
  if (parsed.type === terminatorType) {
1083
1083
  cleanup();
1084
- resolve5(collected);
1084
+ resolve6(collected);
1085
1085
  return;
1086
1086
  }
1087
1087
  collected.push(parsed);
@@ -1094,7 +1094,7 @@ var OpenMatesWsClient = class {
1094
1094
  };
1095
1095
  const onClose = () => {
1096
1096
  cleanup();
1097
- resolve5(collected);
1097
+ resolve6(collected);
1098
1098
  };
1099
1099
  const timeout = setTimeout(() => {
1100
1100
  cleanup();
@@ -1132,7 +1132,7 @@ var OpenMatesWsClient = class {
1132
1132
  const timeoutMs = options?.timeoutMs ?? 9e4;
1133
1133
  const onStream = options?.onStream;
1134
1134
  const asyncEmbedWaitMs = options?.asyncEmbedWaitMs ?? 12e4;
1135
- return new Promise((resolve5, reject) => {
1135
+ return new Promise((resolve6, reject) => {
1136
1136
  let latestContent = "";
1137
1137
  let messageId = null;
1138
1138
  let taskId = null;
@@ -1189,7 +1189,7 @@ var OpenMatesWsClient = class {
1189
1189
  if (waitingForUserPayload) {
1190
1190
  if (pendingSubChatHandlers.size > 0) return;
1191
1191
  cleanup();
1192
- resolve5({
1192
+ resolve6({
1193
1193
  status: "waiting_for_user",
1194
1194
  messageId,
1195
1195
  taskId,
@@ -1209,7 +1209,7 @@ var OpenMatesWsClient = class {
1209
1209
  if (processingEmbedIds.size > 0 && !asyncEmbedTimer) {
1210
1210
  asyncEmbedTimer = setTimeout(() => {
1211
1211
  cleanup();
1212
- resolve5({
1212
+ resolve6({
1213
1213
  status: "completed",
1214
1214
  messageId,
1215
1215
  taskId,
@@ -1226,7 +1226,7 @@ var OpenMatesWsClient = class {
1226
1226
  }
1227
1227
  if (processingEmbedIds.size > 0) return;
1228
1228
  cleanup();
1229
- resolve5({
1229
+ resolve6({
1230
1230
  status: "completed",
1231
1231
  messageId,
1232
1232
  taskId,
@@ -1440,7 +1440,7 @@ var OpenMatesWsClient = class {
1440
1440
  const onClose = () => {
1441
1441
  if (aiResponseDone) {
1442
1442
  cleanup();
1443
- resolve5({
1443
+ resolve6({
1444
1444
  status: "completed",
1445
1445
  messageId,
1446
1446
  taskId,
@@ -2805,13 +2805,21 @@ var OpenMatesClient = class _OpenMatesClient {
2805
2805
  }
2806
2806
  const chatId = `anonymous-${randomUUID2()}`;
2807
2807
  const messageId = `anonymous-message-${randomUUID2()}`;
2808
- const response = await this.http.post("/v1/anonymous/chat/stream", {
2808
+ const requestBody = {
2809
2809
  anonymous_id: anonymousId,
2810
2810
  client_chat_id: chatId,
2811
2811
  client_message_id: messageId,
2812
2812
  plaintext_message: params.message,
2813
2813
  message_history: []
2814
- });
2814
+ };
2815
+ if (params.learningMode?.enabled === true) {
2816
+ requestBody.learning_mode = {
2817
+ enabled: true,
2818
+ age_group: params.learningMode.ageGroup ?? null,
2819
+ source: params.learningMode.source ?? "anonymous_session"
2820
+ };
2821
+ }
2822
+ const response = await this.http.post("/v1/anonymous/chat/stream", requestBody);
2815
2823
  if (!response.ok) {
2816
2824
  const detail = response.data.detail;
2817
2825
  const message = typeof detail === "object" && detail?.message ? detail.message : typeof detail === "string" ? detail : `Anonymous chat failed with HTTP ${response.status}`;
@@ -2979,6 +2987,41 @@ var OpenMatesClient = class _OpenMatesClient {
2979
2987
  saveSession(session);
2980
2988
  return response.data.user ?? {};
2981
2989
  }
2990
+ async getLearningModeStatus() {
2991
+ this.requireSession();
2992
+ const response = await this.http.get(
2993
+ "/v1/learning-mode",
2994
+ this.getCliRequestHeaders()
2995
+ );
2996
+ if (!response.ok) {
2997
+ throw new Error(`Learning Mode status failed with HTTP ${response.status}`);
2998
+ }
2999
+ return response.data;
3000
+ }
3001
+ async activateLearningMode(params) {
3002
+ this.requireSession();
3003
+ const response = await this.http.post(
3004
+ "/v1/learning-mode/activate",
3005
+ { age_group: params.ageGroup, passcode: params.passcode },
3006
+ this.getCliRequestHeaders()
3007
+ );
3008
+ if (!response.ok) {
3009
+ throw new Error(`Learning Mode activation failed with HTTP ${response.status}`);
3010
+ }
3011
+ return response.data;
3012
+ }
3013
+ async deactivateLearningMode(passcode) {
3014
+ this.requireSession();
3015
+ const response = await this.http.post(
3016
+ "/v1/learning-mode/deactivate",
3017
+ { passcode },
3018
+ this.getCliRequestHeaders()
3019
+ );
3020
+ if (!response.ok) {
3021
+ throw new Error(`Learning Mode deactivation failed with HTTP ${response.status}`);
3022
+ }
3023
+ return response.data;
3024
+ }
2982
3025
  async logout() {
2983
3026
  if (this.session) {
2984
3027
  await this.http.post("/v1/auth/logout", {}, this.getCliRequestHeaders()).catch(() => void 0);
@@ -3677,6 +3720,29 @@ var OpenMatesClient = class _OpenMatesClient {
3677
3720
  if (connectedAccountTokenRefs.length > 0) {
3678
3721
  messagePayload.connected_account_token_refs = connectedAccountTokenRefs;
3679
3722
  }
3723
+ if (params.benchmarkMetadata) {
3724
+ messagePayload.benchmark_metadata = params.benchmarkMetadata;
3725
+ }
3726
+ if (params.learningMode) {
3727
+ messagePayload.learning_mode = {
3728
+ enabled: params.learningMode.enabled,
3729
+ age_group: params.learningMode.ageGroup ?? null
3730
+ };
3731
+ }
3732
+ if (params.incognito) {
3733
+ const providedHistory = (params.messageHistory ?? []).map((historyMessage) => ({
3734
+ ...historyMessage,
3735
+ chat_id: historyMessage.chat_id ?? chatId
3736
+ }));
3737
+ messagePayload.message_history = [...providedHistory, {
3738
+ message_id: messageId,
3739
+ chat_id: chatId,
3740
+ role: "user",
3741
+ sender_name: "User",
3742
+ content: params.message,
3743
+ created_at: createdAt
3744
+ }];
3745
+ }
3680
3746
  let chatKeyBytes = null;
3681
3747
  let encryptedChatKey = null;
3682
3748
  let baselineMessagesV = 0;
@@ -3735,6 +3801,7 @@ var OpenMatesClient = class _OpenMatesClient {
3735
3801
  if (encryptedEmbeds.length > 0) {
3736
3802
  messagePayload.encrypted_embeds = encryptedEmbeds;
3737
3803
  }
3804
+ const precollectedResponse = params.precollectResponse ? ws.collectAiResponse(messageId, chatId, { onStream: params.onStream }) : null;
3738
3805
  const confirmed = ws.waitForMessage(
3739
3806
  "chat_message_confirmed",
3740
3807
  (payload) => {
@@ -3949,7 +4016,7 @@ var OpenMatesClient = class _OpenMatesClient {
3949
4016
  };
3950
4017
  if (params.incognito) {
3951
4018
  try {
3952
- const resp = await ws.collectAiResponse(messageId, chatId, streamOpts);
4019
+ const resp = await (precollectedResponse ?? ws.collectAiResponse(messageId, chatId, streamOpts));
3953
4020
  assistantMessageId = resp.messageId;
3954
4021
  assistant = resp.content;
3955
4022
  category = resp.category;
@@ -4301,7 +4368,7 @@ var OpenMatesClient = class _OpenMatesClient {
4301
4368
  if (response.data.status === "failed") {
4302
4369
  throw new Error(response.data.error ?? "Task failed");
4303
4370
  }
4304
- await new Promise((resolve5) => setTimeout(resolve5, SKILL_TASK_POLL_INTERVAL_MS));
4371
+ await new Promise((resolve6) => setTimeout(resolve6, SKILL_TASK_POLL_INTERVAL_MS));
4305
4372
  }
4306
4373
  throw new Error(`Task ${taskId} did not complete within ${SKILL_TASK_POLL_TIMEOUT_MS / 1e3}s`);
4307
4374
  }
@@ -4522,7 +4589,7 @@ var OpenMatesClient = class _OpenMatesClient {
4522
4589
  `Rate limited by settings API; retrying in ${Math.ceil(SETTINGS_GET_RATE_LIMIT_RETRY_MS / 1e3)}s...
4523
4590
  `
4524
4591
  );
4525
- await new Promise((resolve5) => setTimeout(resolve5, SETTINGS_GET_RATE_LIMIT_RETRY_MS));
4592
+ await new Promise((resolve6) => setTimeout(resolve6, SETTINGS_GET_RATE_LIMIT_RETRY_MS));
4526
4593
  response = await this.http.get(normalizedPath, this.getCliRequestHeaders());
4527
4594
  }
4528
4595
  if (!response.ok) {
@@ -6023,7 +6090,7 @@ function filenameFromContentDisposition(header2) {
6023
6090
  return plain?.trim() ?? null;
6024
6091
  }
6025
6092
  function sleep(ms) {
6026
- return new Promise((resolve5) => setTimeout(resolve5, ms));
6093
+ return new Promise((resolve6) => setTimeout(resolve6, ms));
6027
6094
  }
6028
6095
  function printLogo() {
6029
6096
  const W = "\x1B[1;37m";
@@ -6039,9 +6106,9 @@ function printLogo() {
6039
6106
 
6040
6107
  // src/cli.ts
6041
6108
  import { createInterface as createInterface3 } from "readline/promises";
6042
- import { realpathSync, writeFileSync as writeFileSync4 } from "fs";
6043
- import { fileURLToPath } from "url";
6044
- import { basename as basename3, dirname } from "path";
6109
+ import { realpathSync, writeFileSync as writeFileSync5 } from "fs";
6110
+ import { fileURLToPath as fileURLToPath2 } from "url";
6111
+ import { basename as basename3, dirname as dirname3 } from "path";
6045
6112
  import WebSocket2 from "ws";
6046
6113
 
6047
6114
  // ../secret-scanner/src/registry.ts
@@ -7741,8 +7808,8 @@ async function renderRemotionShareLink(embedId, client, ln) {
7741
7808
  }
7742
7809
  }
7743
7810
  function generateQr(value) {
7744
- return new Promise((resolve5) => {
7745
- qrcode2.generate(value, { small: true }, (qr) => resolve5(qr));
7811
+ return new Promise((resolve6) => {
7812
+ qrcode2.generate(value, { small: true }, (qr) => resolve6(qr));
7746
7813
  });
7747
7814
  }
7748
7815
  function remotionMeta(c) {
@@ -8510,8 +8577,9 @@ import { execSync, spawn as nodeSpawn } from "child_process";
8510
8577
  import { randomBytes as randomBytes2 } from "crypto";
8511
8578
  import { copyFileSync, existsSync as existsSync5, mkdirSync as mkdirSync3, readFileSync as readFileSync5, rmSync as rmSync3, writeFileSync as writeFileSync3 } from "fs";
8512
8579
  import { createInterface as createInterface2 } from "readline";
8580
+ import { createInterface as createPromptInterface } from "readline/promises";
8513
8581
  import { homedir as homedir5 } from "os";
8514
- import { join as join3, resolve as resolve3 } from "path";
8582
+ import { dirname, join as join3, resolve as resolve3 } from "path";
8515
8583
  var SOURCE_COMPOSE_FILE = join3("backend", "core", "docker-compose.yml");
8516
8584
  var IMAGE_COMPOSE_FILE = join3("backend", "core", "docker-compose.selfhost.yml");
8517
8585
  var COMPOSE_OVERRIDE = join3("backend", "core", "docker-compose.override.yml");
@@ -8528,6 +8596,43 @@ var IMAGE_CHANNEL_TAGS = {
8528
8596
  main: MAIN_BRANCH,
8529
8597
  dev: DEV_BRANCH
8530
8598
  };
8599
+ var BACKEND_CONFIG_FILE = join3("backend", "config", "backend_config.yml");
8600
+ var IMAGE_RUNTIME_CONFIG_FILE = join3("config", "backend_config.yml");
8601
+ var LOCAL_AI_MODELS_FILE = "local-ai-models.yml";
8602
+ var OFF_BY_DEFAULT_FEATURES = /* @__PURE__ */ new Map([
8603
+ ["embed:code:application", "Application previews are still unstable"],
8604
+ ["platform:projects", "Projects workspace is not ready by default"],
8605
+ ["platform:workflows", "Workflows workspace is not implemented yet"],
8606
+ ["platform:tasks", "Tasks workspace is not implemented yet"]
8607
+ ]);
8608
+ var LOCAL_MODEL_RUNTIME_DEFAULTS = {
8609
+ ollama: {
8610
+ label: "Ollama",
8611
+ serverId: "ollama",
8612
+ baseUrl: "http://host.docker.internal:11434/v1",
8613
+ apiKey: "ollama"
8614
+ },
8615
+ "lm-studio": {
8616
+ label: "LM Studio",
8617
+ serverId: "lm_studio",
8618
+ baseUrl: "http://host.docker.internal:1234/v1",
8619
+ apiKey: "lm-studio"
8620
+ },
8621
+ custom: {
8622
+ label: "Custom OpenAI-compatible API",
8623
+ serverId: "custom_openai_compatible",
8624
+ baseUrl: "",
8625
+ apiKey: "local"
8626
+ }
8627
+ };
8628
+ var MODEL_CREATOR_OPTIONS = [
8629
+ { id: "alibaba", name: "Alibaba / Qwen", match: /(^|[-_:/])qwen/i },
8630
+ { id: "google", name: "Google / Gemma", match: /(^|[-_:/])gemma/i },
8631
+ { id: "mistral", name: "Mistral", match: /(^|[-_:/])(mistral|mixtral|ministral)/i },
8632
+ { id: "openai", name: "OpenAI", match: /(^|[-_:/])gpt-oss/i },
8633
+ { id: "deepseek", name: "DeepSeek", match: /(^|[-_:/])deepseek/i },
8634
+ { id: "custom", name: "Custom", match: null }
8635
+ ];
8531
8636
  var MINIMAL_ENV_TEMPLATE = `# OpenMates self-host image-mode environment
8532
8637
  SECRET__MISTRAL_AI__API_KEY=
8533
8638
  SECRET__CEREBRAS__API_KEY=
@@ -8597,9 +8702,9 @@ function exec(cmd, cwd) {
8597
8702
  return execSync(cmd, { cwd, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
8598
8703
  }
8599
8704
  function runInteractive(cmd, args, cwd) {
8600
- return new Promise((resolve5, reject) => {
8705
+ return new Promise((resolve6, reject) => {
8601
8706
  const child = nodeSpawn(cmd, args, { cwd, stdio: "inherit", shell: false });
8602
- child.on("close", (code) => resolve5(code ?? 1));
8707
+ child.on("close", (code) => resolve6(code ?? 1));
8603
8708
  child.on("error", reject);
8604
8709
  });
8605
8710
  }
@@ -8615,6 +8720,65 @@ function getInstallMode(installPath, config = loadConfigForInstallPath(installPa
8615
8720
  function shouldPullImages() {
8616
8721
  return process.env.OPENMATES_SKIP_IMAGE_PULL !== "1";
8617
8722
  }
8723
+ function normalizeFeatureList(items) {
8724
+ const seen = /* @__PURE__ */ new Set();
8725
+ const normalized = [];
8726
+ for (const item of items) {
8727
+ const value = item.trim();
8728
+ if (!value || seen.has(value)) continue;
8729
+ seen.add(value);
8730
+ normalized.push(value);
8731
+ }
8732
+ return normalized;
8733
+ }
8734
+ function parseListBlock(content, key) {
8735
+ const match = content.match(new RegExp(`^${key}:\\n((?:[ \\t]+.*\\n?)*)`, "m"));
8736
+ if (!match) return [];
8737
+ const block = match[1] ?? "";
8738
+ return normalizeFeatureList(
8739
+ [...block.matchAll(/^\s*-\s*["']?([^"'\n#]+)["']?/gm)].map((item) => item[1] ?? "")
8740
+ );
8741
+ }
8742
+ function parseFeatureOverrides(content) {
8743
+ const overridesMatch = content.match(/^feature_overrides:\n((?:[ \t]+.*\n?)*)/m);
8744
+ const overridesBlock = overridesMatch?.[1] ?? "";
8745
+ const enabled = parseListBlock(overridesBlock.replace(/^ {2}/gm, ""), "enabled");
8746
+ const disabled = parseListBlock(overridesBlock.replace(/^ {2}/gm, ""), "disabled");
8747
+ const legacyDisabledApps = parseListBlock(content, "disabled_apps").map(
8748
+ (appId) => appId.startsWith("app:") ? appId : `app:${appId}`
8749
+ );
8750
+ return {
8751
+ enabled: normalizeFeatureList(enabled),
8752
+ disabled: normalizeFeatureList([...disabled, ...legacyDisabledApps])
8753
+ };
8754
+ }
8755
+ function renderFeatureOverrides(overrides) {
8756
+ const renderList = (key, items) => {
8757
+ if (!items.length) return ` ${key}: []`;
8758
+ return [` ${key}:`, ...items.map((item) => ` - "${item}"`)].join("\n");
8759
+ };
8760
+ return [
8761
+ "# Admin feature overrides. Changes require a server restart.",
8762
+ "feature_overrides:",
8763
+ renderList("enabled", overrides.enabled),
8764
+ renderList("disabled", overrides.disabled),
8765
+ ""
8766
+ ].join("\n");
8767
+ }
8768
+ function removeConfigBlock(content, key) {
8769
+ return content.replace(new RegExp(`(?:^|\\n)#.*\\n${key}:\\n(?:[ \\t]+.*\\n?)*`, "m"), "\n").replace(new RegExp(`^${key}:\\n(?:[ \\t]+.*\\n?)*`, "m"), "");
8770
+ }
8771
+ function updateFeatureOverridesContent(content, overrides) {
8772
+ let next = removeConfigBlock(content, "feature_overrides");
8773
+ next = removeConfigBlock(next, "disabled_apps");
8774
+ next = next.trimEnd();
8775
+ return `${next}
8776
+
8777
+ ${renderFeatureOverrides(overrides)}`;
8778
+ }
8779
+ function featureKind(featureId) {
8780
+ return featureId.split(":", 1)[0] || "unknown";
8781
+ }
8618
8782
  function composeArgs(installPath, withOverrides, installMode = getInstallMode(installPath)) {
8619
8783
  const composeFile = installMode === "image" ? IMAGE_COMPOSE_FILE : SOURCE_COMPOSE_FILE;
8620
8784
  const args = ["compose", "--env-file", ".env", "-f", composeFile];
@@ -8765,8 +8929,10 @@ async function writeImageModeRuntimeFiles(installPath, imageTag) {
8765
8929
  const coreDir = join3(installPath, "backend", "core");
8766
8930
  const vaultConfigDir = join3(coreDir, "vault", "config");
8767
8931
  mkdirSync3(vaultConfigDir, { recursive: true });
8932
+ mkdirSync3(join3(installPath, "config", "providers"), { recursive: true });
8768
8933
  writeFileSync3(join3(coreDir, "docker-compose.selfhost.yml"), await loadSelfHostComposeTemplate(templateRefForImageTag(imageTag, getPackageVersion())));
8769
8934
  writeFileSync3(join3(vaultConfigDir, "vault.hcl"), VAULT_CONFIG_TEMPLATE);
8935
+ ensureImageRuntimeConfig(installPath);
8770
8936
  const envPath = join3(installPath, ".env");
8771
8937
  let envContent = existsSync5(envPath) ? readFileSync5(envPath, "utf-8") : MINIMAL_ENV_TEMPLATE;
8772
8938
  envContent = setEnvIfEmpty(envContent, "DATABASE_ADMIN_PASSWORD", randomHex(12));
@@ -8830,6 +8996,7 @@ function defaultCloneBranchForVersion(version) {
8830
8996
  }
8831
8997
  function hasLlmCredentials(envPath) {
8832
8998
  if (!existsSync5(envPath)) return false;
8999
+ if (hasLocalAiModels(dirname(envPath))) return true;
8833
9000
  const content = readFileSync5(envPath, "utf-8");
8834
9001
  for (const line of content.split("\n")) {
8835
9002
  const trimmed = line.trim();
@@ -8854,22 +9021,180 @@ function warnIfMissingLlmCredentials(installPath) {
8854
9021
  }
8855
9022
  if (!hasLlmCredentials(envPath)) {
8856
9023
  console.error(
8857
- "No LLM provider API key found in .env.\nOpenMates will start, but AI chat/model processing will stay unavailable until you add one.\n\nAdd at least one of these to your .env file:\n SECRET__OPENAI__API_KEY=sk-...\n SECRET__ANTHROPIC__API_KEY=sk-ant-...\n SECRET__GOOGLE_AI_STUDIO__API_KEY=...\n\nAfter updating .env, run 'openmates server restart'."
9024
+ "No LLM provider API key found in .env.\nOpenMates will start, but AI chat/model processing will stay unavailable until you add one.\n\nRun 'openmates server ai models add' to add a local Ollama/LM Studio model, or add at least one of these to your .env file:\n SECRET__OPENAI__API_KEY=sk-...\n SECRET__ANTHROPIC__API_KEY=sk-ant-...\n SECRET__GOOGLE_AI_STUDIO__API_KEY=...\n\nAfter updating .env, run 'openmates server restart'."
8858
9025
  );
8859
9026
  }
8860
9027
  }
8861
9028
  async function confirmDestructive(phrase) {
8862
9029
  const rl = createInterface2({ input: process.stdin, output: process.stderr });
8863
- return new Promise((resolve5) => {
9030
+ return new Promise((resolve6) => {
8864
9031
  rl.question(`Type "${phrase}" to confirm: `, (answer) => {
8865
9032
  rl.close();
8866
- resolve5(answer.trim() === phrase);
9033
+ resolve6(answer.trim() === phrase);
8867
9034
  });
8868
9035
  });
8869
9036
  }
8870
9037
  function printJson(data) {
8871
9038
  console.log(JSON.stringify(data, null, 2));
8872
9039
  }
9040
+ function localModelsOverlayPath(installPath, installMode = getInstallMode(installPath)) {
9041
+ if (installMode === "source") return join3(installPath, "backend", "providers", LOCAL_AI_MODELS_FILE);
9042
+ return join3(installPath, "config", "providers", LOCAL_AI_MODELS_FILE);
9043
+ }
9044
+ function imageBackendConfigPath(installPath) {
9045
+ return join3(installPath, IMAGE_RUNTIME_CONFIG_FILE);
9046
+ }
9047
+ function ensureImageRuntimeConfig(installPath) {
9048
+ const configPath = imageBackendConfigPath(installPath);
9049
+ if (existsSync5(configPath)) return;
9050
+ mkdirSync3(dirname(configPath), { recursive: true });
9051
+ writeFileSync3(configPath, renderFeatureOverrides({ enabled: [], disabled: [] }));
9052
+ }
9053
+ function readLocalModelOverlay(path) {
9054
+ if (!existsSync5(path)) return { providers: [] };
9055
+ try {
9056
+ const parsed = JSON.parse(readFileSync5(path, "utf-8"));
9057
+ return { providers: Array.isArray(parsed.providers) ? parsed.providers : [] };
9058
+ } catch (error) {
9059
+ throw new Error(
9060
+ `Could not parse ${path}. This file is managed by the OpenMates CLI and must remain JSON-compatible YAML. ${error instanceof Error ? error.message : String(error)}`
9061
+ );
9062
+ }
9063
+ }
9064
+ function writeLocalModelOverlay(path, overlay) {
9065
+ mkdirSync3(dirname(path), { recursive: true });
9066
+ writeFileSync3(path, `${JSON.stringify(overlay, null, 2)}
9067
+ `);
9068
+ }
9069
+ function hasLocalAiModels(installPath) {
9070
+ const imagePath = join3(installPath, "config", "providers", LOCAL_AI_MODELS_FILE);
9071
+ const sourcePath = join3(installPath, "backend", "providers", LOCAL_AI_MODELS_FILE);
9072
+ return [imagePath, sourcePath].some((path) => {
9073
+ if (!existsSync5(path)) return false;
9074
+ try {
9075
+ return readLocalModelOverlay(path).providers.some((provider) => provider.models.length > 0);
9076
+ } catch {
9077
+ return false;
9078
+ }
9079
+ });
9080
+ }
9081
+ function sanitizeModelId(raw) {
9082
+ return raw.trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "local-model";
9083
+ }
9084
+ function inferCreatorId(modelId) {
9085
+ return MODEL_CREATOR_OPTIONS.find((option) => option.match?.test(modelId))?.id ?? "custom";
9086
+ }
9087
+ function creatorDisplayName(creatorId) {
9088
+ return MODEL_CREATOR_OPTIONS.find((option) => option.id === creatorId)?.name ?? creatorId;
9089
+ }
9090
+ function normalizeCreatorId(value) {
9091
+ const normalized = value.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "_").replace(/^_+|_+$/g, "");
9092
+ return normalized || "custom_local";
9093
+ }
9094
+ function normalizeRuntimeKey(value) {
9095
+ const normalized = value.trim().toLowerCase().replace(/_/g, "-");
9096
+ if (normalized === "lmstudio" || normalized === "lm-studio") return "lm-studio";
9097
+ if (normalized === "ollama") return "ollama";
9098
+ return "custom";
9099
+ }
9100
+ function boolFromFlag(value, defaultValue = false) {
9101
+ if (value === void 0) return defaultValue;
9102
+ if (value === true) return true;
9103
+ if (value === false) return false;
9104
+ const normalized = value.toLowerCase();
9105
+ return ["1", "true", "yes", "y", "on"].includes(normalized);
9106
+ }
9107
+ async function promptText(question, defaultValue = "") {
9108
+ const rl = createPromptInterface({ input: process.stdin, output: process.stderr });
9109
+ try {
9110
+ const suffix = defaultValue ? ` (${defaultValue})` : "";
9111
+ const answer = await rl.question(`${question}${suffix}: `);
9112
+ return answer.trim() || defaultValue;
9113
+ } finally {
9114
+ rl.close();
9115
+ }
9116
+ }
9117
+ async function promptChoice(question, choices, defaultValue) {
9118
+ console.error(question);
9119
+ choices.forEach((choice, index) => {
9120
+ const marker = choice.value === defaultValue ? " [default]" : "";
9121
+ console.error(` ${index + 1}. ${choice.label}${marker}`);
9122
+ });
9123
+ const answer = await promptText("Choose number or value", defaultValue);
9124
+ const numeric = Number.parseInt(answer, 10);
9125
+ if (Number.isInteger(numeric) && numeric >= 1 && numeric <= choices.length) {
9126
+ return choices[numeric - 1].value;
9127
+ }
9128
+ const direct = choices.find((choice) => choice.value === answer || choice.label.toLowerCase() === answer.toLowerCase());
9129
+ return direct?.value ?? answer;
9130
+ }
9131
+ async function fetchLocalModelIds(baseUrl, apiKey) {
9132
+ const response = await fetch(`${baseUrl.replace(/\/+$/, "")}/models`, {
9133
+ headers: { Authorization: `Bearer ${apiKey || "local"}` }
9134
+ });
9135
+ if (!response.ok) throw new Error(`Failed to fetch local models: HTTP ${response.status}`);
9136
+ const body = await response.json();
9137
+ return (body.data ?? []).map((model) => model.id).filter((id) => Boolean(id));
9138
+ }
9139
+ async function testLocalModel(baseUrl, apiKey, modelId) {
9140
+ const controller = new AbortController();
9141
+ const timeout = setTimeout(() => controller.abort(), 3e4);
9142
+ try {
9143
+ const response = await fetch(`${baseUrl.replace(/\/+$/, "")}/chat/completions`, {
9144
+ method: "POST",
9145
+ signal: controller.signal,
9146
+ headers: {
9147
+ Authorization: `Bearer ${apiKey || "local"}`,
9148
+ "Content-Type": "application/json"
9149
+ },
9150
+ body: JSON.stringify({
9151
+ model: modelId,
9152
+ messages: [
9153
+ { role: "system", content: "Answer with only the number." },
9154
+ { role: "user", content: "1+2?" }
9155
+ ],
9156
+ stream: false,
9157
+ temperature: 0
9158
+ })
9159
+ });
9160
+ if (!response.ok) throw new Error(`HTTP ${response.status}: ${await response.text()}`);
9161
+ const body = await response.json();
9162
+ return body.choices?.[0]?.message?.content?.trim() ?? "";
9163
+ } finally {
9164
+ clearTimeout(timeout);
9165
+ }
9166
+ }
9167
+ function upsertLocalModel(path, providerId, providerName, model) {
9168
+ const overlay = readLocalModelOverlay(path);
9169
+ let provider = overlay.providers.find((item) => item.provider_id === providerId);
9170
+ if (!provider) {
9171
+ provider = {
9172
+ provider_id: providerId,
9173
+ name: providerName,
9174
+ description: `Local self-hosted models for ${providerName}.`,
9175
+ models: []
9176
+ };
9177
+ overlay.providers.push(provider);
9178
+ }
9179
+ const modelId = model.id;
9180
+ provider.models = provider.models.filter((existing) => existing.id !== modelId);
9181
+ provider.models.push(model);
9182
+ writeLocalModelOverlay(path, overlay);
9183
+ }
9184
+ function removeLocalModel(path, fullModelId) {
9185
+ const [providerId, modelId] = fullModelId.split("/", 2);
9186
+ if (!providerId || !modelId) throw new Error("Use provider/model-id format.");
9187
+ const overlay = readLocalModelOverlay(path);
9188
+ const provider = overlay.providers.find((item) => item.provider_id === providerId);
9189
+ if (!provider) return false;
9190
+ const before = provider.models.length;
9191
+ provider.models = provider.models.filter((model) => model.id !== modelId);
9192
+ writeLocalModelOverlay(path, overlay);
9193
+ return provider.models.length !== before;
9194
+ }
9195
+ function localModelsFromOverlay(overlay) {
9196
+ return overlay.providers.flatMap((provider) => provider.models.map((model) => ({ providerId: provider.provider_id, model })));
9197
+ }
8873
9198
  async function serverStatus(flags) {
8874
9199
  requireDocker();
8875
9200
  const installPath = resolveServerPath(flags);
@@ -9324,6 +9649,245 @@ async function serverMakeAdmin(rest, flags) {
9324
9649
  console.log(`Admin privileges granted to ${email}.`);
9325
9650
  }
9326
9651
  }
9652
+ async function serverFeatures(rest, flags) {
9653
+ const action = rest[0] ?? "list";
9654
+ const featureId = rest[1];
9655
+ const installPath = resolveServerPath(flags);
9656
+ const installMode = getInstallMode(installPath);
9657
+ if (installMode === "image") ensureImageRuntimeConfig(installPath);
9658
+ const configPath = installMode === "image" ? imageBackendConfigPath(installPath) : join3(installPath, BACKEND_CONFIG_FILE);
9659
+ if (!existsSync5(configPath)) {
9660
+ throw new Error(`Backend config not found at ${configPath}. Run 'openmates server install' first or pass --path <dir>.`);
9661
+ }
9662
+ const content = readFileSync5(configPath, "utf-8");
9663
+ const overrides = parseFeatureOverrides(content);
9664
+ const writeOverrides = (nextOverrides) => {
9665
+ writeFileSync3(configPath, updateFeatureOverridesContent(content, nextOverrides));
9666
+ console.log(`Updated ${configPath}`);
9667
+ console.log("Restart the server for feature changes to take effect: openmates server restart");
9668
+ };
9669
+ if (action === "list") {
9670
+ console.log("Feature overrides:");
9671
+ console.log(` enabled: ${overrides.enabled.length ? overrides.enabled.join(", ") : "none"}`);
9672
+ console.log(` disabled: ${overrides.disabled.length ? overrides.disabled.join(", ") : "none"}`);
9673
+ console.log("\nKnown off-by-default features:");
9674
+ for (const [id, reason] of OFF_BY_DEFAULT_FEATURES.entries()) {
9675
+ const override = overrides.enabled.includes(id) ? "enabled override" : overrides.disabled.includes(id) ? "disabled override" : "default off";
9676
+ console.log(` ${id} (${featureKind(id)}): ${override} - ${reason}`);
9677
+ }
9678
+ return;
9679
+ }
9680
+ if (!featureId) {
9681
+ throw new Error(`Usage: openmates server features ${action} <feature-id>`);
9682
+ }
9683
+ if (action === "enable") {
9684
+ writeOverrides({
9685
+ enabled: normalizeFeatureList([...overrides.enabled, featureId]),
9686
+ disabled: overrides.disabled.filter((id) => id !== featureId)
9687
+ });
9688
+ return;
9689
+ }
9690
+ if (action === "disable") {
9691
+ writeOverrides({
9692
+ enabled: overrides.enabled.filter((id) => id !== featureId),
9693
+ disabled: normalizeFeatureList([...overrides.disabled, featureId])
9694
+ });
9695
+ return;
9696
+ }
9697
+ if (action === "reset") {
9698
+ writeOverrides({
9699
+ enabled: overrides.enabled.filter((id) => id !== featureId),
9700
+ disabled: overrides.disabled.filter((id) => id !== featureId)
9701
+ });
9702
+ return;
9703
+ }
9704
+ if (action === "explain") {
9705
+ const defaultReason = OFF_BY_DEFAULT_FEATURES.get(featureId);
9706
+ const override = overrides.enabled.includes(featureId) ? "enabled" : overrides.disabled.includes(featureId) ? "disabled" : "none";
9707
+ const defaultState = defaultReason ? "off" : "on";
9708
+ const effective = override === "enabled" ? "enabled" : override === "disabled" ? "disabled" : defaultState === "on" ? "enabled" : "disabled";
9709
+ console.log(`Feature: ${featureId}`);
9710
+ console.log(`Kind: ${featureKind(featureId)}`);
9711
+ console.log(`Default: ${defaultState}${defaultReason ? ` (${defaultReason})` : ""}`);
9712
+ console.log(`Override: ${override}`);
9713
+ console.log(`Effective after restart: ${effective}`);
9714
+ return;
9715
+ }
9716
+ throw new Error(`Unknown server features command '${action}'. Use list, enable, disable, reset, or explain.`);
9717
+ }
9718
+ async function serverAiModelsAdd(flags) {
9719
+ const installPath = resolveServerPath(flags);
9720
+ const installMode = getInstallMode(installPath);
9721
+ const overlayPath = localModelsOverlayPath(installPath, installMode);
9722
+ const runtimeInput = typeof flags.runtime === "string" ? flags.runtime : await promptChoice(
9723
+ "Which local runtime should OpenMates use?",
9724
+ [
9725
+ { value: "ollama", label: "Ollama" },
9726
+ { value: "lm-studio", label: "LM Studio" },
9727
+ { value: "custom", label: "Custom OpenAI-compatible API" }
9728
+ ],
9729
+ "ollama"
9730
+ );
9731
+ const runtimeKey = normalizeRuntimeKey(runtimeInput);
9732
+ const runtime = LOCAL_MODEL_RUNTIME_DEFAULTS[runtimeKey];
9733
+ const baseUrl = typeof flags["base-url"] === "string" ? flags["base-url"] : await promptText("OpenAI-compatible base URL", runtime.baseUrl);
9734
+ const apiKey = typeof flags["api-key"] === "string" ? flags["api-key"] : runtimeKey === "custom" ? await promptText("API key", runtime.apiKey) : runtime.apiKey;
9735
+ let availableModels = [];
9736
+ try {
9737
+ availableModels = await fetchLocalModelIds(baseUrl, apiKey);
9738
+ } catch (error) {
9739
+ console.error(`Could not fetch /v1/models: ${error instanceof Error ? error.message : String(error)}`);
9740
+ }
9741
+ const rawModelId = typeof flags.model === "string" ? flags.model : availableModels.length ? await promptChoice(
9742
+ "Which installed model should OpenMates add?",
9743
+ [...availableModels.map((id) => ({ value: id, label: id })), { value: "manual", label: "Enter manually" }],
9744
+ availableModels[0]
9745
+ ) : await promptText("Local model ID");
9746
+ const serverModelId = rawModelId === "manual" ? await promptText("Local model ID") : rawModelId;
9747
+ if (!serverModelId) throw new Error("Model ID is required.");
9748
+ const inferredCreator = inferCreatorId(serverModelId);
9749
+ const creatorInput = typeof flags.creator === "string" ? flags.creator : await promptChoice(
9750
+ "Who created the model?",
9751
+ MODEL_CREATOR_OPTIONS.map((option) => ({ value: option.id, label: option.name })),
9752
+ inferredCreator
9753
+ );
9754
+ const providerId = creatorInput === "custom" ? normalizeCreatorId(await promptText("Custom creator/provider ID", "custom_local")) : normalizeCreatorId(creatorInput);
9755
+ const providerName = providerId === creatorInput ? creatorDisplayName(providerId) : providerId;
9756
+ const internalModelId = typeof flags.id === "string" ? sanitizeModelId(flags.id) : `${sanitizeModelId(serverModelId)}-local`;
9757
+ const displayName = typeof flags.name === "string" ? flags.name : await promptText("Display name", serverModelId);
9758
+ const supportsImages = boolFromFlag(flags.images ?? flags.image ?? flags.vision, false);
9759
+ const supportsTools = boolFromFlag(flags.tools ?? flags["tool-use"], false);
9760
+ const contextWindowInput = typeof flags["context-window"] === "string" ? flags["context-window"] : await promptText("Context window tokens", "32768");
9761
+ const contextWindow = Number.parseInt(contextWindowInput, 10) || 32768;
9762
+ if (flags["skip-test"] !== true) {
9763
+ console.error(`Testing ${runtime.label} model '${serverModelId}'...`);
9764
+ const testOutput = await testLocalModel(baseUrl, apiKey, serverModelId);
9765
+ if (!testOutput) throw new Error("Local model test returned no content. Not saving model.");
9766
+ console.error(`Test response: ${testOutput}`);
9767
+ }
9768
+ const model = {
9769
+ id: internalModelId,
9770
+ name: displayName,
9771
+ description: `${displayName} served locally through ${runtime.label}.`,
9772
+ country_origin: "local",
9773
+ for_app_skill: "ai.ask",
9774
+ allow_auto_select: false,
9775
+ local: true,
9776
+ self_hosted: true,
9777
+ input_types: supportsImages ? ["text", "image"] : ["text"],
9778
+ output_types: ["text"],
9779
+ default_server: runtime.serverId,
9780
+ servers: [
9781
+ {
9782
+ id: runtime.serverId,
9783
+ name: runtime.label,
9784
+ model_id: serverModelId,
9785
+ region: "local",
9786
+ base_url: baseUrl.replace(/\/+$/, ""),
9787
+ api_key: apiKey,
9788
+ supports_tools: supportsTools
9789
+ }
9790
+ ],
9791
+ pricing: { fixed: { credits: 0 } },
9792
+ costs: {
9793
+ input_per_million_token: { price: 0, currency: "USD", max_context: contextWindow },
9794
+ output_per_million_token: { price: 0, currency: "USD", max_context: contextWindow }
9795
+ },
9796
+ features: {
9797
+ streaming: true,
9798
+ tool_use: supportsTools,
9799
+ max_context: contextWindow
9800
+ }
9801
+ };
9802
+ upsertLocalModel(overlayPath, providerId, providerName, model);
9803
+ if (installMode === "image") ensureImageRuntimeConfig(installPath);
9804
+ if (flags.json === true) {
9805
+ printJson({ command: "server ai models add", status: "success", model: `${providerId}/${internalModelId}`, overlayPath });
9806
+ } else {
9807
+ console.log(`Added local model: ${providerId}/${internalModelId}`);
9808
+ console.log(`Updated ${overlayPath}`);
9809
+ console.log("Restart the server for model changes to take effect: openmates server restart");
9810
+ console.log("Self-hosted local models charge 0 credits; token usage may still be recorded in usage history.");
9811
+ }
9812
+ }
9813
+ async function serverAiModelsList(flags) {
9814
+ const installPath = resolveServerPath(flags);
9815
+ const overlayPath = localModelsOverlayPath(installPath);
9816
+ const overlay = readLocalModelOverlay(overlayPath);
9817
+ const models = localModelsFromOverlay(overlay).map(({ providerId, model }) => ({
9818
+ id: `${providerId}/${String(model.id ?? "")}`,
9819
+ name: model.name ?? model.id,
9820
+ server: Array.isArray(model.servers) ? model.servers[0]?.id : void 0,
9821
+ serverModelId: Array.isArray(model.servers) ? model.servers[0]?.model_id : void 0
9822
+ }));
9823
+ if (flags.json === true) {
9824
+ printJson({ overlayPath, models });
9825
+ return;
9826
+ }
9827
+ if (!models.length) {
9828
+ console.log("No local AI models configured. Add one with: openmates server ai models add");
9829
+ return;
9830
+ }
9831
+ console.log("Local AI models:");
9832
+ for (const model of models) {
9833
+ console.log(` ${model.id} (${model.server}: ${model.serverModelId})`);
9834
+ }
9835
+ }
9836
+ async function serverAiModelsTest(rest, flags) {
9837
+ const installPath = resolveServerPath(flags);
9838
+ const overlayPath = localModelsOverlayPath(installPath);
9839
+ const fullModelId = rest[0];
9840
+ if (!fullModelId || !fullModelId.includes("/")) throw new Error("Usage: openmates server ai models test <provider/model-id>");
9841
+ const [providerId, modelId] = fullModelId.split("/", 2);
9842
+ const overlay = readLocalModelOverlay(overlayPath);
9843
+ const provider = overlay.providers.find((item) => item.provider_id === providerId);
9844
+ const model = provider?.models.find((item) => item.id === modelId);
9845
+ const server = Array.isArray(model?.servers) ? model.servers[0] : void 0;
9846
+ if (!server) throw new Error(`Local model not found in ${overlayPath}: ${fullModelId}`);
9847
+ const baseUrl = String(server.base_url ?? "");
9848
+ const apiKey = String(server.api_key ?? "local");
9849
+ const serverModelId = String(server.model_id ?? "");
9850
+ const output = await testLocalModel(baseUrl, apiKey, serverModelId);
9851
+ if (flags.json === true) {
9852
+ printJson({ model: fullModelId, status: "success", output });
9853
+ } else {
9854
+ console.log(`Model test succeeded: ${fullModelId}`);
9855
+ console.log(`Response: ${output}`);
9856
+ }
9857
+ }
9858
+ async function serverAiModelsRemove(rest, flags) {
9859
+ const installPath = resolveServerPath(flags);
9860
+ const overlayPath = localModelsOverlayPath(installPath);
9861
+ const fullModelId = rest[0];
9862
+ if (!fullModelId) throw new Error("Usage: openmates server ai models remove <provider/model-id>");
9863
+ const removed = removeLocalModel(overlayPath, fullModelId);
9864
+ if (flags.json === true) {
9865
+ printJson({ command: "server ai models remove", status: removed ? "success" : "not_found", model: fullModelId });
9866
+ } else if (removed) {
9867
+ console.log(`Removed local model: ${fullModelId}`);
9868
+ console.log("Restart the server for model changes to take effect: openmates server restart");
9869
+ } else {
9870
+ console.log(`Local model not found: ${fullModelId}`);
9871
+ }
9872
+ }
9873
+ async function serverAi(rest, flags) {
9874
+ const area = rest[0];
9875
+ const action = rest[1] ?? "list";
9876
+ const args = rest.slice(2);
9877
+ if (area !== "models") throw new Error("Usage: openmates server ai models <add|list|test|remove>");
9878
+ switch (action) {
9879
+ case "add":
9880
+ return serverAiModelsAdd(flags);
9881
+ case "list":
9882
+ return serverAiModelsList(flags);
9883
+ case "test":
9884
+ return serverAiModelsTest(args, flags);
9885
+ case "remove":
9886
+ return serverAiModelsRemove(args, flags);
9887
+ default:
9888
+ throw new Error(`Unknown server ai models command '${action}'. Use add, list, test, or remove.`);
9889
+ }
9890
+ }
9327
9891
  async function serverUninstall(flags) {
9328
9892
  requireDocker();
9329
9893
  const installPath = resolveServerPath(flags);
@@ -9391,6 +9955,7 @@ Commands:
9391
9955
  logs Display server logs
9392
9956
  update Update to latest version (pull images, or git pull + rebuild for source installs)
9393
9957
  make-admin Grant admin privileges to a user
9958
+ ai Manage self-hosted local AI models
9394
9959
  reset Reset server data (requires confirmation)
9395
9960
  uninstall Completely remove OpenMates (requires confirmation)
9396
9961
 
@@ -9435,11 +10000,27 @@ Command Options:
9435
10000
  make-admin:
9436
10001
  openmates server make-admin <email>
9437
10002
 
10003
+ features:
10004
+ openmates server features list
10005
+ openmates server features enable <feature-id>
10006
+ openmates server features disable <feature-id>
10007
+ openmates server features reset <feature-id>
10008
+ openmates server features explain <feature-id>
10009
+
10010
+ ai models:
10011
+ openmates server ai models add
10012
+ openmates server ai models list
10013
+ openmates server ai models test <provider/model-id>
10014
+ openmates server ai models remove <provider/model-id>
10015
+
9438
10016
  Examples:
9439
10017
  openmates server install
9440
10018
  openmates server start --with-overrides
9441
10019
  openmates server logs --container api --follow
9442
10020
  openmates server make-admin user@example.com
10021
+ openmates server features disable app:videos
10022
+ openmates server ai models add
10023
+ openmates server features enable embed:code:application
9443
10024
  openmates server update
9444
10025
  openmates server update --dry-run
9445
10026
  openmates server update --image-tag v0.12.0-alpha.1
@@ -9471,6 +10052,10 @@ async function handleServer(subcommand, rest, flags) {
9471
10052
  return serverReset(flags);
9472
10053
  case "make-admin":
9473
10054
  return serverMakeAdmin(rest, flags);
10055
+ case "ai":
10056
+ return serverAi(rest, flags);
10057
+ case "features":
10058
+ return serverFeatures(rest, flags);
9474
10059
  case "uninstall":
9475
10060
  return serverUninstall(flags);
9476
10061
  default:
@@ -20671,7 +21256,7 @@ line_count: 37`,
20671
21256
  metadata: {
20672
21257
  featured: true,
20673
21258
  order: 2,
20674
- content_embed_examples: ["code.application"]
21259
+ content_embed_examples: []
20675
21260
  }
20676
21261
  };
20677
21262
 
@@ -26232,6 +26817,9 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
26232
26817
  anonymous_free_usage: {
26233
26818
  feature_notice: {
26234
26819
  text: "You are using free anonymous credits. File uploads, memories, chat sync, and some other features require creating an account."
26820
+ },
26821
+ daily_credits_exhausted: {
26822
+ text: "You used up your free daily credits. Sign up & buy credits to make full use of OpenMates."
26235
26823
  }
26236
26824
  },
26237
26825
  interactive_question_failed: {
@@ -27900,6 +28488,50 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
27900
28488
  account_created: {
27901
28489
  text: "Account created"
27902
28490
  },
28491
+ account_created_second_login_title: {
28492
+ text: "Add a second login method"
28493
+ },
28494
+ account_created_second_login_info: {
28495
+ text: "If you signed up with a passkey, add password plus 2FA as a backup. If you signed up with password plus 2FA, add a passkey for faster secure login."
28496
+ },
28497
+ existing_account: {
28498
+ subject: {
28499
+ text: "An OpenMates account already exists for this email"
28500
+ },
28501
+ title: {
28502
+ text: "An account with your email already exists"
28503
+ },
28504
+ intro: {
28505
+ text: "Someone tried to create a new OpenMates account with this email address, but this email is already connected to an existing account."
28506
+ },
28507
+ saved_logins_title: {
28508
+ text: "Check your saved logins"
28509
+ },
28510
+ saved_logins_body: {
28511
+ text: "Your browser, password manager, or device may already have the login saved. Look for OpenMates in saved passwords or passkeys before creating a new account."
28512
+ },
28513
+ login_methods_title: {
28514
+ text: "Try your login methods"
28515
+ },
28516
+ login_methods_body: {
28517
+ text: "You can log in with a passkey, or with your password plus 2FA. If you set up backup codes, keep them ready for the 2FA step."
28518
+ },
28519
+ login_button: {
28520
+ text: "Log in to OpenMates"
28521
+ },
28522
+ recovery_key_title: {
28523
+ text: "Have your recovery key?"
28524
+ },
28525
+ recovery_key_body: {
28526
+ text: "If you do not remember your password or passkey but still have your recovery key, use recovery key login. It preserves your encrypted chats and account data."
28527
+ },
28528
+ recovery_button: {
28529
+ text: "Open login and use recovery key"
28530
+ },
28531
+ reset_warning: {
28532
+ text: "If you lost every login method and your recovery key, account reset is the last resort. Because OpenMates encrypts your data, resetting the account can permanently delete encrypted chats, memories, app settings, embeds, passkeys, and API keys."
28533
+ }
28534
+ },
27903
28535
  password_security_reminder: {
27904
28536
  subject: {
27905
28537
  text: "Action needed to secure your OpenMates account"
@@ -28129,10 +28761,7 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
28129
28761
  text: "Welcome to OpenMates!"
28130
28762
  },
28131
28763
  complete_signup_info: {
28132
- text: "Once you completed the signup process by purchasing usage credits or redeeming a gift card, you can start using OpenMates!"
28133
- },
28134
- auto_delete_warning: {
28135
- text: "Please note: Accounts that haven't completed the signup process will be automatically deleted after 7 days."
28764
+ text: "Your account is ready. Here are a few helpful next steps to protect your access and keep a copy of your data."
28136
28765
  },
28137
28766
  want_to_delete_account: {
28138
28767
  text: "Want to delete your account?"
@@ -28920,6 +29549,9 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
28920
29549
  }
28921
29550
  },
28922
29551
  embeds: {
29552
+ learning_mode_shortened_notice: {
29553
+ text: "Shortened since Learning Mode is active."
29554
+ },
28923
29555
  weather: {
28924
29556
  rain_radar: {
28925
29557
  no_rain: {
@@ -29262,6 +29894,15 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
29262
29894
  copy_failed: {
29263
29895
  text: "Failed to copy to clipboard"
29264
29896
  },
29897
+ code_file_downloaded: {
29898
+ text: "Code file downloaded successfully"
29899
+ },
29900
+ code_file_download_failed: {
29901
+ text: "Failed to download code file"
29902
+ },
29903
+ action_failed: {
29904
+ text: "Failed to perform action"
29905
+ },
29265
29906
  download_itinerary: {
29266
29907
  text: "Download itinerary"
29267
29908
  },
@@ -37618,6 +38259,99 @@ As of mid-2026, the severe supply shocks from the 2024\u20132025 avian flu have
37618
38259
  incognito: {
37619
38260
  text: "Incognito"
37620
38261
  },
38262
+ learning_mode: {
38263
+ text: "Learning"
38264
+ },
38265
+ learning_mode_active: {
38266
+ text: "Active for all chats on this account"
38267
+ },
38268
+ learning_mode_inactive: {
38269
+ text: "Teach step-by-step instead of giving complete answers"
38270
+ },
38271
+ learning_mode_load_error: {
38272
+ text: "Could not load Learning Mode status."
38273
+ },
38274
+ learning_mode_save_error: {
38275
+ text: "Could not update Learning Mode."
38276
+ },
38277
+ learning_mode_enabled: {
38278
+ text: "Learning Mode enabled."
38279
+ },
38280
+ learning_mode_disabled: {
38281
+ text: "Learning Mode disabled."
38282
+ },
38283
+ learning_mode_age_group_prompt: {
38284
+ text: "Learner age group: under_10, 10_12, 13_15, 16_18, or adult"
38285
+ },
38286
+ learning_mode_enable_passcode_prompt: {
38287
+ text: "Set a Learning Mode passcode."
38288
+ },
38289
+ learning_mode_disable_passcode_prompt: {
38290
+ text: "Enter the Learning Mode passcode to disable it."
38291
+ },
38292
+ learning_mode_invalid_age_group: {
38293
+ text: "Invalid age group. Use under_10, 10_12, 13_15, 16_18, or adult."
38294
+ },
38295
+ learning_mode_enable_description: {
38296
+ text: "Choose the learner age group and set a passcode. Learning Mode will apply to every chat on this account."
38297
+ },
38298
+ learning_mode_disable_description: {
38299
+ text: "Enter the Learning Mode passcode to turn it off for this account."
38300
+ },
38301
+ learning_mode_guest_description: {
38302
+ text: "Choose the learner age group. For guests, Learning Mode lasts only for this browser session and has no passcode."
38303
+ },
38304
+ learning_mode_active_detail: {
38305
+ text: "Learning Mode is active for every chat on this account. Answers stay teaching-first and selected tools are restricted."
38306
+ },
38307
+ learning_mode_inactive_detail: {
38308
+ text: "When enabled, OpenMates guides the learner step by step instead of giving complete answers."
38309
+ },
38310
+ learning_mode_guest_active_detail: {
38311
+ text: "Learning Mode is active for anonymous chats in this browser session. Create an account to lock it with a passcode across devices."
38312
+ },
38313
+ learning_mode_guest_inactive_detail: {
38314
+ text: "Guests can try Learning Mode for anonymous chats in this browser session. Create an account to lock it with a passcode."
38315
+ },
38316
+ learning_mode_age_group_label: {
38317
+ text: "Learner age group"
38318
+ },
38319
+ learning_mode_age_under_10: {
38320
+ text: "Under 10"
38321
+ },
38322
+ learning_mode_age_10_12: {
38323
+ text: "10 to 12"
38324
+ },
38325
+ learning_mode_age_13_15: {
38326
+ text: "13 to 15"
38327
+ },
38328
+ learning_mode_age_16_18: {
38329
+ text: "16 to 18"
38330
+ },
38331
+ learning_mode_age_adult: {
38332
+ text: "Adult"
38333
+ },
38334
+ learning_mode_enable_passcode_label: {
38335
+ text: "Set passcode"
38336
+ },
38337
+ learning_mode_disable_passcode_label: {
38338
+ text: "Enter passcode"
38339
+ },
38340
+ learning_mode_enable_passcode_placeholder: {
38341
+ text: "Create a passcode"
38342
+ },
38343
+ learning_mode_disable_passcode_placeholder: {
38344
+ text: "Learning Mode passcode"
38345
+ },
38346
+ learning_mode_passcode_required: {
38347
+ text: "Enter a passcode to continue."
38348
+ },
38349
+ learning_mode_enable_button: {
38350
+ text: "Start Learning Mode"
38351
+ },
38352
+ learning_mode_disable_button: {
38353
+ text: "Turn off Learning Mode"
38354
+ },
37621
38355
  incognito_explainer_description: {
37622
38356
  text: "Incognito mode applies only to new chats you create. New chats created while incognito mode is active are not synced across devices, not stored on the server, and not cached. These chats exist only in your current browser session. Existing chats remain unchanged."
37623
38357
  },
@@ -41455,7 +42189,12 @@ function isInteractiveQuestionPayload(value) {
41455
42189
  if (!isQuestionType(payload.type)) return false;
41456
42190
  if (typeof payload.id !== "string" || payload.id.trim().length === 0) return false;
41457
42191
  if (payload.type === "choice") {
41458
- return typeof payload.question === "string" && payload.question.trim().length > 0 && Array.isArray(payload.options) && payload.options.length > 0;
42192
+ if (typeof payload.question !== "string" || payload.question.trim().length === 0) return false;
42193
+ if (!Array.isArray(payload.options) || payload.options.length === 0) return false;
42194
+ if (payload.custom_option_id !== void 0) {
42195
+ return typeof payload.custom_option_id === "string" && payload.options.some((option) => option.id === payload.custom_option_id);
42196
+ }
42197
+ return true;
41459
42198
  }
41460
42199
  if (payload.type === "input") return Array.isArray(payload.fields) && payload.fields.length > 0;
41461
42200
  if (payload.type === "slider") {
@@ -41463,13 +42202,27 @@ function isInteractiveQuestionPayload(value) {
41463
42202
  }
41464
42203
  if (payload.type === "swipe") return Array.isArray(payload.cards) && payload.cards.length > 0;
41465
42204
  if (payload.type === "rating") {
41466
- return typeof payload.question === "string" && payload.question.trim().length > 0 && (typeof payload.max === "number" || typeof payload.scale === "number");
42205
+ return typeof payload.question === "string" && payload.question.trim().length > 0 && (typeof payload.max_stars === "number" || typeof payload.max === "number" || typeof payload.scale === "number");
41467
42206
  }
41468
42207
  return false;
41469
42208
  }
41470
42209
  function isQuestionType(value) {
41471
42210
  return value === "choice" || value === "input" || value === "slider" || value === "swipe" || value === "rating";
41472
42211
  }
42212
+ function formatInteractiveQuestionAnswer(question, answer) {
42213
+ const responsePayload = buildResponsePayload(question, answer);
42214
+ const displayText = buildDisplayText(question, answer);
42215
+ const protocol = JSON.stringify(responsePayload, null, 2);
42216
+ return {
42217
+ displayText,
42218
+ messageContent: `${displayText}
42219
+
42220
+ \`\`\`interactive_response
42221
+ ${protocol}
42222
+ \`\`\``,
42223
+ responsePayload
42224
+ };
42225
+ }
41473
42226
  function toWaitingForUserResult(params) {
41474
42227
  return {
41475
42228
  status: "waiting_for_user",
@@ -41479,6 +42232,50 @@ function toWaitingForUserResult(params) {
41479
42232
  question: params.question
41480
42233
  };
41481
42234
  }
42235
+ function buildResponsePayload(question, answer) {
42236
+ return {
42237
+ id: question.id,
42238
+ ...answer
42239
+ };
42240
+ }
42241
+ function buildDisplayText(question, answer) {
42242
+ if (question.type === "choice") {
42243
+ const selection = Array.isArray(answer.selection) ? answer.selection.map(String) : [];
42244
+ const optionsById = new Map((question.options ?? []).map((option) => [option.id, option.text]));
42245
+ const customAnswer = answer.custom_answer == null ? "" : String(answer.custom_answer).trim();
42246
+ return selection.map((id) => customAnswer && isCustomChoiceOption(question, id) ? customAnswer : optionsById.get(id) ?? id).join(", ");
42247
+ }
42248
+ if (question.type === "input") {
42249
+ const values = answer.inputs && typeof answer.inputs === "object" ? answer.inputs : answer.values && typeof answer.values === "object" ? answer.values : answer;
42250
+ return Object.entries(values).filter(([key]) => key !== "id").map(([, value]) => String(value)).filter(Boolean).join("\n");
42251
+ }
42252
+ if (question.type === "slider") {
42253
+ return answer.value == null ? "" : String(answer.value);
42254
+ }
42255
+ if (question.type === "swipe") {
42256
+ const liked = Array.isArray(answer.liked) ? answer.liked.map(String) : [];
42257
+ const cardsById = new Map((question.cards ?? []).map((card) => [card.id, card.text]));
42258
+ return liked.map((id) => cardsById.get(id) ?? id).join(", ");
42259
+ }
42260
+ if (question.type === "rating") {
42261
+ const rating = answer.rating == null ? "" : String(answer.rating);
42262
+ const comment = answer.comment == null ? "" : String(answer.comment).trim();
42263
+ return [rating, comment].filter(Boolean).join("\n");
42264
+ }
42265
+ return "";
42266
+ }
42267
+ function isCustomChoiceOption(question, optionId) {
42268
+ if (question.custom_option_id) return optionId === question.custom_option_id;
42269
+ const optionText = (question.options ?? []).find((option) => option.id === optionId)?.text.trim().toLowerCase() ?? "";
42270
+ return [
42271
+ "i give you my own answer",
42272
+ "my own answer",
42273
+ "own answer",
42274
+ "custom answer",
42275
+ "something else",
42276
+ "other"
42277
+ ].some((pattern) => optionText === pattern || optionText.includes(pattern));
42278
+ }
41482
42279
 
41483
42280
  // src/feedback.ts
41484
42281
  var ASSISTANT_FEEDBACK_THANKS = "Thanks for the feedback!";
@@ -41502,6 +42299,1022 @@ function buildAssistantFeedbackDecision(rating) {
41502
42299
  };
41503
42300
  }
41504
42301
 
42302
+ // src/benchmark.ts
42303
+ import { randomUUID as randomUUID3 } from "crypto";
42304
+ import { existsSync as existsSync6, mkdtempSync, readFileSync as readFileSync6, readdirSync, writeFileSync as writeFileSync4 } from "fs";
42305
+ import { tmpdir } from "os";
42306
+ import { dirname as dirname2, join as join4, resolve as resolve5 } from "path";
42307
+ import { fileURLToPath } from "url";
42308
+ var DEFAULT_JUDGE_MODEL = "google/gemini-3-flash-preview";
42309
+ var DEFAULT_EXTENSIVE_SIZE = 10;
42310
+ var DEFAULT_PARALLEL = 4;
42311
+ var FIXTURE_IMAGE_SVG = `<?xml version="1.0" encoding="UTF-8"?>
42312
+ <svg xmlns="http://www.w3.org/2000/svg" width="1200" height="800" viewBox="0 0 1200 800">
42313
+ <rect width="1200" height="800" fill="#d8ecff"/>
42314
+ <rect y="560" width="1200" height="240" fill="#d7c39a"/>
42315
+ <text x="600" y="88" text-anchor="middle" font-family="Arial, sans-serif" font-size="44" font-weight="700" fill="#23344d">Brandenburger Tor, Berlin</text>
42316
+ <g transform="translate(160 170)" fill="#c9aa6a" stroke="#5d4522" stroke-width="8">
42317
+ <rect x="80" y="160" width="800" height="58"/>
42318
+ <rect x="120" y="218" width="720" height="48"/>
42319
+ <rect x="150" y="266" width="660" height="42"/>
42320
+ <g fill="#d9bd7d">
42321
+ <rect x="170" y="308" width="54" height="250"/>
42322
+ <rect x="285" y="308" width="54" height="250"/>
42323
+ <rect x="400" y="308" width="54" height="250"/>
42324
+ <rect x="515" y="308" width="54" height="250"/>
42325
+ <rect x="630" y="308" width="54" height="250"/>
42326
+ <rect x="745" y="308" width="54" height="250"/>
42327
+ </g>
42328
+ <rect x="130" y="558" width="700" height="50"/>
42329
+ <path d="M480 30 C530 72 620 88 682 48 L720 84 C652 142 530 124 456 78 Z" fill="#3e6f5f"/>
42330
+ <circle cx="510" cy="92" r="22" fill="#3e6f5f"/>
42331
+ <circle cx="625" cy="92" r="22" fill="#3e6f5f"/>
42332
+ <path d="M565 38 l26 78 h-52 z" fill="#3e6f5f"/>
42333
+ </g>
42334
+ <text x="600" y="740" text-anchor="middle" font-family="Arial, sans-serif" font-size="32" fill="#23344d">Neoclassical gate with Quadriga on top</text>
42335
+ </svg>
42336
+ `;
42337
+ var QUICK_CASES = [
42338
+ {
42339
+ id: "quick-exact-token",
42340
+ suite: "quick",
42341
+ title: "Exact token smoke test",
42342
+ prompt: "Reply with exactly this token and no extra text: BENCHMARK_SMOKE_OK",
42343
+ complexity: "basic",
42344
+ category: "smoke",
42345
+ expectedIncludes: "BENCHMARK_SMOKE_OK",
42346
+ judge: true,
42347
+ estimatedInputTokens: 12e3,
42348
+ estimatedOutputTokens: 64
42349
+ },
42350
+ {
42351
+ id: "quick-arithmetic",
42352
+ suite: "quick",
42353
+ title: "Arithmetic direct answer",
42354
+ prompt: "Compute 19 * 23. Reply with only the integer result.",
42355
+ complexity: "basic",
42356
+ category: "math",
42357
+ expectedIncludes: "437",
42358
+ judge: true,
42359
+ estimatedInputTokens: 12e3,
42360
+ estimatedOutputTokens: 64
42361
+ },
42362
+ {
42363
+ id: "quick-code",
42364
+ suite: "quick",
42365
+ title: "Small code generation",
42366
+ prompt: "Write a TypeScript function isPalindrome(input: string): boolean that ignores spaces, punctuation, and case. Include only the function and one short usage example.",
42367
+ complexity: "medium",
42368
+ category: "coding",
42369
+ judge: true,
42370
+ estimatedInputTokens: 12200,
42371
+ estimatedOutputTokens: 650
42372
+ },
42373
+ {
42374
+ id: "quick-image-brandenburger-tor",
42375
+ suite: "quick",
42376
+ title: "Default image understanding",
42377
+ prompt: "Look at the attached image. What landmark is shown, when was it built, and who designed it? Answer in three concise bullet points.",
42378
+ complexity: "medium",
42379
+ category: "image",
42380
+ image: "default",
42381
+ expectedIncludes: "Brandenburg",
42382
+ judge: true,
42383
+ estimatedInputTokens: 13500,
42384
+ estimatedOutputTokens: 350
42385
+ },
42386
+ {
42387
+ id: "quick-followup-continuity",
42388
+ suite: "quick",
42389
+ title: "Short multi-turn continuity",
42390
+ prompt: "Create a three-step plan for evaluating whether a new AI model is ready for production use.",
42391
+ complexity: "medium",
42392
+ category: "multi_turn",
42393
+ judge: true,
42394
+ estimatedInputTokens: 14e3,
42395
+ estimatedOutputTokens: 900,
42396
+ followUps: [
42397
+ { prompt: "Now make step 2 more concrete with two measurable checks." },
42398
+ { prompt: "Summarize the final plan in one sentence." }
42399
+ ]
42400
+ }
42401
+ ];
42402
+ var EXTENSIVE_CASES = [
42403
+ ...QUICK_CASES,
42404
+ {
42405
+ id: "extensive-coding-debug",
42406
+ suite: "extensive",
42407
+ title: "Debug a JavaScript bug",
42408
+ prompt: "A JavaScript function returns NaN when summing prices from [{price: '12.50'}, {price: undefined}]. Explain the bug and write a corrected function.",
42409
+ complexity: "medium",
42410
+ category: "coding",
42411
+ judge: true,
42412
+ estimatedInputTokens: 12300,
42413
+ estimatedOutputTokens: 850
42414
+ },
42415
+ {
42416
+ id: "extensive-coding-api-design",
42417
+ suite: "extensive",
42418
+ title: "Design a small API contract",
42419
+ prompt: "Design a minimal JSON API for creating and listing benchmark runs. Include request/response examples and one validation error.",
42420
+ complexity: "advanced",
42421
+ category: "coding",
42422
+ judge: true,
42423
+ estimatedInputTokens: 12300,
42424
+ estimatedOutputTokens: 1e3
42425
+ },
42426
+ {
42427
+ id: "extensive-reasoning-tradeoffs",
42428
+ suite: "extensive",
42429
+ title: "Reason about benchmark tradeoffs",
42430
+ prompt: "Compare deterministic assertions and LLM-as-judge evaluation for model benchmarks. Give two strengths and two risks for each.",
42431
+ complexity: "medium",
42432
+ category: "reasoning",
42433
+ judge: true,
42434
+ estimatedInputTokens: 12200,
42435
+ estimatedOutputTokens: 800
42436
+ },
42437
+ {
42438
+ id: "extensive-planning",
42439
+ suite: "extensive",
42440
+ title: "Operational rollout plan",
42441
+ prompt: "Create a rollout checklist for switching a production chatbot from one model to another. Include monitoring, rollback, and user-visible risk checks.",
42442
+ complexity: "advanced",
42443
+ category: "synthesis",
42444
+ judge: true,
42445
+ estimatedInputTokens: 12300,
42446
+ estimatedOutputTokens: 950
42447
+ },
42448
+ {
42449
+ id: "extensive-long-context-followup",
42450
+ suite: "extensive",
42451
+ title: "Prebuilt 20-message long chat follow-up",
42452
+ prompt: "Based on the earlier discussion, choose the best launch strategy and explain why in five bullets.",
42453
+ complexity: "advanced",
42454
+ category: "long_context",
42455
+ longContext: true,
42456
+ judge: true,
42457
+ estimatedInputTokens: 18500,
42458
+ estimatedOutputTokens: 900
42459
+ },
42460
+ {
42461
+ id: "extensive-policy-summary",
42462
+ suite: "extensive",
42463
+ title: "Policy summarization",
42464
+ prompt: "Summarize why privacy-preserving benchmark logs should avoid raw user prompts. Include a concrete safer alternative.",
42465
+ complexity: "medium",
42466
+ category: "reasoning",
42467
+ judge: true,
42468
+ estimatedInputTokens: 12200,
42469
+ estimatedOutputTokens: 650
42470
+ },
42471
+ {
42472
+ id: "extensive-structured-output",
42473
+ suite: "extensive",
42474
+ title: "Structured JSON output",
42475
+ prompt: "Return only JSON with keys risk, mitigation, and confidence for the risk: benchmark results are biased by prompt wording.",
42476
+ complexity: "medium",
42477
+ category: "synthesis",
42478
+ judge: true,
42479
+ estimatedInputTokens: 12200,
42480
+ estimatedOutputTokens: 350
42481
+ },
42482
+ {
42483
+ id: "extensive-creative-constraint",
42484
+ suite: "extensive",
42485
+ title: "Creative constrained response",
42486
+ prompt: "Write a six-line product note announcing model comparisons. Each line must be under 70 characters and avoid hype words like revolutionary or magical.",
42487
+ complexity: "medium",
42488
+ category: "synthesis",
42489
+ judge: true,
42490
+ estimatedInputTokens: 12200,
42491
+ estimatedOutputTokens: 500
42492
+ },
42493
+ {
42494
+ id: "extensive-data-reasoning",
42495
+ suite: "extensive",
42496
+ title: "Interpret metrics",
42497
+ prompt: "A benchmark has pass rates 8/10, 7/10, and 9/10 across three runs. Explain what you can and cannot conclude from this sample.",
42498
+ complexity: "medium",
42499
+ category: "reasoning",
42500
+ judge: true,
42501
+ estimatedInputTokens: 12200,
42502
+ estimatedOutputTokens: 600
42503
+ },
42504
+ {
42505
+ id: "extensive-security-review",
42506
+ suite: "extensive",
42507
+ title: "Security review",
42508
+ prompt: "Review this benchmark design for security risks: it logs prompts, outputs, model ids, and usage costs to a shared file. List risks and safer defaults.",
42509
+ complexity: "advanced",
42510
+ category: "reasoning",
42511
+ judge: true,
42512
+ estimatedInputTokens: 12300,
42513
+ estimatedOutputTokens: 850
42514
+ },
42515
+ {
42516
+ id: "extensive-followup-requirements",
42517
+ suite: "extensive",
42518
+ title: "Three-turn requirements refinement",
42519
+ prompt: "Draft acceptance criteria for a CLI benchmark comparison feature.",
42520
+ complexity: "advanced",
42521
+ category: "multi_turn",
42522
+ judge: true,
42523
+ estimatedInputTokens: 14500,
42524
+ estimatedOutputTokens: 1100,
42525
+ followUps: [
42526
+ { prompt: "Add one criterion about cost estimation before live runs." },
42527
+ { prompt: "Add one criterion about partial results after interruption." },
42528
+ { prompt: "Now compress the criteria to five bullets total." }
42529
+ ]
42530
+ },
42531
+ {
42532
+ id: "extensive-coding-tests",
42533
+ suite: "extensive",
42534
+ title: "Write tests for parser behavior",
42535
+ prompt: "Write Node.js test cases for a function parseSuites(value) that accepts quick, extensive, all, and comma-separated lists, and rejects unknown suites.",
42536
+ complexity: "medium",
42537
+ category: "coding",
42538
+ judge: true,
42539
+ estimatedInputTokens: 12300,
42540
+ estimatedOutputTokens: 950
42541
+ },
42542
+ {
42543
+ id: "extensive-coding-refactor",
42544
+ suite: "extensive",
42545
+ title: "Refactor duplicated code",
42546
+ prompt: "Given two duplicated TypeScript loops that build arrays of result objects, explain when to extract a helper and write the helper signature.",
42547
+ complexity: "medium",
42548
+ category: "coding",
42549
+ judge: true,
42550
+ estimatedInputTokens: 12300,
42551
+ estimatedOutputTokens: 750
42552
+ },
42553
+ {
42554
+ id: "extensive-comparison-analysis",
42555
+ suite: "extensive",
42556
+ title: "Compare two model outputs",
42557
+ prompt: "Explain how you would compare two model outputs when one is concise but misses caveats and the other is verbose but complete.",
42558
+ complexity: "medium",
42559
+ category: "reasoning",
42560
+ judge: true,
42561
+ estimatedInputTokens: 12200,
42562
+ estimatedOutputTokens: 650
42563
+ },
42564
+ {
42565
+ id: "extensive-failure-mode",
42566
+ suite: "extensive",
42567
+ title: "Failure-mode analysis",
42568
+ prompt: "List five failure modes for image-understanding benchmarks and one mitigation for each.",
42569
+ complexity: "advanced",
42570
+ category: "image",
42571
+ judge: true,
42572
+ estimatedInputTokens: 12300,
42573
+ estimatedOutputTokens: 900
42574
+ }
42575
+ ];
42576
+ async function handleBenchmark(client, subcommand, rest, flags) {
42577
+ if (!subcommand || subcommand === "help" || flags.help === true) {
42578
+ printBenchmarkHelp();
42579
+ return;
42580
+ }
42581
+ if (subcommand !== "model") {
42582
+ throw new Error(`Unknown benchmark command '${subcommand}'. Run 'openmates benchmark --help'.`);
42583
+ }
42584
+ const targetModels = rest.filter((arg) => !arg.startsWith("--"));
42585
+ if (targetModels.length === 0) {
42586
+ throw new Error("Missing target model. Usage: openmates benchmark model <provider/model> [model-b] --confirm-spend-credits");
42587
+ }
42588
+ const compare = flags.compare === true;
42589
+ if (targetModels.length > 1 && !compare) {
42590
+ throw new Error("Multiple target models require --compare.");
42591
+ }
42592
+ if (compare && targetModels.length < 2) {
42593
+ throw new Error("--compare requires at least two target models.");
42594
+ }
42595
+ const judgeModel = typeof flags["judge-model"] === "string" ? flags["judge-model"] : DEFAULT_JUDGE_MODEL;
42596
+ const suites = parseSuites(flags.suite);
42597
+ const runs = parseRuns(flags.runs);
42598
+ const extensiveSize = parseExtensiveSize(flags["extensive-size"]);
42599
+ const parallel = parseParallel(flags.parallel);
42600
+ const caseIds = parseCaseIds(flags.case);
42601
+ const dryRun = flags["dry-run"] === true;
42602
+ const output = typeof flags.output === "string" ? flags.output : void 0;
42603
+ const runId = typeof flags["run-id"] === "string" ? flags["run-id"] : randomUUID3();
42604
+ const imagePath = typeof flags.image === "string" ? resolve5(flags.image) : defaultImageFixturePath();
42605
+ if (!dryRun && flags["confirm-spend-credits"] !== true) {
42606
+ throw new Error(
42607
+ "Benchmark runs spend real credits from the logged-in account. Rerun with --confirm-spend-credits, or use --dry-run to preview the plan."
42608
+ );
42609
+ }
42610
+ const cases = filterCases(expandCases(suites, runs, extensiveSize), caseIds);
42611
+ const pricing = loadPricingForModels([...targetModels, judgeModel]);
42612
+ const estimate = estimateCredits(cases, targetModels, judgeModel, pricing);
42613
+ const result = makeBaseResult({
42614
+ runId,
42615
+ targetModels,
42616
+ judgeModel,
42617
+ suites,
42618
+ runs,
42619
+ compare,
42620
+ parallel,
42621
+ extensiveSize,
42622
+ dryRun,
42623
+ estimate,
42624
+ totalJobs: cases.length * targetModels.length
42625
+ });
42626
+ if (dryRun) {
42627
+ writeBenchmarkResult(result, flags, output);
42628
+ return;
42629
+ }
42630
+ if (!client.hasSession()) {
42631
+ throw new Error("Benchmark runs require login. Run 'openmates login' first.");
42632
+ }
42633
+ let interrupted = false;
42634
+ const onInterrupt = () => {
42635
+ interrupted = true;
42636
+ };
42637
+ process.once("SIGINT", onInterrupt);
42638
+ try {
42639
+ const jobs = cases.flatMap((benchmarkCase) => targetModels.map((model) => ({ model, benchmarkCase })));
42640
+ await runPool(jobs, parallel, async (job) => {
42641
+ if (interrupted) return;
42642
+ const caseResult = await runCaseJob({ client, job, judgeModel, runId, imagePath });
42643
+ result.cases.push(caseResult);
42644
+ recomputeResult(result, jobs.length, interrupted);
42645
+ });
42646
+ } finally {
42647
+ process.off("SIGINT", onInterrupt);
42648
+ }
42649
+ recomputeResult(result, cases.length * targetModels.length, interrupted);
42650
+ writeBenchmarkResult(result, flags, output);
42651
+ }
42652
+ function printBenchmarkHelp() {
42653
+ console.log(`Benchmark commands:
42654
+ openmates benchmark model <provider/model> [provider/model...] --confirm-spend-credits [--compare] [--suite quick|extensive|all] [--json]
42655
+
42656
+ Runs real incognito chat requests through the OpenMates product path. Live runs
42657
+ spend the logged-in user's credits and usage entries are grouped as benchmark spend.
42658
+
42659
+ Options:
42660
+ --confirm-spend-credits Required for live benchmark runs
42661
+ --dry-run Preview the benchmark plan without inference or spend
42662
+ --compare Compare two or more target models
42663
+ --suite <list> Comma-separated suites: quick, extensive, all (default: quick)
42664
+ --case <id[,id...]> Run only specific case id(s) from the selected suites
42665
+ --extensive-size <n> Extensive cases to run: 5, 10, or 20 (default: ${DEFAULT_EXTENSIVE_SIZE})
42666
+ --runs <n> Repeat each selected case (default: 1)
42667
+ --parallel <n> Concurrent target case requests (default: ${DEFAULT_PARALLEL})
42668
+ --judge-model <provider/model> Judge for evaluated cases (default: ${DEFAULT_JUDGE_MODEL})
42669
+ --image <path> Override default Brandenburger Tor image fixture
42670
+ --run-id <id> Reuse a benchmark run id for grouping
42671
+ --output <path> Save JSON result to a file
42672
+ --json Print JSON result`);
42673
+ }
42674
+ function parseSuites(value) {
42675
+ if (value === void 0 || value === false) return ["quick"];
42676
+ if (value === true) throw new Error("--suite requires a value");
42677
+ const suites = value.split(",").map((suite) => suite.trim()).filter(Boolean);
42678
+ if (suites.includes("all")) return ["quick", "extensive"];
42679
+ const allowed = /* @__PURE__ */ new Set(["quick", "extensive"]);
42680
+ const invalid = suites.filter((suite) => !allowed.has(suite));
42681
+ if (invalid.length > 0 || suites.length === 0) {
42682
+ throw new Error("Invalid --suite. Use quick, extensive, or all.");
42683
+ }
42684
+ return [...new Set(suites)];
42685
+ }
42686
+ function parseRuns(value) {
42687
+ if (value === void 0 || value === false) return 1;
42688
+ if (value === true) throw new Error("--runs requires a value");
42689
+ const parsed = Number.parseInt(value, 10);
42690
+ if (!Number.isInteger(parsed) || parsed < 1 || parsed > 20) {
42691
+ throw new Error("--runs must be an integer from 1 to 20");
42692
+ }
42693
+ return parsed;
42694
+ }
42695
+ function parseExtensiveSize(value) {
42696
+ if (value === void 0 || value === false) return DEFAULT_EXTENSIVE_SIZE;
42697
+ if (value === true) throw new Error("--extensive-size requires a value");
42698
+ const parsed = Number.parseInt(value, 10);
42699
+ if (![5, 10, 20].includes(parsed)) {
42700
+ throw new Error("--extensive-size must be 5, 10, or 20");
42701
+ }
42702
+ return parsed;
42703
+ }
42704
+ function parseParallel(value) {
42705
+ if (value === void 0 || value === false) return DEFAULT_PARALLEL;
42706
+ if (value === true) throw new Error("--parallel requires a value");
42707
+ const parsed = Number.parseInt(value, 10);
42708
+ if (!Number.isInteger(parsed) || parsed < 1 || parsed > 20) {
42709
+ throw new Error("--parallel must be an integer from 1 to 20");
42710
+ }
42711
+ return parsed;
42712
+ }
42713
+ function parseCaseIds(value) {
42714
+ if (value === void 0 || value === false) return [];
42715
+ if (value === true) throw new Error("--case requires a case id");
42716
+ const caseIds = value.split(",").map((caseId) => caseId.trim()).filter(Boolean);
42717
+ if (caseIds.length === 0) throw new Error("--case requires at least one case id");
42718
+ return [...new Set(caseIds)];
42719
+ }
42720
+ function filterCases(cases, caseIds) {
42721
+ if (caseIds.length === 0) return cases;
42722
+ const availableIds = new Set(cases.map((benchmarkCase) => benchmarkCase.id));
42723
+ const missing = caseIds.filter((caseId) => !availableIds.has(caseId));
42724
+ if (missing.length > 0) {
42725
+ throw new Error(
42726
+ `Unknown benchmark case id(s): ${missing.join(", ")}. Available in selected suite(s): ${[...availableIds].sort().join(", ")}`
42727
+ );
42728
+ }
42729
+ return cases.filter((benchmarkCase) => caseIds.includes(benchmarkCase.id));
42730
+ }
42731
+ function expandCases(suites, runs, extensiveSize) {
42732
+ const selected = [];
42733
+ if (suites.includes("quick")) selected.push(...QUICK_CASES);
42734
+ if (suites.includes("extensive")) selected.push(...selectExtensiveCases(extensiveSize));
42735
+ const uniqueSelected = dedupeCases(selected);
42736
+ const expanded = [];
42737
+ for (let run = 1; run <= runs; run += 1) {
42738
+ for (const benchmarkCase of uniqueSelected) expanded.push({ ...benchmarkCase, run });
42739
+ }
42740
+ return expanded;
42741
+ }
42742
+ function selectExtensiveCases(size) {
42743
+ const cases = dedupeCases(EXTENSIVE_CASES).slice(0, size);
42744
+ const minimumCoding = Math.ceil(size * 0.15);
42745
+ const codingCount = cases.filter((benchmarkCase) => benchmarkCase.category === "coding").length;
42746
+ if (codingCount >= minimumCoding) return cases;
42747
+ const selectedIds = new Set(cases.map((benchmarkCase) => benchmarkCase.id));
42748
+ const codingBackfill = EXTENSIVE_CASES.filter(
42749
+ (benchmarkCase) => benchmarkCase.category === "coding" && !selectedIds.has(benchmarkCase.id)
42750
+ );
42751
+ const result = [...cases];
42752
+ for (const codingCase of codingBackfill) {
42753
+ let replaceIndex = -1;
42754
+ for (let index = result.length - 1; index >= 0; index -= 1) {
42755
+ if (result[index]?.category !== "coding") {
42756
+ replaceIndex = index;
42757
+ break;
42758
+ }
42759
+ }
42760
+ if (replaceIndex === -1) break;
42761
+ result[replaceIndex] = codingCase;
42762
+ if (result.filter((benchmarkCase) => benchmarkCase.category === "coding").length >= minimumCoding) break;
42763
+ }
42764
+ return result;
42765
+ }
42766
+ function dedupeCases(cases) {
42767
+ const seen = /* @__PURE__ */ new Set();
42768
+ const result = [];
42769
+ for (const benchmarkCase of cases) {
42770
+ if (seen.has(benchmarkCase.id)) continue;
42771
+ seen.add(benchmarkCase.id);
42772
+ result.push(benchmarkCase);
42773
+ }
42774
+ return result;
42775
+ }
42776
+ async function runCaseJob(params) {
42777
+ const { client, job, judgeModel, runId, imagePath } = params;
42778
+ const { model, benchmarkCase } = job;
42779
+ const startedAt = Date.now();
42780
+ const turns = [];
42781
+ const history = benchmarkCase.longContext ? buildLongContextHistory() : [];
42782
+ let chatId;
42783
+ try {
42784
+ const initialPrompt = await buildPromptWithAttachments(client, benchmarkCase, model, imagePath);
42785
+ const targetResponse = await sendBenchmarkTurn({
42786
+ client,
42787
+ model,
42788
+ judgeModel,
42789
+ runId,
42790
+ benchmarkCase,
42791
+ prompt: initialPrompt.message,
42792
+ chatId,
42793
+ history,
42794
+ preparedEmbeds: initialPrompt.embeds,
42795
+ caseId: benchmarkCase.id
42796
+ });
42797
+ chatId = targetResponse.chatId;
42798
+ turns.push(targetResponse.turn);
42799
+ appendHistory(history, "user", initialPrompt.message);
42800
+ appendHistory(history, "assistant", targetResponse.turn.assistant);
42801
+ for (const [index, followUp] of (benchmarkCase.followUps ?? []).entries()) {
42802
+ const response = await sendBenchmarkTurn({
42803
+ client,
42804
+ model,
42805
+ judgeModel,
42806
+ runId,
42807
+ benchmarkCase,
42808
+ prompt: `${modelMention(model)} ${followUp.prompt}`,
42809
+ chatId,
42810
+ history,
42811
+ caseId: `${benchmarkCase.id}:followup-${index + 1}`
42812
+ });
42813
+ chatId = response.chatId;
42814
+ turns.push(response.turn);
42815
+ appendHistory(history, "user", response.rawPrompt);
42816
+ appendHistory(history, "assistant", response.turn.assistant);
42817
+ }
42818
+ const assistant = turns.at(-1)?.assistant ?? "";
42819
+ const caseResult = {
42820
+ id: benchmarkCase.id,
42821
+ suite: benchmarkCase.suite,
42822
+ title: benchmarkCase.title,
42823
+ model,
42824
+ run: benchmarkCase.run,
42825
+ complexity: benchmarkCase.complexity,
42826
+ category: benchmarkCase.category,
42827
+ prompt: benchmarkCase.prompt,
42828
+ assistant,
42829
+ modelName: turns.at(-1)?.modelName ?? null,
42830
+ passed: benchmarkCase.expectedIncludes ? assistant.includes(benchmarkCase.expectedIncludes) : true,
42831
+ durationMs: Date.now() - startedAt,
42832
+ expectedIncludes: benchmarkCase.expectedIncludes,
42833
+ turns
42834
+ };
42835
+ if (benchmarkCase.judge) {
42836
+ caseResult.judge = await judgeCase({ client, judgeModel, targetModel: model, benchmarkCase, caseResult, runId });
42837
+ caseResult.passed = caseResult.judge.score !== null && caseResult.judge.score >= 4 && caseResult.passed;
42838
+ }
42839
+ return caseResult;
42840
+ } catch (error) {
42841
+ const message = error instanceof Error ? error.message : String(error);
42842
+ return {
42843
+ id: benchmarkCase.id,
42844
+ suite: benchmarkCase.suite,
42845
+ title: benchmarkCase.title,
42846
+ model,
42847
+ run: benchmarkCase.run,
42848
+ complexity: benchmarkCase.complexity,
42849
+ category: benchmarkCase.category,
42850
+ prompt: benchmarkCase.prompt,
42851
+ assistant: turns.at(-1)?.assistant ?? "",
42852
+ modelName: turns.at(-1)?.modelName ?? null,
42853
+ passed: false,
42854
+ durationMs: Date.now() - startedAt,
42855
+ expectedIncludes: benchmarkCase.expectedIncludes,
42856
+ turns,
42857
+ error: message
42858
+ };
42859
+ }
42860
+ }
42861
+ async function sendBenchmarkTurn(params) {
42862
+ const startedAt = Date.now();
42863
+ const response = await params.client.sendMessage({
42864
+ message: params.prompt,
42865
+ chatId: params.chatId,
42866
+ incognito: true,
42867
+ autoApproveSubChats: true,
42868
+ benchmarkMetadata: benchmarkMetadata({
42869
+ runId: params.runId,
42870
+ suite: params.benchmarkCase.suite,
42871
+ caseId: params.caseId,
42872
+ targetModel: params.model,
42873
+ judgeModel: params.judgeModel
42874
+ }),
42875
+ messageHistory: params.history,
42876
+ preparedEmbeds: params.preparedEmbeds,
42877
+ precollectResponse: true
42878
+ });
42879
+ return {
42880
+ chatId: response.chatId,
42881
+ rawPrompt: params.prompt,
42882
+ turn: {
42883
+ prompt: params.prompt,
42884
+ assistant: response.assistant,
42885
+ modelName: response.modelName,
42886
+ durationMs: Date.now() - startedAt
42887
+ }
42888
+ };
42889
+ }
42890
+ async function buildPromptWithAttachments(client, benchmarkCase, model, imagePath) {
42891
+ const baseMessage = `${modelMention(model)} ${benchmarkCase.prompt}`;
42892
+ if (benchmarkCase.image !== "default") return { message: baseMessage };
42893
+ const attachment = await prepareImageAttachment(client, imagePath);
42894
+ return { message: `${baseMessage}
42895
+
42896
+ ${attachment.messageSuffix}`, embeds: attachment.embeds };
42897
+ }
42898
+ async function prepareImageAttachment(client, imagePath) {
42899
+ if (!existsSync6(imagePath)) throw new Error(`Benchmark image not found: ${imagePath}`);
42900
+ const processed = processFiles([imagePath], null);
42901
+ if (processed.blocked.length > 0 || processed.errors.length > 0 || processed.embeds.length === 0) {
42902
+ const reason = [...processed.blocked, ...processed.errors].map((entry) => entry.error).join("; ") || "no image embed produced";
42903
+ throw new Error(`Failed to prepare benchmark image: ${reason}`);
42904
+ }
42905
+ const fileEmbed = processed.embeds[0];
42906
+ if (!fileEmbed.requiresUpload || !fileEmbed.localPath) {
42907
+ return { messageSuffix: fileEmbed.referenceBlock, embeds: [fileEmbed.embed] };
42908
+ }
42909
+ await uploadBenchmarkImage(client, fileEmbed);
42910
+ return { messageSuffix: fileEmbed.referenceBlock, embeds: [fileEmbed.embed] };
42911
+ }
42912
+ async function uploadBenchmarkImage(client, fileEmbed) {
42913
+ if (!fileEmbed.localPath) return;
42914
+ const uploadResult = await uploadFile(fileEmbed.localPath, client.getSession());
42915
+ const embedRef = fileEmbed.embed.embedRef ?? `benchmark-image-${uploadResult.embed_id.slice(0, 8)}`;
42916
+ fileEmbed.embed.embedRef = embedRef;
42917
+ fileEmbed.embed.content = toonEncodeContent({
42918
+ type: "image",
42919
+ app_id: "images",
42920
+ skill_id: "upload",
42921
+ status: "finished",
42922
+ filename: fileEmbed.displayName,
42923
+ embed_ref: embedRef,
42924
+ content_hash: uploadResult.content_hash,
42925
+ s3_base_url: uploadResult.s3_base_url,
42926
+ files: uploadResult.files,
42927
+ aes_key: uploadResult.aes_key,
42928
+ aes_nonce: uploadResult.aes_nonce,
42929
+ vault_wrapped_aes_key: uploadResult.vault_wrapped_aes_key,
42930
+ ai_detection: uploadResult.ai_detection
42931
+ });
42932
+ fileEmbed.embed.status = "finished";
42933
+ fileEmbed.embed.contentHash = uploadResult.content_hash;
42934
+ fileEmbed.embed.embedId = uploadResult.embed_id;
42935
+ fileEmbed.referenceBlock = createBenchmarkEmbedReferenceBlock(fileEmbed.embed.embedId, fileEmbed.embed.type);
42936
+ }
42937
+ function createBenchmarkEmbedReferenceBlock(embedId, embedType) {
42938
+ return `
42939
+
42940
+ \`\`\`json
42941
+ ${JSON.stringify({ type: embedType, embed_id: embedId })}
42942
+ \`\`\``;
42943
+ }
42944
+ async function judgeCase(params) {
42945
+ const startedAt = Date.now();
42946
+ const judgeResponse = await params.client.sendMessage({
42947
+ message: `${modelMention(params.judgeModel)} ${judgePrompt(params.targetModel, params.benchmarkCase, params.caseResult)}`,
42948
+ incognito: true,
42949
+ autoApproveSubChats: true,
42950
+ benchmarkMetadata: benchmarkMetadata({
42951
+ runId: params.runId,
42952
+ suite: params.benchmarkCase.suite,
42953
+ caseId: `${params.benchmarkCase.id}:judge:${params.targetModel}`,
42954
+ targetModel: params.targetModel,
42955
+ judgeModel: params.judgeModel
42956
+ }),
42957
+ precollectResponse: true
42958
+ });
42959
+ const judgment = parseJudgment(judgeResponse.assistant);
42960
+ return {
42961
+ model: params.judgeModel,
42962
+ score: judgment.score,
42963
+ reason: judgment.reason,
42964
+ raw: judgeResponse.assistant,
42965
+ durationMs: Date.now() - startedAt
42966
+ };
42967
+ }
42968
+ async function runPool(items, parallel, worker) {
42969
+ let index = 0;
42970
+ const workers = Array.from({ length: Math.min(parallel, items.length) }, async () => {
42971
+ while (index < items.length) {
42972
+ const item = items[index];
42973
+ index += 1;
42974
+ await worker(item);
42975
+ }
42976
+ });
42977
+ await Promise.all(workers);
42978
+ }
42979
+ function buildLongContextHistory() {
42980
+ const now = Math.floor(Date.now() / 1e3) - 2e3;
42981
+ const topics = [
42982
+ ["user", "We need to launch a CLI benchmark for model comparisons."],
42983
+ ["assistant", "The first goal should be a quick suite with deterministic checks."],
42984
+ ["user", "The benchmark also needs image inference."],
42985
+ ["assistant", "Use a public fixture image and ask a factual visual question."],
42986
+ ["user", "We should avoid wasting credits."],
42987
+ ["assistant", "Run a pricing preflight and require explicit spend confirmation."],
42988
+ ["user", "What about longer conversations?"],
42989
+ ["assistant", "Add a 20-message predefined history and a dependent follow-up."],
42990
+ ["user", "The extensive suite should not be too small."],
42991
+ ["assistant", "Default to 10 cases and allow 5 or 20 as alternatives."],
42992
+ ["user", "Coding quality matters."],
42993
+ ["assistant", "Reserve at least 15 percent of extensive cases for coding prompts."],
42994
+ ["user", "We also need comparison mode."],
42995
+ ["assistant", "Accept multiple models with --compare and run target jobs in parallel."],
42996
+ ["user", "How should judging work?"],
42997
+ ["assistant", "Judge each completed case immediately with Gemini so partial results remain useful."],
42998
+ ["user", "What if the process is interrupted?"],
42999
+ ["assistant", "Print or write a partial summary with completed judgments and skipped counts."],
43000
+ ["user", "What is the best launch strategy?"],
43001
+ ["assistant", "Ship quick and comparison first, then use extensive for slower releases."]
43002
+ ];
43003
+ return topics.map(([role, content], index) => ({
43004
+ message_id: `benchmark-history-${index + 1}`,
43005
+ role,
43006
+ sender_name: role === "user" ? "User" : "Assistant",
43007
+ content,
43008
+ created_at: now + index * 30
43009
+ }));
43010
+ }
43011
+ function appendHistory(history, role, content) {
43012
+ history.push({
43013
+ message_id: randomUUID3(),
43014
+ role,
43015
+ sender_name: role === "user" ? "User" : "Assistant",
43016
+ content,
43017
+ created_at: Math.floor(Date.now() / 1e3)
43018
+ });
43019
+ }
43020
+ function modelMention(model) {
43021
+ const separator = model.indexOf("/");
43022
+ if (separator === -1) return `@ai-model:${model}`;
43023
+ const provider = model.slice(0, separator);
43024
+ const modelId = model.slice(separator + 1);
43025
+ if (!provider || !modelId) return `@ai-model:${model}`;
43026
+ return `@ai-model:${modelId}:${provider}`;
43027
+ }
43028
+ function benchmarkMetadata(params) {
43029
+ return {
43030
+ source: "benchmark",
43031
+ benchmark_run_id: params.runId,
43032
+ benchmark_suite: params.suite,
43033
+ benchmark_case: params.caseId,
43034
+ benchmark_target_model: params.targetModel,
43035
+ benchmark_judge_model: params.judgeModel
43036
+ };
43037
+ }
43038
+ function judgePrompt(targetModel, benchmarkCase, result) {
43039
+ return [
43040
+ "You are judging a real OpenMates model benchmark response.",
43041
+ "Return exactly two plain-text lines, with no markdown, no code block, and no tool use.",
43042
+ "Line 1 format: BENCHMARK_SCORE=<integer from 1 to 5>",
43043
+ "Line 2 format: BENCHMARK_REASON=<one short sentence>",
43044
+ "Score for correctness, instruction-following, usefulness, and continuity where relevant.",
43045
+ `Target model: ${targetModel}`,
43046
+ `Benchmark case: ${benchmarkCase.id} (${benchmarkCase.category}, ${benchmarkCase.complexity})`,
43047
+ `Initial prompt: ${JSON.stringify(benchmarkCase.prompt)}`,
43048
+ `Turns: ${JSON.stringify(result.turns.map((turn) => ({ prompt: turn.prompt, assistant: turn.assistant })))}`
43049
+ ].join("\n");
43050
+ }
43051
+ function parseJudgment(answer) {
43052
+ const markerScore = answer.match(/BENCHMARK_SCORE\s*=\s*([1-5])/i);
43053
+ if (markerScore) {
43054
+ const reasonMatch = answer.match(/BENCHMARK_REASON\s*=\s*(.+)/i);
43055
+ return {
43056
+ score: Number.parseInt(markerScore[1], 10),
43057
+ reason: reasonMatch?.[1]?.trim() ?? null
43058
+ };
43059
+ }
43060
+ const jsonText = extractJsonObject(answer);
43061
+ if (!jsonText) return { score: null, reason: null };
43062
+ try {
43063
+ const parsed = JSON.parse(jsonText);
43064
+ const score = typeof parsed.score === "number" && Number.isFinite(parsed.score) ? parsed.score : null;
43065
+ const reason = typeof parsed.reason === "string" ? parsed.reason : null;
43066
+ return { score, reason };
43067
+ } catch {
43068
+ return { score: null, reason: null };
43069
+ }
43070
+ }
43071
+ function extractJsonObject(text) {
43072
+ const fenced = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
43073
+ if (fenced) return fenced[1];
43074
+ const start = text.indexOf("{");
43075
+ const end = text.lastIndexOf("}");
43076
+ if (start === -1 || end === -1 || end <= start) return null;
43077
+ return text.slice(start, end + 1);
43078
+ }
43079
+ function loadPricingForModels(models) {
43080
+ const availablePricing = loadProviderPricing();
43081
+ const pricing = /* @__PURE__ */ new Map();
43082
+ const missing = [];
43083
+ for (const model of [...new Set(models)]) {
43084
+ const key = normalizeModelKey(model);
43085
+ const modelPricing = availablePricing.get(key);
43086
+ if (!modelPricing) {
43087
+ missing.push(model);
43088
+ continue;
43089
+ }
43090
+ pricing.set(model, modelPricing);
43091
+ }
43092
+ if (missing.length > 0) {
43093
+ throw new Error(
43094
+ `Cannot estimate benchmark cost because pricing metadata is unavailable for: ${missing.join(", ")}. Use provider/model ids with backend provider pricing metadata.`
43095
+ );
43096
+ }
43097
+ return pricing;
43098
+ }
43099
+ function loadProviderPricing() {
43100
+ const providersDir = findProvidersDir();
43101
+ const pricing = /* @__PURE__ */ new Map();
43102
+ if (!providersDir) return pricing;
43103
+ for (const fileName of readdirSync(providersDir)) {
43104
+ if (!fileName.endsWith(".yml")) continue;
43105
+ const filePath = join4(providersDir, fileName);
43106
+ const text = readFileSync6(filePath, "utf-8");
43107
+ const provider = parseProviderId(text) ?? fileName.replace(/\.yml$/, "");
43108
+ for (const modelPricing of parseModelPricing(text, provider)) {
43109
+ pricing.set(`${modelPricing.provider}/${modelPricing.modelId}`, modelPricing);
43110
+ pricing.set(modelPricing.modelId, modelPricing);
43111
+ }
43112
+ }
43113
+ return pricing;
43114
+ }
43115
+ function parseProviderId(text) {
43116
+ const match = text.match(/^provider_id:\s*["']?([^"'\n]+)["']?/m);
43117
+ return match?.[1]?.trim() ?? null;
43118
+ }
43119
+ function parseModelPricing(text, provider) {
43120
+ const lines = text.split("\n");
43121
+ const results = [];
43122
+ let modelId = null;
43123
+ let inModel = false;
43124
+ let inputTokensPerCredit = null;
43125
+ let outputTokensPerCredit = null;
43126
+ for (const line of lines) {
43127
+ const modelMatch = line.match(/^\s{2}-\s+id:\s*["']?([^"'\n#]+)["']?/);
43128
+ if (modelMatch) {
43129
+ if (inModel && modelId && inputTokensPerCredit && outputTokensPerCredit) {
43130
+ results.push({ provider, modelId, inputTokensPerCredit, outputTokensPerCredit });
43131
+ }
43132
+ inModel = true;
43133
+ modelId = modelMatch[1].trim();
43134
+ inputTokensPerCredit = null;
43135
+ outputTokensPerCredit = null;
43136
+ continue;
43137
+ }
43138
+ if (!inModel) continue;
43139
+ const inputMatch = line.match(/^\s{10}per_credit_unit:\s*(\d+)/);
43140
+ if (inputMatch && inputTokensPerCredit === null) {
43141
+ inputTokensPerCredit = Number.parseInt(inputMatch[1], 10);
43142
+ continue;
43143
+ }
43144
+ if (inputMatch && inputTokensPerCredit !== null && outputTokensPerCredit === null) {
43145
+ outputTokensPerCredit = Number.parseInt(inputMatch[1], 10);
43146
+ }
43147
+ }
43148
+ if (inModel && modelId && inputTokensPerCredit && outputTokensPerCredit) {
43149
+ results.push({ provider, modelId, inputTokensPerCredit, outputTokensPerCredit });
43150
+ }
43151
+ return results;
43152
+ }
43153
+ function normalizeModelKey(model) {
43154
+ return model.includes("/") ? model : model;
43155
+ }
43156
+ function findProvidersDir() {
43157
+ const currentFile = fileURLToPath(import.meta.url);
43158
+ let current = dirname2(currentFile);
43159
+ for (let index = 0; index < 8; index += 1) {
43160
+ const candidate = join4(current, "backend", "providers");
43161
+ if (existsSync6(candidate)) return candidate;
43162
+ const parentCandidate = join4(current, "..", "..", "backend", "providers");
43163
+ if (existsSync6(parentCandidate)) return resolve5(parentCandidate);
43164
+ const next = dirname2(current);
43165
+ if (next === current) break;
43166
+ current = next;
43167
+ }
43168
+ return null;
43169
+ }
43170
+ function estimateCredits(cases, targetModels, judgeModel, pricing) {
43171
+ let targetCredits = 0;
43172
+ let judgeCredits = 0;
43173
+ let targetInputTokens = 0;
43174
+ let targetOutputTokens = 0;
43175
+ let judgeInputTokens = 0;
43176
+ let judgeOutputTokens = 0;
43177
+ for (const benchmarkCase of cases) {
43178
+ const turnCount = 1 + (benchmarkCase.followUps?.length ?? 0);
43179
+ for (const model of targetModels) {
43180
+ const modelPricing = pricing.get(model);
43181
+ if (!modelPricing) continue;
43182
+ const input = benchmarkCase.estimatedInputTokens * turnCount;
43183
+ const output = benchmarkCase.estimatedOutputTokens * turnCount;
43184
+ targetInputTokens += input;
43185
+ targetOutputTokens += output;
43186
+ targetCredits += creditsFor(modelPricing, input, output);
43187
+ if (benchmarkCase.judge) {
43188
+ const judgePricing = pricing.get(judgeModel);
43189
+ if (!judgePricing) continue;
43190
+ const judgeInput = Math.max(2e3, Math.ceil(output * 1.5));
43191
+ const judgeOutput = 350;
43192
+ judgeInputTokens += judgeInput;
43193
+ judgeOutputTokens += judgeOutput;
43194
+ judgeCredits += creditsFor(judgePricing, judgeInput, judgeOutput);
43195
+ }
43196
+ }
43197
+ }
43198
+ return {
43199
+ targetCredits,
43200
+ judgeCredits,
43201
+ totalCredits: targetCredits + judgeCredits,
43202
+ assumptions: { targetInputTokens, targetOutputTokens, judgeInputTokens, judgeOutputTokens }
43203
+ };
43204
+ }
43205
+ function creditsFor(pricing, inputTokens, outputTokens) {
43206
+ return Math.ceil(inputTokens / pricing.inputTokensPerCredit) + Math.ceil(outputTokens / pricing.outputTokensPerCredit);
43207
+ }
43208
+ function makeBaseResult(params) {
43209
+ return {
43210
+ command: "benchmark model",
43211
+ status: params.dryRun ? "planned" : "completed",
43212
+ runId: params.runId,
43213
+ targetModel: params.targetModels[0],
43214
+ targetModels: params.targetModels,
43215
+ judgeModel: params.judgeModel,
43216
+ suites: params.suites,
43217
+ runs: params.runs,
43218
+ compare: params.compare,
43219
+ parallel: params.parallel,
43220
+ extensiveSize: params.extensiveSize,
43221
+ spendsCredits: !params.dryRun,
43222
+ estimatedCredits: params.estimate,
43223
+ cases: [],
43224
+ modelSummaries: params.targetModels.map((model) => ({
43225
+ model,
43226
+ total: 0,
43227
+ passed: 0,
43228
+ failed: 0,
43229
+ averageJudgeScore: null,
43230
+ averageDurationMs: null
43231
+ })),
43232
+ summary: {
43233
+ total: params.totalJobs,
43234
+ completed: 0,
43235
+ passed: 0,
43236
+ failed: 0,
43237
+ skipped: params.dryRun ? params.totalJobs : 0,
43238
+ interrupted: false
43239
+ }
43240
+ };
43241
+ }
43242
+ function recomputeResult(result, totalJobs, interrupted) {
43243
+ const completed = result.cases.length;
43244
+ const passed = result.cases.filter((caseResult) => caseResult.passed).length;
43245
+ const failed = result.cases.filter((caseResult) => !caseResult.passed).length;
43246
+ result.summary = {
43247
+ total: totalJobs,
43248
+ completed,
43249
+ passed,
43250
+ failed,
43251
+ skipped: Math.max(0, totalJobs - completed),
43252
+ interrupted
43253
+ };
43254
+ result.status = interrupted || completed < totalJobs ? "partial" : "completed";
43255
+ result.modelSummaries = result.targetModels.map((model) => summarizeModel(model, result.cases));
43256
+ if (result.compare) result.comparison = buildComparison(result.modelSummaries);
43257
+ }
43258
+ function summarizeModel(model, cases) {
43259
+ const modelCases = cases.filter((caseResult) => caseResult.model === model);
43260
+ const scores = modelCases.map((caseResult) => caseResult.judge?.score).filter((score) => typeof score === "number" && Number.isFinite(score));
43261
+ const durations = modelCases.map((caseResult) => caseResult.durationMs).filter((value) => value > 0);
43262
+ return {
43263
+ model,
43264
+ total: modelCases.length,
43265
+ passed: modelCases.filter((caseResult) => caseResult.passed).length,
43266
+ failed: modelCases.filter((caseResult) => !caseResult.passed).length,
43267
+ averageJudgeScore: scores.length > 0 ? round2(scores.reduce((sum, score) => sum + score, 0) / scores.length) : null,
43268
+ averageDurationMs: durations.length > 0 ? Math.round(durations.reduce((sum, value) => sum + value, 0) / durations.length) : null
43269
+ };
43270
+ }
43271
+ function buildComparison(summaries) {
43272
+ const ranking = [...summaries].sort((a, b) => (b.averageJudgeScore ?? -1) - (a.averageJudgeScore ?? -1) || b.passed - a.passed).map((summary) => ({
43273
+ model: summary.model,
43274
+ averageJudgeScore: summary.averageJudgeScore,
43275
+ passed: summary.passed,
43276
+ total: summary.total
43277
+ }));
43278
+ const notes = ranking.length > 0 ? [`Top model so far: ${ranking[0].model} (${ranking[0].passed}/${ranking[0].total} passed).`] : [];
43279
+ return { ranking, notes };
43280
+ }
43281
+ function round2(value) {
43282
+ return Math.round(value * 100) / 100;
43283
+ }
43284
+ function defaultImageFixturePath() {
43285
+ const fixtureDir = join4(dirname2(fileURLToPath(import.meta.url)), "..", "fixtures");
43286
+ const fixturePath = join4(fixtureDir, "brandenburger-tor.png");
43287
+ if (existsSync6(fixturePath)) return fixturePath;
43288
+ const tempDir = mkdtempSync(join4(tmpdir(), "openmates-benchmark-"));
43289
+ const tempPath = join4(tempDir, "brandenburger-tor.svg");
43290
+ writeFileSync4(tempPath, FIXTURE_IMAGE_SVG, "utf-8");
43291
+ return tempPath;
43292
+ }
43293
+ function writeBenchmarkResult(result, flags, output) {
43294
+ const json = `${JSON.stringify(result, null, 2)}
43295
+ `;
43296
+ if (output) writeFileSync4(output, json, "utf-8");
43297
+ if (flags.json === true || output) {
43298
+ process.stdout.write(json);
43299
+ return;
43300
+ }
43301
+ console.log(`Benchmark ${result.status}: ${result.targetModels.join(", ")}`);
43302
+ console.log(`Run ID: ${result.runId}`);
43303
+ console.log(`Suites: ${result.suites.join(", ")}`);
43304
+ console.log(`Judge: ${result.judgeModel}`);
43305
+ console.log(`Estimated credits: ${result.estimatedCredits.totalCredits}`);
43306
+ console.log(`Spend credits: ${result.spendsCredits ? "yes" : "no"}`);
43307
+ if (result.status !== "planned") {
43308
+ console.log(`Passed: ${result.summary.passed}/${result.summary.completed} completed (${result.summary.skipped} skipped)`);
43309
+ for (const benchmarkCase of result.cases) {
43310
+ const mark = benchmarkCase.passed ? "PASS" : "FAIL";
43311
+ const judge = benchmarkCase.judge ? ` judge=${benchmarkCase.judge.score ?? "unparsed"}` : "";
43312
+ const error = benchmarkCase.error ? ` error=${benchmarkCase.error}` : "";
43313
+ console.log(`${mark} ${benchmarkCase.model} ${benchmarkCase.suite}/${benchmarkCase.id} (${benchmarkCase.durationMs}ms)${judge}${error}`);
43314
+ }
43315
+ }
43316
+ }
43317
+
41505
43318
  // src/cli.ts
41506
43319
  async function main() {
41507
43320
  const parsed = parseArgs(process.argv.slice(2));
@@ -41536,6 +43349,10 @@ async function main() {
41536
43349
  printSettingsHelp(client);
41537
43350
  return;
41538
43351
  }
43352
+ if (command === "learning-mode") {
43353
+ printLearningModeHelp();
43354
+ return;
43355
+ }
41539
43356
  if (command === "signup") {
41540
43357
  printSignupHelp();
41541
43358
  return;
@@ -41572,6 +43389,10 @@ async function main() {
41572
43389
  printDocsHelp();
41573
43390
  return;
41574
43391
  }
43392
+ if (command === "benchmark") {
43393
+ printBenchmarkHelp();
43394
+ return;
43395
+ }
41575
43396
  printHelp();
41576
43397
  return;
41577
43398
  }
@@ -41630,6 +43451,10 @@ async function main() {
41630
43451
  await handleSettings(client, subcommand, rest, parsed.flags);
41631
43452
  return;
41632
43453
  }
43454
+ if (command === "learning-mode") {
43455
+ await handleLearningMode(client, subcommand, parsed.flags);
43456
+ return;
43457
+ }
41633
43458
  if (command === "inspirations") {
41634
43459
  await handleInspirations(client, parsed.flags);
41635
43460
  return;
@@ -41642,10 +43467,22 @@ async function main() {
41642
43467
  handleFeedback(subcommand, rest, parsed.flags);
41643
43468
  return;
41644
43469
  }
43470
+ if (command === "benchmark") {
43471
+ await handleBenchmark(client, subcommand, rest, parsed.flags);
43472
+ return;
43473
+ }
41645
43474
  throw new Error(`Unknown command '${command}'. Run 'openmates help'.`);
41646
43475
  }
41647
43476
  function shouldInitializeRedactor(command, subcommand) {
41648
- return command === "chats" && ["new", "send", "incognito"].includes(subcommand ?? "");
43477
+ return command === "chats" && ["new", "send", "answer-interactive", "incognito"].includes(subcommand ?? "");
43478
+ }
43479
+ function parseJsonFlag(value, flagName) {
43480
+ try {
43481
+ return JSON.parse(value);
43482
+ } catch (error) {
43483
+ const message = error instanceof Error ? error.message : String(error);
43484
+ throw new Error(`Invalid JSON for ${flagName}: ${message}`);
43485
+ }
41649
43486
  }
41650
43487
  async function handleChats(client, subcommand, rest, flags, redactor) {
41651
43488
  if (!subcommand || subcommand === "help" || flags.help === true) {
@@ -41698,7 +43535,8 @@ async function handleChats(client, subcommand, rest, flags, redactor) {
41698
43535
  json: flags.json === true,
41699
43536
  autoApproveSubChats: flags["auto-approve"] === true,
41700
43537
  autoApproveMemories: flags["auto-approve-memories"] === true,
41701
- piiDetection: flags["no-pii-detection"] !== true
43538
+ piiDetection: flags["no-pii-detection"] !== true,
43539
+ anonymousLearningMode: client.hasSession() ? void 0 : parseAnonymousLearningModeFlags(flags)
41702
43540
  },
41703
43541
  redactor
41704
43542
  );
@@ -41779,6 +43617,34 @@ Run 'openmates chats show ` + chatId + "' to check if suggestions have been save
41779
43617
  if (flags.json === true) printJson2(result);
41780
43618
  return;
41781
43619
  }
43620
+ if (subcommand === "answer-interactive") {
43621
+ const chatId = typeof flags.chat === "string" ? flags.chat : void 0;
43622
+ const questionJson = typeof flags["question-json"] === "string" ? flags["question-json"] : void 0;
43623
+ const answerJson = typeof flags["answer-json"] === "string" ? flags["answer-json"] : void 0;
43624
+ if (!chatId || !questionJson || !answerJson) {
43625
+ throw new Error(
43626
+ "Missing interactive answer data. Usage: openmates chats answer-interactive --chat <id> --question-json '<json>' --answer-json '<json>'"
43627
+ );
43628
+ }
43629
+ const question = parseJsonFlag(questionJson, "--question-json");
43630
+ const answer = parseJsonFlag(answerJson, "--answer-json");
43631
+ const formatted = formatInteractiveQuestionAnswer(question, answer);
43632
+ const result = await sendMessageStreaming(
43633
+ client,
43634
+ {
43635
+ message: formatted.messageContent,
43636
+ chatId,
43637
+ incognito: false,
43638
+ json: flags.json === true,
43639
+ autoApproveSubChats: flags["auto-approve"] === true,
43640
+ autoApproveMemories: flags["auto-approve-memories"] === true,
43641
+ piiDetection: flags["no-pii-detection"] !== true
43642
+ },
43643
+ redactor
43644
+ );
43645
+ if (flags.json === true) printJson2(result);
43646
+ return;
43647
+ }
41782
43648
  if (subcommand === "incognito") {
41783
43649
  const message = rest.join(" ").trim();
41784
43650
  if (!message)
@@ -41878,10 +43744,10 @@ Run 'openmates chats show ` + chatId + "' to check if suggestions have been save
41878
43744
  input: process.stdin,
41879
43745
  output: process.stdout
41880
43746
  });
41881
- const answer = await new Promise((resolve5) => {
43747
+ const answer = await new Promise((resolve6) => {
41882
43748
  iface.question(
41883
43749
  `Delete ${resolved.length} chat(s)? This cannot be undone. [y/N] `,
41884
- resolve5
43750
+ resolve6
41885
43751
  );
41886
43752
  });
41887
43753
  iface.close();
@@ -42041,16 +43907,16 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
42041
43907
  }
42042
43908
  }
42043
43909
  const { mkdir, writeFile } = await import("fs/promises");
42044
- const { join: join4 } = await import("path");
43910
+ const { join: join5 } = await import("path");
42045
43911
  if (useZip) {
42046
- const tmpDir = join4(outputDir, `.${filenameBase}_tmp`);
43912
+ const tmpDir = join5(outputDir, `.${filenameBase}_tmp`);
42047
43913
  await mkdir(tmpDir, { recursive: true });
42048
- await writeFile(join4(tmpDir, `${filenameBase}.yml`), yamlContent);
42049
- await writeFile(join4(tmpDir, `${filenameBase}.md`), mdContent);
43914
+ await writeFile(join5(tmpDir, `${filenameBase}.yml`), yamlContent);
43915
+ await writeFile(join5(tmpDir, `${filenameBase}.md`), mdContent);
42050
43916
  if (codeEmbeds.length > 0) {
42051
43917
  for (const ce of codeEmbeds) {
42052
43918
  const fpath = ce.filePath ?? ce.filename ?? `${ce.embedId.slice(0, 8)}.${getExtForLang(ce.language)}`;
42053
- const fullPath = join4(tmpDir, "code", fpath);
43919
+ const fullPath = join5(tmpDir, "code", fpath);
42054
43920
  await mkdir(fullPath.substring(0, fullPath.lastIndexOf("/")), {
42055
43921
  recursive: true
42056
43922
  });
@@ -42058,13 +43924,13 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
42058
43924
  }
42059
43925
  }
42060
43926
  if (transcriptEmbeds.length > 0) {
42061
- const tDir = join4(tmpDir, "transcripts");
43927
+ const tDir = join5(tmpDir, "transcripts");
42062
43928
  await mkdir(tDir, { recursive: true });
42063
43929
  for (const te of transcriptEmbeds) {
42064
- await writeFile(join4(tDir, te.filename), te.content);
43930
+ await writeFile(join5(tDir, te.filename), te.content);
42065
43931
  }
42066
43932
  }
42067
- const zipPath = join4(outputDir, `${filenameBase}.zip`);
43933
+ const zipPath = join5(outputDir, `${filenameBase}.zip`);
42068
43934
  const { execSync: execSync2 } = await import("child_process");
42069
43935
  try {
42070
43936
  execSync2(`cd "${tmpDir}" && zip -r "${zipPath}" .`, { stdio: "pipe" });
@@ -42079,17 +43945,17 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
42079
43945
  );
42080
43946
  }
42081
43947
  } else {
42082
- const chatDir = join4(outputDir, filenameBase);
43948
+ const chatDir = join5(outputDir, filenameBase);
42083
43949
  await mkdir(chatDir, { recursive: true });
42084
43950
  const written = [];
42085
- await writeFile(join4(chatDir, `${filenameBase}.yml`), yamlContent);
43951
+ await writeFile(join5(chatDir, `${filenameBase}.yml`), yamlContent);
42086
43952
  written.push(`${filenameBase}.yml`);
42087
- await writeFile(join4(chatDir, `${filenameBase}.md`), mdContent);
43953
+ await writeFile(join5(chatDir, `${filenameBase}.md`), mdContent);
42088
43954
  written.push(`${filenameBase}.md`);
42089
43955
  if (codeEmbeds.length > 0) {
42090
43956
  for (const ce of codeEmbeds) {
42091
43957
  const fpath = ce.filePath ?? ce.filename ?? `${ce.embedId.slice(0, 8)}.${getExtForLang(ce.language)}`;
42092
- const fullPath = join4(chatDir, "code", fpath);
43958
+ const fullPath = join5(chatDir, "code", fpath);
42093
43959
  await mkdir(fullPath.substring(0, fullPath.lastIndexOf("/")), {
42094
43960
  recursive: true
42095
43961
  });
@@ -42098,10 +43964,10 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
42098
43964
  }
42099
43965
  }
42100
43966
  if (transcriptEmbeds.length > 0) {
42101
- const tDir = join4(chatDir, "transcripts");
43967
+ const tDir = join5(chatDir, "transcripts");
42102
43968
  await mkdir(tDir, { recursive: true });
42103
43969
  for (const te of transcriptEmbeds) {
42104
- await writeFile(join4(tDir, te.filename), te.content);
43970
+ await writeFile(join5(tDir, te.filename), te.content);
42105
43971
  written.push(`transcripts/${te.filename}`);
42106
43972
  }
42107
43973
  }
@@ -42137,7 +44003,7 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
42137
44003
  printJson2({
42138
44004
  chat_id: chat.id,
42139
44005
  title: chat.title,
42140
- output_dir: useZip ? join4(outputDir, `${filenameBase}.zip`) : join4(outputDir, filenameBase),
44006
+ output_dir: useZip ? join5(outputDir, `${filenameBase}.zip`) : join5(outputDir, filenameBase),
42141
44007
  files,
42142
44008
  code_embeds: codeEmbeds.length,
42143
44009
  transcript_embeds: transcriptEmbeds.length
@@ -42658,7 +44524,7 @@ async function handleCodeRun(client, flags, apiKey) {
42658
44524
  }
42659
44525
  }
42660
44526
  async function streamCodeRunToTerminal(url, jsonMode) {
42661
- return await new Promise((resolve5, reject) => {
44527
+ return await new Promise((resolve6, reject) => {
42662
44528
  const ws = new WebSocket2(url);
42663
44529
  let lastStatus = {};
42664
44530
  ws.on("message", (data) => {
@@ -42677,7 +44543,7 @@ async function streamCodeRunToTerminal(url, jsonMode) {
42677
44543
  const status = String(payload.status ?? "");
42678
44544
  if (["finished", "failed", "timeout", "cancelled"].includes(status)) {
42679
44545
  ws.close();
42680
- resolve5(lastStatus);
44546
+ resolve6(lastStatus);
42681
44547
  }
42682
44548
  }
42683
44549
  } catch (err) {
@@ -42687,7 +44553,7 @@ async function streamCodeRunToTerminal(url, jsonMode) {
42687
44553
  });
42688
44554
  ws.on("error", () => reject(new Error("Code Run stream failed.")));
42689
44555
  ws.on("close", () => {
42690
- if (Object.keys(lastStatus).length > 0) resolve5(lastStatus);
44556
+ if (Object.keys(lastStatus).length > 0) resolve6(lastStatus);
42691
44557
  });
42692
44558
  });
42693
44559
  }
@@ -42698,7 +44564,7 @@ async function pollCodeRunStatus(client, statusPath, apiKey, jsonMode) {
42698
44564
  if (!jsonMode && value) process.stderr.write(`Code Run status: ${value}
42699
44565
  `);
42700
44566
  if (["finished", "failed", "timeout", "cancelled"].includes(value)) return status;
42701
- await new Promise((resolve5) => setTimeout(resolve5, 1e3));
44567
+ await new Promise((resolve6) => setTimeout(resolve6, 1e3));
42702
44568
  }
42703
44569
  }
42704
44570
  function buildSkillInput(flags, inlineTokens, schemaParams) {
@@ -42898,7 +44764,7 @@ async function handleEmbeds(client, subcommand, rest, flags) {
42898
44764
  throw new Error("Embed version content was not available after local reconstruction.");
42899
44765
  }
42900
44766
  if (typeof flags.output === "string") {
42901
- writeFileSync4(flags.output, result.content, "utf-8");
44767
+ writeFileSync5(flags.output, result.content, "utf-8");
42902
44768
  if (flags.json === true) {
42903
44769
  printJson2({ ...result, output: flags.output });
42904
44770
  } else {
@@ -43039,6 +44905,25 @@ async function printSettingsMutationResult(resultPromise, flags) {
43039
44905
  process.stdout.write("\x1B[32m\u2713\x1B[0m Settings updated\n");
43040
44906
  if (result && typeof result === "object") printGenericObject(result);
43041
44907
  }
44908
+ function printReportIssueCreateResult(result, flags) {
44909
+ if (flags.json === true) {
44910
+ printJson2(result);
44911
+ return;
44912
+ }
44913
+ process.stdout.write("\x1B[32m\u2713\x1B[0m Issue reported\n");
44914
+ const obj = result && typeof result === "object" ? result : {};
44915
+ const issueId = typeof obj.issue_id === "string" ? obj.issue_id : "";
44916
+ const shortIssueId = typeof obj.short_issue_id === "string" ? obj.short_issue_id : "";
44917
+ if (shortIssueId || issueId) {
44918
+ console.log(`Issue reference: ${shortIssueId || issueId}`);
44919
+ }
44920
+ if (issueId && shortIssueId && issueId !== shortIssueId) {
44921
+ console.log(`Internal issue ID: ${issueId}`);
44922
+ }
44923
+ if (typeof obj.message === "string") {
44924
+ console.log(obj.message);
44925
+ }
44926
+ }
43042
44927
  function addQueryParam(params, key, value) {
43043
44928
  if (typeof value === "string" && value.length > 0) params.set(key, value);
43044
44929
  }
@@ -43182,11 +45067,11 @@ function parseYamlScalar(value) {
43182
45067
  }
43183
45068
  async function saveDownloadedDocument(document, output) {
43184
45069
  const { mkdir, writeFile } = await import("fs/promises");
43185
- const { join: join4, basename: basename4, dirname: dirname2 } = await import("path");
45070
+ const { join: join5, basename: basename4, dirname: dirname4 } = await import("path");
43186
45071
  const target = typeof output === "string" ? output : ".";
43187
45072
  const filename = basename4(document.filename || "document.pdf");
43188
- const filePath = target.endsWith(".pdf") ? target : join4(target, filename);
43189
- await mkdir(dirname2(filePath), { recursive: true });
45073
+ const filePath = target.endsWith(".pdf") ? target : join5(target, filename);
45074
+ await mkdir(dirname4(filePath), { recursive: true });
43190
45075
  await writeFile(filePath, document.data);
43191
45076
  return filePath;
43192
45077
  }
@@ -43214,7 +45099,7 @@ function printMateInfo(mateId, json) {
43214
45099
  async function confirmOrExit(question) {
43215
45100
  const rl = await import("readline");
43216
45101
  const iface = rl.createInterface({ input: process.stdin, output: process.stdout });
43217
- const answer = await new Promise((resolve5) => iface.question(question, resolve5));
45102
+ const answer = await new Promise((resolve6) => iface.question(question, resolve6));
43218
45103
  iface.close();
43219
45104
  if (answer.trim().toLowerCase() !== "y") {
43220
45105
  console.log("Aborted.");
@@ -43224,7 +45109,7 @@ async function confirmOrExit(question) {
43224
45109
  async function promptLine(question) {
43225
45110
  const rl = await import("readline");
43226
45111
  const iface = rl.createInterface({ input: process.stdin, output: process.stdout });
43227
- const answer = await new Promise((resolve5) => iface.question(question, resolve5));
45112
+ const answer = await new Promise((resolve6) => iface.question(question, resolve6));
43228
45113
  iface.close();
43229
45114
  return answer.trim();
43230
45115
  }
@@ -43232,7 +45117,7 @@ async function promptSecret(question) {
43232
45117
  if (!process.stdin.isTTY) {
43233
45118
  return promptLine(question);
43234
45119
  }
43235
- return new Promise((resolve5) => {
45120
+ return new Promise((resolve6) => {
43236
45121
  const stdin2 = process.stdin;
43237
45122
  const wasRaw = stdin2.isRaw;
43238
45123
  let value = "";
@@ -43245,7 +45130,7 @@ async function promptSecret(question) {
43245
45130
  stdin2.off("data", onData);
43246
45131
  stdin2.setRawMode(wasRaw);
43247
45132
  process.stdout.write("\n");
43248
- resolve5(value);
45133
+ resolve6(value);
43249
45134
  return;
43250
45135
  }
43251
45136
  if (char === "") {
@@ -43265,7 +45150,7 @@ async function promptSecret(question) {
43265
45150
  }
43266
45151
  async function writeSecretFile(filePath, content, force = false) {
43267
45152
  const { mkdir, writeFile, stat: stat2 } = await import("fs/promises");
43268
- const { dirname: dirname2 } = await import("path");
45153
+ const { dirname: dirname4 } = await import("path");
43269
45154
  try {
43270
45155
  await stat2(filePath);
43271
45156
  if (!force) throw new Error(`${filePath} already exists. Use --force to overwrite.`);
@@ -43275,7 +45160,7 @@ async function writeSecretFile(filePath, content, force = false) {
43275
45160
  }
43276
45161
  if (error instanceof Error && !("code" in error)) throw error;
43277
45162
  }
43278
- await mkdir(dirname2(filePath), { recursive: true });
45163
+ await mkdir(dirname4(filePath), { recursive: true });
43279
45164
  await writeFile(filePath, content, { mode: 384 });
43280
45165
  return filePath;
43281
45166
  }
@@ -43902,7 +45787,8 @@ async function handleSettings(client, subcommand, rest, flags) {
43902
45787
  const title = typeof flags.title === "string" ? flags.title : void 0;
43903
45788
  const body = typeof flags.body === "string" ? flags.body : void 0;
43904
45789
  if (!title || !body) throw new Error("Provide --title and --body.");
43905
- await printSettingsMutationResult(client.settingsPost("issues", { title, description: body }), flags);
45790
+ const result = await client.settingsPost("issues", { title, description: body });
45791
+ printReportIssueCreateResult(result, flags);
43906
45792
  return;
43907
45793
  }
43908
45794
  if (matches(tokens, ["report-issue", "status"])) {
@@ -43973,6 +45859,79 @@ async function handleSettings(client, subcommand, rest, flags) {
43973
45859
  printSettingsHelp(client, [subcommand]);
43974
45860
  process.exit(1);
43975
45861
  }
45862
+ var LEARNING_MODE_AGE_GROUPS = /* @__PURE__ */ new Set([
45863
+ "under_10",
45864
+ "10_12",
45865
+ "13_15",
45866
+ "16_18",
45867
+ "adult"
45868
+ ]);
45869
+ async function handleLearningMode(client, subcommand, flags) {
45870
+ if (!subcommand || subcommand === "help" || flags.help === true) {
45871
+ printLearningModeHelp();
45872
+ return;
45873
+ }
45874
+ if (subcommand === "status") {
45875
+ printLearningModeStatus(await client.getLearningModeStatus(), flags.json === true);
45876
+ return;
45877
+ }
45878
+ if (subcommand === "enable") {
45879
+ const ageGroup = parseLearningModeAgeGroup(flags["age-group"]);
45880
+ const passcode = parseRequiredStringFlag(flags.passcode, "--passcode");
45881
+ printLearningModeStatus(
45882
+ await client.activateLearningMode({ ageGroup, passcode }),
45883
+ flags.json === true
45884
+ );
45885
+ return;
45886
+ }
45887
+ if (subcommand === "disable") {
45888
+ const passcode = parseRequiredStringFlag(flags.passcode, "--passcode");
45889
+ printLearningModeStatus(await client.deactivateLearningMode(passcode), flags.json === true);
45890
+ return;
45891
+ }
45892
+ console.error(`Unknown learning-mode command '${subcommand}'.
45893
+ `);
45894
+ printLearningModeHelp();
45895
+ process.exit(1);
45896
+ }
45897
+ function parseLearningModeAgeGroup(value) {
45898
+ if (typeof value !== "string" || !LEARNING_MODE_AGE_GROUPS.has(value)) {
45899
+ throw new Error("Provide --age-group as one of: under_10, 10_12, 13_15, 16_18, adult.");
45900
+ }
45901
+ return value;
45902
+ }
45903
+ function parseAnonymousLearningModeFlags(flags) {
45904
+ if (flags["learning-mode"] !== true) return void 0;
45905
+ return {
45906
+ enabled: true,
45907
+ ageGroup: parseLearningModeAgeGroup(flags["age-group"]),
45908
+ source: "anonymous_session"
45909
+ };
45910
+ }
45911
+ function parseRequiredStringFlag(value, name) {
45912
+ if (typeof value !== "string" || value.length === 0) {
45913
+ throw new Error(`Provide ${name}.`);
45914
+ }
45915
+ return value;
45916
+ }
45917
+ function printLearningModeStatus(status, json) {
45918
+ if (json) {
45919
+ printJson2(status);
45920
+ return;
45921
+ }
45922
+ console.log(`Learning Mode: ${status.enabled ? "enabled" : "disabled"}`);
45923
+ if (status.age_group) console.log(`Age group: ${status.age_group}`);
45924
+ if (status.failed_attempts > 0) console.log(`Failed disable attempts: ${status.failed_attempts}`);
45925
+ if (status.deactivation_blocked_until) {
45926
+ console.log(`Disable blocked until: ${new Date(status.deactivation_blocked_until * 1e3).toISOString()}`);
45927
+ }
45928
+ }
45929
+ function learningModeStatusToContext(status) {
45930
+ return {
45931
+ enabled: status.enabled,
45932
+ ageGroup: status.age_group
45933
+ };
45934
+ }
43976
45935
  async function handleMemories(client, rest, flags) {
43977
45936
  const action = rest[0];
43978
45937
  if (!action || action === "help") {
@@ -44563,7 +46522,10 @@ async function sendMessageStreaming(client, params, redactor) {
44563
46522
  if (!client.hasSession()) {
44564
46523
  let result2;
44565
46524
  try {
44566
- result2 = await client.sendAnonymousMessage({ message: finalMessage });
46525
+ result2 = await client.sendAnonymousMessage({
46526
+ message: finalMessage,
46527
+ learningMode: params.anonymousLearningMode
46528
+ });
44567
46529
  } finally {
44568
46530
  clearTyping();
44569
46531
  }
@@ -44584,6 +46546,7 @@ async function sendMessageStreaming(client, params, redactor) {
44584
46546
  const urlResult = prepareUrlEmbeds(finalMessage);
44585
46547
  finalMessage = urlResult.message;
44586
46548
  preparedEmbeds.push(...urlResult.embeds);
46549
+ const learningMode = learningModeStatusToContext(await client.getLearningModeStatus());
44587
46550
  const result = await client.sendMessage({
44588
46551
  message: finalMessage,
44589
46552
  chatId: params.chatId,
@@ -44593,6 +46556,7 @@ async function sendMessageStreaming(client, params, redactor) {
44593
46556
  onSubChatApprovalRequest,
44594
46557
  autoApproveSubChats: params.autoApproveSubChats,
44595
46558
  autoApproveMemories: params.autoApproveMemories,
46559
+ learningMode,
44596
46560
  preparedEmbeds: preparedEmbeds.length > 0 ? preparedEmbeds : void 0,
44597
46561
  piiMappings: piiResult.mappings.map((mapping) => ({
44598
46562
  placeholder: mapping.placeholder,
@@ -45899,9 +47863,11 @@ Commands:
45899
47863
  openmates mentions [--help] List available @mentions
45900
47864
  openmates embeds [--help] Embed commands (show)
45901
47865
  openmates settings [--help] Predefined settings commands
47866
+ openmates learning-mode [--help] Account-wide Learning Mode controls
45902
47867
  openmates inspirations [--lang <code>] [--json] Daily inspirations
45903
47868
  openmates newchatsuggestions [--limit <n>] [--json] Personalized new chat suggestions
45904
47869
  openmates feedback [--help] Assistant response feedback helpers
47870
+ openmates benchmark [--help] Run real model benchmarks with usage tagged as benchmark spend
45905
47871
  openmates server [--help] Server management (install, start, stop, ...)
45906
47872
  openmates docs [--help] Browse, search, and download documentation
45907
47873
  openmates e2e provision-auth-accounts Provision local E2E auth-account artifacts
@@ -45924,6 +47890,22 @@ Options:
45924
47890
  --rating <1-5> Required star rating
45925
47891
  --json Output the decision contract as JSON`);
45926
47892
  }
47893
+ function printLearningModeHelp() {
47894
+ console.log(`Learning Mode commands:
47895
+ openmates learning-mode status [--json]
47896
+ openmates learning-mode enable --age-group <group> --passcode <passcode> [--json]
47897
+ openmates learning-mode disable --passcode <passcode> [--json]
47898
+
47899
+ Learning Mode is account-wide and applies to CLI, web, Apple, and API chat requests.
47900
+
47901
+ Age groups:
47902
+ under_10, 10_12, 13_15, 16_18, adult
47903
+
47904
+ Options:
47905
+ --age-group <group> Required for enable
47906
+ --passcode <value> Required for enable and disable
47907
+ --json Output backend status JSON`);
47908
+ }
45927
47909
  function printSignupHelp() {
45928
47910
  console.log(`Signup command:
45929
47911
  openmates signup --email <email> --username <name> --invite-code <code>
@@ -45964,9 +47946,10 @@ function printChatsHelp() {
45964
47946
  openmates chats show <chat-id> [--raw] [--json]
45965
47947
  openmates chats open [<n|example-id|slug>] [--json]
45966
47948
  openmates chats search <query> [--json]
45967
- openmates chats new <message> [--json] [--auto-approve] [--auto-approve-memories] [--no-pii-detection]
47949
+ openmates chats new <message> [--json] [--learning-mode --age-group <group>] [--auto-approve] [--auto-approve-memories] [--no-pii-detection]
45968
47950
  openmates chats send [--chat <id>] [--incognito] <message> [--json] [--auto-approve] [--auto-approve-memories] [--no-pii-detection]
45969
47951
  openmates chats send --chat <id> --followup <n> [--json] [--auto-approve] [--auto-approve-memories]
47952
+ openmates chats answer-interactive --chat <id> --question-json '<json>' --answer-json '<json>' [--json]
45970
47953
  openmates chats download <chat-id> [--output <path>] [--zip] [--json]
45971
47954
  openmates chats delete <id1> [id2] [id3] ... [--yes]
45972
47955
  openmates chats share [<chat-id>] [--expires <seconds>] [--password <pwd>] [--json]
@@ -45995,6 +47978,11 @@ Options for 'send':
45995
47978
  typing the full message (requires --chat)
45996
47979
  --incognito Send without saving to chat history
45997
47980
 
47981
+ Options for 'answer-interactive':
47982
+ --chat <id> Chat containing the interactive question
47983
+ --question-json The question payload returned by 'chats send --json'
47984
+ --answer-json Structured answer JSON, for example '{"selection":["opt_a"]}'
47985
+
45998
47986
  Options for 'new', 'send', and 'incognito':
45999
47987
  --auto-approve Automatically approve server-requested sub-chat batches.
46000
47988
  Without this, the CLI prompts in the terminal like the web app.
@@ -46004,6 +47992,11 @@ Options for 'new', 'send', and 'incognito':
46004
47992
  --no-pii-detection Send the message exactly as typed. By default, the CLI
46005
47993
  replaces detected PII with placeholders before send.
46006
47994
 
47995
+ Guest-only options for logged-out 'new':
47996
+ --learning-mode Opt anonymous chat into request-scoped Learning Mode.
47997
+ --age-group <group> Required with --learning-mode: under_10, 10_12,
47998
+ 13_15, 16_18, or adult.
47999
+
46007
48000
  Options for 'download':
46008
48001
  --output <path> Target directory (default: current directory)
46009
48002
  --zip Create a .zip archive instead of a folder
@@ -46040,6 +48033,7 @@ Examples:
46040
48033
  openmates chats show "Flight Connections Berlin to Bangkok"
46041
48034
  openmates chats search "Madrid"
46042
48035
  openmates chats new "Hello, what can you help me with?"
48036
+ openmates chats new "Help me understand fractions" --learning-mode --age-group 10_12
46043
48037
  openmates chats send --chat d262cb68 "follow-up question"
46044
48038
  openmates chats send --chat d262cb68 --followup 1
46045
48039
  openmates chats send --chat d262cb68 --followup 3
@@ -46232,7 +48226,7 @@ async function handleDocs(client, subcommand, rest, flags) {
46232
48226
  }
46233
48227
  if (subcommand === "download") {
46234
48228
  const { writeFile, mkdir } = await import("fs/promises");
46235
- const { join: join4, dirname: dirname2 } = await import("path");
48229
+ const { join: join5, dirname: dirname4 } = await import("path");
46236
48230
  if (flags.all === true) {
46237
48231
  const outputDir = typeof flags.output === "string" ? flags.output : "./openmates-docs";
46238
48232
  const tree = await client.listDocs();
@@ -46241,8 +48235,8 @@ async function handleDocs(client, subcommand, rest, flags) {
46241
48235
  let count = 0;
46242
48236
  for (const slug2 of slugs) {
46243
48237
  const content2 = await client.getDoc(slug2);
46244
- const filePath = join4(outputDir, `${slug2}.md`);
46245
- await mkdir(dirname2(filePath), { recursive: true });
48238
+ const filePath = join5(outputDir, `${slug2}.md`);
48239
+ await mkdir(dirname4(filePath), { recursive: true });
46246
48240
  await writeFile(filePath, content2, "utf-8");
46247
48241
  count++;
46248
48242
  process.stderr.write(`\r Downloaded ${count}/${slugs.length}`);
@@ -46314,8 +48308,8 @@ function isCliEntrypoint() {
46314
48308
  if (!entrypoint) return false;
46315
48309
  try {
46316
48310
  const invokedPath = realpathSync(entrypoint);
46317
- const modulePath = realpathSync(fileURLToPath(import.meta.url));
46318
- return invokedPath === modulePath || basename3(invokedPath) === "cli.js" && dirname(invokedPath) === dirname(modulePath);
48311
+ const modulePath = realpathSync(fileURLToPath2(import.meta.url));
48312
+ return invokedPath === modulePath || basename3(invokedPath) === "cli.js" && dirname3(invokedPath) === dirname3(modulePath);
46319
48313
  } catch {
46320
48314
  return false;
46321
48315
  }