openmates 0.12.0-alpha.10 → 0.12.0-alpha.11

package/dist/{chunk-D7RIGVLZ.js → chunk-2QG4XPEB.js}

@@ -3677,6 +3677,19 @@ var OpenMatesClient = class _OpenMatesClient {
     if (connectedAccountTokenRefs.length > 0) {
       messagePayload.connected_account_token_refs = connectedAccountTokenRefs;
     }
+    if (params.benchmarkMetadata) {
+      messagePayload.benchmark_metadata = params.benchmarkMetadata;
+    }
+    if (params.incognito) {
+      messagePayload.message_history = [{
+        message_id: messageId,
+        chat_id: chatId,
+        role: "user",
+        sender_name: "User",
+        content: params.message,
+        created_at: createdAt
+      }];
+    }
     let chatKeyBytes = null;
     let encryptedChatKey = null;
     let baselineMessagesV = 0;
@@ -3735,6 +3748,7 @@ var OpenMatesClient = class _OpenMatesClient {
     if (encryptedEmbeds.length > 0) {
       messagePayload.encrypted_embeds = encryptedEmbeds;
     }
+    const precollectedResponse = params.precollectResponse ? ws.collectAiResponse(messageId, chatId, { onStream: params.onStream }) : null;
     const confirmed = ws.waitForMessage(
       "chat_message_confirmed",
       (payload) => {
@@ -3949,7 +3963,7 @@ var OpenMatesClient = class _OpenMatesClient {
     };
     if (params.incognito) {
       try {
-        const resp = await ws.collectAiResponse(messageId, chatId, streamOpts);
+        const resp = await (precollectedResponse ?? ws.collectAiResponse(messageId, chatId, streamOpts));
         assistantMessageId = resp.messageId;
         assistant = resp.content;
         category = resp.category;
@@ -6039,7 +6053,7 @@ function printLogo() {
 
 // src/cli.ts
 import { createInterface as createInterface3 } from "readline/promises";
-import { realpathSync, writeFileSync as writeFileSync4 } from "fs";
+import { realpathSync, writeFileSync as writeFileSync5 } from "fs";
 import { fileURLToPath } from "url";
 import { basename as basename3, dirname } from "path";
 import WebSocket2 from "ws";
@@ -27900,6 +27914,12 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
     account_created: {
       text: "Account created"
     },
+    account_created_second_login_title: {
+      text: "Add a second login method"
+    },
+    account_created_second_login_info: {
+      text: "If you signed up with a passkey, add password plus 2FA as a backup. If you signed up with password plus 2FA, add a passkey for faster secure login."
+    },
     password_security_reminder: {
       subject: {
         text: "Action needed to secure your OpenMates account"
@@ -28129,10 +28149,7 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
       text: "Welcome to OpenMates!"
     },
     complete_signup_info: {
-      text: "Once you completed the signup process by purchasing usage credits or redeeming a gift card, you can start using OpenMates!"
-    },
-    auto_delete_warning: {
-      text: "Please note: Accounts that haven't completed the signup process will be automatically deleted after 7 days."
+      text: "Your account is ready. Here are a few helpful next steps to protect your access and keep a copy of your data."
     },
     want_to_delete_account: {
       text: "Want to delete your account?"
@@ -30004,6 +30021,15 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
     anonymous_terms_reminder: {
       text: "By sending a message you accept the terms & privacy policy of OpenMates."
     },
+    anonymous_terms_reminder_prefix: {
+      text: "By sending a message you accept the "
+    },
+    anonymous_terms_reminder_connector: {
+      text: " & "
+    },
+    anonymous_terms_reminder_suffix: {
+      text: " of OpenMates."
+    },
     send: {
       text: "Send"
     },
@@ -33661,10 +33687,10 @@ As of mid-2026, the severe supply shocks from the 2024\u20132025 avian flu have
           text: "Account: Email address, username/display name, profile image, locale, and security settings (e.g., 2FA enabled). Email and username are encrypted with your key before storage. We also keep a separate server-side Vault-encrypted copy of your verified email address for mandatory account lifecycle notices, such as account verification, security alerts, and deletion reminders. Passwords are stored as salted hashes."
         },
         usage: {
-          text: "Usage: Server logs, event timestamps, feature usage, error logs, and device recognition identifiers (hashed). IP addresses may be temporarily processed for security and rate limiting."
+          text: "Usage: Server logs, event timestamps, feature usage, error logs, and device recognition identifiers (hashed). IP addresses may be temporarily processed for security, rate limiting, and anonymous free-usage budget checks. Anonymous free usage uses a first-party random local identifier and sends only a server-side HMAC-hashed form for per-identity abuse limits."
         },
         content: {
-          text: "Content: Chat messages, prompts, attachments, and uploaded images/videos necessary to deliver the service (subject to moderation where applicable)."
+          text: "Content: Chat messages, prompts, attachments, and uploaded images/videos necessary to deliver the service (subject to moderation where applicable). Anonymous free-usage chats stay local-only and encrypted with per-chat keys before signup; they are uploaded only if you sign up and promote them into account sync."
         },
         payments: {
           text: "Payments: Payment method tokens, transaction IDs, billing address and VAT information as required for invoicing (processed primarily by Stripe). We do not store full card numbers."
@@ -34202,7 +34228,7 @@ As of mid-2026, the severe supply shocks from the 2024\u20132025 avian flu have
           text: "Credits and Payments"
         },
         description: {
-          text: "OpenMates uses a credit-based payment system. You purchase credit packs which are consumed when using AI services. Credits do not expire and remain in your account until used. Payment processing is handled by Stripe - see our Privacy Policy for details about payment data handling."
+          text: "OpenMates uses a credit-based payment system. You purchase credit packs which are consumed when using AI services. Credits do not expire and remain in your account until used. Official-cloud anonymous free usage, when available, is a limited fair-use trial with shared daily/weekly caps and per-identity abuse limits; it is not an account credit balance and may be unavailable when the budget is exhausted. Payment processing is handled by Stripe - see our Privacy Policy for details about payment data handling."
         },
         refund: {
           text: "Refund Policy: You may request a refund for unused credits within 14 days after purchase. Your right of withdrawal expires once credits are used. See our signup process for the full refund consent details."
@@ -38936,6 +38962,9 @@ As of mid-2026, the severe supply shocks from the 2024\u20132025 avian flu have
         weekly_remaining: {
           text: "Weekly remaining"
         },
+        monthly_remaining: {
+          text: "Monthly remaining"
+        },
         reset_at: {
           text: "Daily reset"
         },
@@ -38968,6 +38997,9 @@ As of mid-2026, the severe supply shocks from the 2024\u20132025 avian flu have
         },
         validation_percent: {
           text: "Percent values must be between 0 and 100."
+        },
+        validation_per_identity_cap: {
+          text: "Per-identity daily cap must be at least 1 credit when the monthly budget is above 0."
         }
       },
       tests: {
@@ -41487,6 +41519,244 @@ function buildAssistantFeedbackDecision(rating) {
   };
 }
 
+// src/benchmark.ts
+import { randomUUID as randomUUID3 } from "crypto";
+import { writeFileSync as writeFileSync4 } from "fs";
+var DEFAULT_JUDGE_MODEL = "google/gemini-3-flash-preview";
+var BENCHMARK_CASES = [
+  {
+    id: "smoke-exact-token",
+    suite: "smoke",
+    prompt: "Reply with exactly this token and no extra text: BENCHMARK_SMOKE_OK",
+    expectedIncludes: "BENCHMARK_SMOKE_OK"
+  },
+  {
+    id: "arithmetic-direct",
+    suite: "tools",
+    prompt: "Compute 19 * 23. Reply with only the integer result.",
+    expectedIncludes: "437"
+  },
+  {
+    id: "quality-concise-explanation",
+    suite: "quality",
+    prompt: "In four concise sentences, explain why deterministic benchmarks still need human-readable evaluation notes.",
+    needsJudge: true
+  }
+];
+async function handleBenchmark(client, subcommand, rest, flags) {
+  if (!subcommand || subcommand === "help" || flags.help === true) {
+    printBenchmarkHelp();
+    return;
+  }
+  if (subcommand !== "model") {
+    throw new Error(`Unknown benchmark command '${subcommand}'. Run 'openmates benchmark --help'.`);
+  }
+  const targetModel = rest[0];
+  if (!targetModel) {
+    throw new Error("Missing target model. Usage: openmates benchmark model <provider/model> --confirm-spend-credits");
+  }
+  const judgeModel = typeof flags["judge-model"] === "string" ? flags["judge-model"] : DEFAULT_JUDGE_MODEL;
+  const suites = parseSuites(flags.suite);
+  const runs = parseRuns(flags.runs);
+  const dryRun = flags["dry-run"] === true;
+  const output = typeof flags.output === "string" ? flags.output : void 0;
+  const runId = typeof flags["run-id"] === "string" ? flags["run-id"] : randomUUID3();
+  if (!dryRun && flags["confirm-spend-credits"] !== true) {
+    throw new Error(
+      "Benchmark runs spend real credits from the logged-in account. Rerun with --confirm-spend-credits, or use --dry-run to preview the plan."
+    );
+  }
+  const cases = expandCases(suites, runs);
+  const baseResult = {
+    command: "benchmark model",
+    status: dryRun ? "planned" : "completed",
+    runId,
+    targetModel,
+    judgeModel,
+    suites,
+    runs,
+    spendsCredits: !dryRun,
+    cases: [],
+    summary: { total: cases.length, passed: 0, failed: 0 }
+  };
+  if (dryRun) {
+    writeBenchmarkResult(baseResult, flags, output);
+    return;
+  }
+  if (!client.hasSession()) {
+    throw new Error("Benchmark runs require login. Run 'openmates login' first.");
+  }
+  for (const benchmarkCase of cases) {
+    const startedAt = Date.now();
+    const targetResponse = await client.sendMessage({
+      message: `${modelMention(targetModel)} ${benchmarkCase.prompt}`,
+      incognito: true,
+      autoApproveSubChats: true,
+      benchmarkMetadata: benchmarkMetadata({
+        runId,
+        suite: benchmarkCase.suite,
+        caseId: benchmarkCase.id,
+        targetModel,
+        judgeModel
+      }),
+      precollectResponse: true
+    });
+    const caseResult = {
+      id: benchmarkCase.id,
+      suite: benchmarkCase.suite,
+      run: benchmarkCase.run,
+      prompt: benchmarkCase.prompt,
+      assistant: targetResponse.assistant,
+      modelName: targetResponse.modelName,
+      passed: benchmarkCase.expectedIncludes ? targetResponse.assistant.includes(benchmarkCase.expectedIncludes) : true,
+      durationMs: Date.now() - startedAt,
+      expectedIncludes: benchmarkCase.expectedIncludes
+    };
+    if (benchmarkCase.needsJudge) {
+      const judgeResponse = await client.sendMessage({
+        message: `${modelMention(judgeModel)} ${judgePrompt(benchmarkCase.prompt, targetResponse.assistant)}`,
+        incognito: true,
+        autoApproveSubChats: true,
+        benchmarkMetadata: benchmarkMetadata({
+          runId,
+          suite: benchmarkCase.suite,
+          caseId: `${benchmarkCase.id}:judge`,
+          targetModel,
+          judgeModel
+        }),
+        precollectResponse: true
+      });
+      const judgment = parseJudgment(judgeResponse.assistant);
+      caseResult.judge = {
+        model: judgeModel,
+        score: judgment.score,
+        reason: judgment.reason,
+        raw: judgeResponse.assistant
+      };
+      caseResult.passed = judgment.score !== null && judgment.score >= 4;
+    }
+    baseResult.cases.push(caseResult);
+  }
+  baseResult.summary.passed = baseResult.cases.filter((result) => result.passed).length;
+  baseResult.summary.failed = baseResult.cases.length - baseResult.summary.passed;
+  writeBenchmarkResult(baseResult, flags, output);
+}
+function printBenchmarkHelp() {
+  console.log(`Benchmark commands:
+  openmates benchmark model <provider/model> --confirm-spend-credits [--suite smoke|tools|quality|all] [--runs <n>] [--json]
+
+Runs real incognito chat requests through the OpenMates product path. Live runs
+spend the logged-in user's credits and usage entries are grouped as benchmark spend.
+
+Options:
+  --confirm-spend-credits       Required for live benchmark runs
+  --dry-run                     Preview the benchmark plan without login or spend
+  --suite <list>                Comma-separated suites: smoke, tools, quality, all (default: smoke)
+  --runs <n>                    Repeat each selected case (default: 1)
+  --judge-model <provider/model> Judge for quality cases (default: ${DEFAULT_JUDGE_MODEL})
+  --run-id <id>                 Reuse a benchmark run id for grouping
+  --output <path>               Save JSON result to a file
+  --json                        Print JSON result`);
+}
+function parseSuites(value) {
+  if (value === void 0 || value === false) return ["smoke"];
+  if (value === true) throw new Error("--suite requires a value");
+  const suites = value.split(",").map((suite) => suite.trim()).filter(Boolean);
+  if (suites.includes("all")) return ["smoke", "tools", "quality"];
+  const allowed = /* @__PURE__ */ new Set(["smoke", "tools", "quality"]);
+  const invalid = suites.filter((suite) => !allowed.has(suite));
+  if (invalid.length > 0 || suites.length === 0) {
+    throw new Error("Invalid --suite. Use smoke, tools, quality, or all.");
+  }
+  return [...new Set(suites)];
+}
+function parseRuns(value) {
+  if (value === void 0 || value === false) return 1;
+  if (value === true) throw new Error("--runs requires a value");
+  const parsed = Number.parseInt(value, 10);
+  if (!Number.isInteger(parsed) || parsed < 1 || parsed > 20) {
+    throw new Error("--runs must be an integer from 1 to 20");
+  }
+  return parsed;
+}
+function expandCases(suites, runs) {
+  const selected = BENCHMARK_CASES.filter((benchmarkCase) => suites.includes(benchmarkCase.suite));
+  const expanded = [];
+  for (let run = 1; run <= runs; run += 1) {
+    for (const benchmarkCase of selected) expanded.push({ ...benchmarkCase, run });
+  }
+  return expanded;
+}
+function modelMention(model) {
+  const separator = model.indexOf("/");
+  if (separator === -1) return `@ai-model:${model}`;
+  const provider = model.slice(0, separator);
+  const modelId = model.slice(separator + 1);
+  if (!provider || !modelId) return `@ai-model:${model}`;
+  return `@ai-model:${modelId}:${provider}`;
+}
+function benchmarkMetadata(params) {
+  return {
+    source: "benchmark",
+    benchmark_run_id: params.runId,
+    benchmark_suite: params.suite,
+    benchmark_case: params.caseId,
+    benchmark_target_model: params.targetModel,
+    benchmark_judge_model: params.judgeModel
+  };
+}
+function judgePrompt(prompt, answer) {
+  return [
+    "You are judging a model benchmark response.",
+    'Return only JSON with shape {"score": number, "reason": string}.',
+    "Score from 1 to 5 for correctness, clarity, and following instructions.",
+    `Benchmark prompt: ${JSON.stringify(prompt)}`,
+    `Candidate answer: ${JSON.stringify(answer)}`
+  ].join("\n");
+}
+function parseJudgment(answer) {
+  const jsonText = extractJsonObject(answer);
+  if (!jsonText) return { score: null, reason: null };
+  try {
+    const parsed = JSON.parse(jsonText);
+    const score = typeof parsed.score === "number" && Number.isFinite(parsed.score) ? parsed.score : null;
+    const reason = typeof parsed.reason === "string" ? parsed.reason : null;
+    return { score, reason };
+  } catch {
+    return { score: null, reason: null };
+  }
+}
+function extractJsonObject(text) {
+  const fenced = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
+  if (fenced) return fenced[1];
+  const start = text.indexOf("{");
+  const end = text.lastIndexOf("}");
+  if (start === -1 || end === -1 || end <= start) return null;
+  return text.slice(start, end + 1);
+}
+function writeBenchmarkResult(result, flags, output) {
+  const json = `${JSON.stringify(result, null, 2)}
+`;
+  if (output) writeFileSync4(output, json, "utf-8");
+  if (flags.json === true || output) {
+    process.stdout.write(json);
+    return;
+  }
+  console.log(`Benchmark ${result.status}: ${result.targetModel}`);
+  console.log(`Run ID: ${result.runId}`);
+  console.log(`Suites: ${result.suites.join(", ")}`);
+  console.log(`Judge: ${result.judgeModel}`);
+  console.log(`Spend credits: ${result.spendsCredits ? "yes" : "no"}`);
+  if (result.status === "completed") {
+    console.log(`Passed: ${result.summary.passed}/${result.summary.total}`);
+    for (const benchmarkCase of result.cases) {
+      const mark = benchmarkCase.passed ? "PASS" : "FAIL";
+      const judge = benchmarkCase.judge?.score !== void 0 ? ` judge=${benchmarkCase.judge.score ?? "unparsed"}` : "";
+      console.log(`${mark} ${benchmarkCase.suite}/${benchmarkCase.id} (${benchmarkCase.durationMs}ms)${judge}`);
+    }
+  }
+}
+
 // src/cli.ts
 async function main() {
   const parsed = parseArgs(process.argv.slice(2));
@@ -41557,6 +41827,10 @@ async function main() {
       printDocsHelp();
       return;
     }
+    if (command === "benchmark") {
+      printBenchmarkHelp();
+      return;
+    }
     printHelp();
     return;
   }
@@ -41627,6 +41901,10 @@ async function main() {
     handleFeedback(subcommand, rest, parsed.flags);
     return;
   }
+  if (command === "benchmark") {
+    await handleBenchmark(client, subcommand, rest, parsed.flags);
+    return;
+  }
   throw new Error(`Unknown command '${command}'. Run 'openmates help'.`);
 }
 function shouldInitializeRedactor(command, subcommand) {
@@ -42883,7 +43161,7 @@ async function handleEmbeds(client, subcommand, rest, flags) {
         throw new Error("Embed version content was not available after local reconstruction.");
       }
       if (typeof flags.output === "string") {
-        writeFileSync4(flags.output, result.content, "utf-8");
+        writeFileSync5(flags.output, result.content, "utf-8");
         if (flags.json === true) {
           printJson2({ ...result, output: flags.output });
         } else {
@@ -45887,6 +46165,7 @@ Commands:
   openmates inspirations [--lang <code>] [--json]   Daily inspirations
   openmates newchatsuggestions [--limit <n>] [--json]   Personalized new chat suggestions
   openmates feedback [--help]                Assistant response feedback helpers
+  openmates benchmark [--help]               Run real model benchmarks with usage tagged as benchmark spend
   openmates server [--help]                   Server management (install, start, stop, ...)
   openmates docs [--help]                     Browse, search, and download documentation
   openmates e2e provision-auth-accounts       Provision local E2E auth-account artifacts

package/dist/cli.js

@@ -2,7 +2,7 @@
 import {
   getExtForLang,
   serializeToYaml
-} from "./chunk-D7RIGVLZ.js";
+} from "./chunk-2QG4XPEB.js";
 import "./chunk-AXNRPVLE.js";
 export {
   getExtForLang,

package/dist/index.d.ts

@@ -298,6 +298,14 @@ interface ChatListPage {
     limit: number;
     hasMore: boolean;
 }
+interface BenchmarkMetadata {
+    source: "benchmark";
+    benchmark_run_id: string;
+    benchmark_suite: string;
+    benchmark_case: string;
+    benchmark_target_model: string;
+    benchmark_judge_model?: string;
+}
 /** Decrypted message for display */
 interface DecryptedMessage {
     id: string;
@@ -688,6 +696,10 @@ declare class OpenMatesClient {
         connectedAccountDirectory?: ConnectedAccountDirectoryEntry[];
         /** Refresh-token envelopes to convert into short-lived token refs before send. */
         connectedAccountTokenRefInputs?: ConnectedAccountTurnTokenRefInput[];
+        /** Non-sensitive CLI benchmark labels for usage-source grouping. */
+        benchmarkMetadata?: BenchmarkMetadata;
+        /** Start collecting before send for latency-sensitive benchmark turns. */
+        precollectResponse?: boolean;
     }): Promise<{
         status: "completed" | "waiting_for_user";
         chatId: string;

package/dist/index.js

@@ -9,7 +9,7 @@ import {
   deriveAppUrl,
   getExtForLang,
   serializeToYaml
-} from "./chunk-D7RIGVLZ.js";
+} from "./chunk-2QG4XPEB.js";
 import "./chunk-AXNRPVLE.js";
 export {
   ASSISTANT_FEEDBACK_REPORT_TITLE,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmates",
-  "version": "0.12.0-alpha.10",
+  "version": "0.12.0-alpha.11",
   "description": "OpenMates CLI and SDK",
   "type": "module",
   "main": "dist/index.js",