openmates 0.12.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/{chunk-UGEZLQKN.js → chunk-PHFCP5AM.js} +2073 -79
- package/dist/cli.js +1 -1
- package/dist/index.d.ts +44 -0
- package/dist/index.js +1 -1
- package/fixtures/brandenburger-tor.png +0 -0
- package/fixtures/brandenburger-tor.svg +25 -0
- package/package.json +5 -3
|
@@ -986,14 +986,14 @@ var OpenMatesWsClient = class {
|
|
|
986
986
|
});
|
|
987
987
|
}
|
|
988
988
|
async open(timeoutMs = 1e4) {
|
|
989
|
-
await new Promise((
|
|
989
|
+
await new Promise((resolve6, reject) => {
|
|
990
990
|
const timeout = setTimeout(
|
|
991
991
|
() => reject(new Error("WebSocket open timeout")),
|
|
992
992
|
timeoutMs
|
|
993
993
|
);
|
|
994
994
|
this.socket.once("open", () => {
|
|
995
995
|
clearTimeout(timeout);
|
|
996
|
-
|
|
996
|
+
resolve6();
|
|
997
997
|
});
|
|
998
998
|
this.socket.once("error", (error) => {
|
|
999
999
|
clearTimeout(timeout);
|
|
@@ -1022,15 +1022,15 @@ var OpenMatesWsClient = class {
|
|
|
1022
1022
|
this.socket.send(JSON.stringify({ type, payload }));
|
|
1023
1023
|
}
|
|
1024
1024
|
sendAsync(type, payload) {
|
|
1025
|
-
return new Promise((
|
|
1025
|
+
return new Promise((resolve6, reject) => {
|
|
1026
1026
|
this.socket.send(JSON.stringify({ type, payload }), (error) => {
|
|
1027
1027
|
if (error) reject(error);
|
|
1028
|
-
else
|
|
1028
|
+
else resolve6();
|
|
1029
1029
|
});
|
|
1030
1030
|
});
|
|
1031
1031
|
}
|
|
1032
1032
|
waitForMessage(expectedType, predicate, timeoutMs = 2e4) {
|
|
1033
|
-
return new Promise((
|
|
1033
|
+
return new Promise((resolve6, reject) => {
|
|
1034
1034
|
const onMessage = (rawData) => {
|
|
1035
1035
|
try {
|
|
1036
1036
|
const parsed = JSON.parse(rawData.toString());
|
|
@@ -1041,7 +1041,7 @@ var OpenMatesWsClient = class {
|
|
|
1041
1041
|
return;
|
|
1042
1042
|
}
|
|
1043
1043
|
cleanup();
|
|
1044
|
-
|
|
1044
|
+
resolve6(parsed);
|
|
1045
1045
|
} catch {
|
|
1046
1046
|
}
|
|
1047
1047
|
};
|
|
@@ -1074,14 +1074,14 @@ var OpenMatesWsClient = class {
|
|
|
1074
1074
|
* Used by ensureSynced to consume the full phased-sync event stream.
|
|
1075
1075
|
*/
|
|
1076
1076
|
collectMessages(terminatorType, timeoutMs = 9e4) {
|
|
1077
|
-
return new Promise((
|
|
1077
|
+
return new Promise((resolve6, reject) => {
|
|
1078
1078
|
const collected = [];
|
|
1079
1079
|
const onMessage = (rawData) => {
|
|
1080
1080
|
try {
|
|
1081
1081
|
const parsed = JSON.parse(rawData.toString());
|
|
1082
1082
|
if (parsed.type === terminatorType) {
|
|
1083
1083
|
cleanup();
|
|
1084
|
-
|
|
1084
|
+
resolve6(collected);
|
|
1085
1085
|
return;
|
|
1086
1086
|
}
|
|
1087
1087
|
collected.push(parsed);
|
|
@@ -1094,7 +1094,7 @@ var OpenMatesWsClient = class {
|
|
|
1094
1094
|
};
|
|
1095
1095
|
const onClose = () => {
|
|
1096
1096
|
cleanup();
|
|
1097
|
-
|
|
1097
|
+
resolve6(collected);
|
|
1098
1098
|
};
|
|
1099
1099
|
const timeout = setTimeout(() => {
|
|
1100
1100
|
cleanup();
|
|
@@ -1132,7 +1132,7 @@ var OpenMatesWsClient = class {
|
|
|
1132
1132
|
const timeoutMs = options?.timeoutMs ?? 9e4;
|
|
1133
1133
|
const onStream = options?.onStream;
|
|
1134
1134
|
const asyncEmbedWaitMs = options?.asyncEmbedWaitMs ?? 12e4;
|
|
1135
|
-
return new Promise((
|
|
1135
|
+
return new Promise((resolve6, reject) => {
|
|
1136
1136
|
let latestContent = "";
|
|
1137
1137
|
let messageId = null;
|
|
1138
1138
|
let taskId = null;
|
|
@@ -1189,7 +1189,7 @@ var OpenMatesWsClient = class {
|
|
|
1189
1189
|
if (waitingForUserPayload) {
|
|
1190
1190
|
if (pendingSubChatHandlers.size > 0) return;
|
|
1191
1191
|
cleanup();
|
|
1192
|
-
|
|
1192
|
+
resolve6({
|
|
1193
1193
|
status: "waiting_for_user",
|
|
1194
1194
|
messageId,
|
|
1195
1195
|
taskId,
|
|
@@ -1209,7 +1209,7 @@ var OpenMatesWsClient = class {
|
|
|
1209
1209
|
if (processingEmbedIds.size > 0 && !asyncEmbedTimer) {
|
|
1210
1210
|
asyncEmbedTimer = setTimeout(() => {
|
|
1211
1211
|
cleanup();
|
|
1212
|
-
|
|
1212
|
+
resolve6({
|
|
1213
1213
|
status: "completed",
|
|
1214
1214
|
messageId,
|
|
1215
1215
|
taskId,
|
|
@@ -1226,7 +1226,7 @@ var OpenMatesWsClient = class {
|
|
|
1226
1226
|
}
|
|
1227
1227
|
if (processingEmbedIds.size > 0) return;
|
|
1228
1228
|
cleanup();
|
|
1229
|
-
|
|
1229
|
+
resolve6({
|
|
1230
1230
|
status: "completed",
|
|
1231
1231
|
messageId,
|
|
1232
1232
|
taskId,
|
|
@@ -1440,7 +1440,7 @@ var OpenMatesWsClient = class {
|
|
|
1440
1440
|
const onClose = () => {
|
|
1441
1441
|
if (aiResponseDone) {
|
|
1442
1442
|
cleanup();
|
|
1443
|
-
|
|
1443
|
+
resolve6({
|
|
1444
1444
|
status: "completed",
|
|
1445
1445
|
messageId,
|
|
1446
1446
|
taskId,
|
|
@@ -2805,13 +2805,21 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
2805
2805
|
}
|
|
2806
2806
|
const chatId = `anonymous-${randomUUID2()}`;
|
|
2807
2807
|
const messageId = `anonymous-message-${randomUUID2()}`;
|
|
2808
|
-
const
|
|
2808
|
+
const requestBody = {
|
|
2809
2809
|
anonymous_id: anonymousId,
|
|
2810
2810
|
client_chat_id: chatId,
|
|
2811
2811
|
client_message_id: messageId,
|
|
2812
2812
|
plaintext_message: params.message,
|
|
2813
2813
|
message_history: []
|
|
2814
|
-
}
|
|
2814
|
+
};
|
|
2815
|
+
if (params.learningMode?.enabled === true) {
|
|
2816
|
+
requestBody.learning_mode = {
|
|
2817
|
+
enabled: true,
|
|
2818
|
+
age_group: params.learningMode.ageGroup ?? null,
|
|
2819
|
+
source: params.learningMode.source ?? "anonymous_session"
|
|
2820
|
+
};
|
|
2821
|
+
}
|
|
2822
|
+
const response = await this.http.post("/v1/anonymous/chat/stream", requestBody);
|
|
2815
2823
|
if (!response.ok) {
|
|
2816
2824
|
const detail = response.data.detail;
|
|
2817
2825
|
const message = typeof detail === "object" && detail?.message ? detail.message : typeof detail === "string" ? detail : `Anonymous chat failed with HTTP ${response.status}`;
|
|
@@ -2979,6 +2987,41 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
2979
2987
|
saveSession(session);
|
|
2980
2988
|
return response.data.user ?? {};
|
|
2981
2989
|
}
|
|
2990
|
+
async getLearningModeStatus() {
|
|
2991
|
+
this.requireSession();
|
|
2992
|
+
const response = await this.http.get(
|
|
2993
|
+
"/v1/learning-mode",
|
|
2994
|
+
this.getCliRequestHeaders()
|
|
2995
|
+
);
|
|
2996
|
+
if (!response.ok) {
|
|
2997
|
+
throw new Error(`Learning Mode status failed with HTTP ${response.status}`);
|
|
2998
|
+
}
|
|
2999
|
+
return response.data;
|
|
3000
|
+
}
|
|
3001
|
+
async activateLearningMode(params) {
|
|
3002
|
+
this.requireSession();
|
|
3003
|
+
const response = await this.http.post(
|
|
3004
|
+
"/v1/learning-mode/activate",
|
|
3005
|
+
{ age_group: params.ageGroup, passcode: params.passcode },
|
|
3006
|
+
this.getCliRequestHeaders()
|
|
3007
|
+
);
|
|
3008
|
+
if (!response.ok) {
|
|
3009
|
+
throw new Error(`Learning Mode activation failed with HTTP ${response.status}`);
|
|
3010
|
+
}
|
|
3011
|
+
return response.data;
|
|
3012
|
+
}
|
|
3013
|
+
async deactivateLearningMode(passcode) {
|
|
3014
|
+
this.requireSession();
|
|
3015
|
+
const response = await this.http.post(
|
|
3016
|
+
"/v1/learning-mode/deactivate",
|
|
3017
|
+
{ passcode },
|
|
3018
|
+
this.getCliRequestHeaders()
|
|
3019
|
+
);
|
|
3020
|
+
if (!response.ok) {
|
|
3021
|
+
throw new Error(`Learning Mode deactivation failed with HTTP ${response.status}`);
|
|
3022
|
+
}
|
|
3023
|
+
return response.data;
|
|
3024
|
+
}
|
|
2982
3025
|
async logout() {
|
|
2983
3026
|
if (this.session) {
|
|
2984
3027
|
await this.http.post("/v1/auth/logout", {}, this.getCliRequestHeaders()).catch(() => void 0);
|
|
@@ -3677,6 +3720,29 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
3677
3720
|
if (connectedAccountTokenRefs.length > 0) {
|
|
3678
3721
|
messagePayload.connected_account_token_refs = connectedAccountTokenRefs;
|
|
3679
3722
|
}
|
|
3723
|
+
if (params.benchmarkMetadata) {
|
|
3724
|
+
messagePayload.benchmark_metadata = params.benchmarkMetadata;
|
|
3725
|
+
}
|
|
3726
|
+
if (params.learningMode) {
|
|
3727
|
+
messagePayload.learning_mode = {
|
|
3728
|
+
enabled: params.learningMode.enabled,
|
|
3729
|
+
age_group: params.learningMode.ageGroup ?? null
|
|
3730
|
+
};
|
|
3731
|
+
}
|
|
3732
|
+
if (params.incognito) {
|
|
3733
|
+
const providedHistory = (params.messageHistory ?? []).map((historyMessage) => ({
|
|
3734
|
+
...historyMessage,
|
|
3735
|
+
chat_id: historyMessage.chat_id ?? chatId
|
|
3736
|
+
}));
|
|
3737
|
+
messagePayload.message_history = [...providedHistory, {
|
|
3738
|
+
message_id: messageId,
|
|
3739
|
+
chat_id: chatId,
|
|
3740
|
+
role: "user",
|
|
3741
|
+
sender_name: "User",
|
|
3742
|
+
content: params.message,
|
|
3743
|
+
created_at: createdAt
|
|
3744
|
+
}];
|
|
3745
|
+
}
|
|
3680
3746
|
let chatKeyBytes = null;
|
|
3681
3747
|
let encryptedChatKey = null;
|
|
3682
3748
|
let baselineMessagesV = 0;
|
|
@@ -3735,6 +3801,7 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
3735
3801
|
if (encryptedEmbeds.length > 0) {
|
|
3736
3802
|
messagePayload.encrypted_embeds = encryptedEmbeds;
|
|
3737
3803
|
}
|
|
3804
|
+
const precollectedResponse = params.precollectResponse ? ws.collectAiResponse(messageId, chatId, { onStream: params.onStream }) : null;
|
|
3738
3805
|
const confirmed = ws.waitForMessage(
|
|
3739
3806
|
"chat_message_confirmed",
|
|
3740
3807
|
(payload) => {
|
|
@@ -3949,7 +4016,7 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
3949
4016
|
};
|
|
3950
4017
|
if (params.incognito) {
|
|
3951
4018
|
try {
|
|
3952
|
-
const resp = await ws.collectAiResponse(messageId, chatId, streamOpts);
|
|
4019
|
+
const resp = await (precollectedResponse ?? ws.collectAiResponse(messageId, chatId, streamOpts));
|
|
3953
4020
|
assistantMessageId = resp.messageId;
|
|
3954
4021
|
assistant = resp.content;
|
|
3955
4022
|
category = resp.category;
|
|
@@ -4301,7 +4368,7 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
4301
4368
|
if (response.data.status === "failed") {
|
|
4302
4369
|
throw new Error(response.data.error ?? "Task failed");
|
|
4303
4370
|
}
|
|
4304
|
-
await new Promise((
|
|
4371
|
+
await new Promise((resolve6) => setTimeout(resolve6, SKILL_TASK_POLL_INTERVAL_MS));
|
|
4305
4372
|
}
|
|
4306
4373
|
throw new Error(`Task ${taskId} did not complete within ${SKILL_TASK_POLL_TIMEOUT_MS / 1e3}s`);
|
|
4307
4374
|
}
|
|
@@ -4522,7 +4589,7 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
4522
4589
|
`Rate limited by settings API; retrying in ${Math.ceil(SETTINGS_GET_RATE_LIMIT_RETRY_MS / 1e3)}s...
|
|
4523
4590
|
`
|
|
4524
4591
|
);
|
|
4525
|
-
await new Promise((
|
|
4592
|
+
await new Promise((resolve6) => setTimeout(resolve6, SETTINGS_GET_RATE_LIMIT_RETRY_MS));
|
|
4526
4593
|
response = await this.http.get(normalizedPath, this.getCliRequestHeaders());
|
|
4527
4594
|
}
|
|
4528
4595
|
if (!response.ok) {
|
|
@@ -6023,7 +6090,7 @@ function filenameFromContentDisposition(header2) {
|
|
|
6023
6090
|
return plain?.trim() ?? null;
|
|
6024
6091
|
}
|
|
6025
6092
|
function sleep(ms) {
|
|
6026
|
-
return new Promise((
|
|
6093
|
+
return new Promise((resolve6) => setTimeout(resolve6, ms));
|
|
6027
6094
|
}
|
|
6028
6095
|
function printLogo() {
|
|
6029
6096
|
const W = "\x1B[1;37m";
|
|
@@ -6039,9 +6106,9 @@ function printLogo() {
|
|
|
6039
6106
|
|
|
6040
6107
|
// src/cli.ts
|
|
6041
6108
|
import { createInterface as createInterface3 } from "readline/promises";
|
|
6042
|
-
import { realpathSync, writeFileSync as
|
|
6043
|
-
import { fileURLToPath } from "url";
|
|
6044
|
-
import { basename as basename3, dirname } from "path";
|
|
6109
|
+
import { realpathSync, writeFileSync as writeFileSync5 } from "fs";
|
|
6110
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
6111
|
+
import { basename as basename3, dirname as dirname3 } from "path";
|
|
6045
6112
|
import WebSocket2 from "ws";
|
|
6046
6113
|
|
|
6047
6114
|
// ../secret-scanner/src/registry.ts
|
|
@@ -7741,8 +7808,8 @@ async function renderRemotionShareLink(embedId, client, ln) {
|
|
|
7741
7808
|
}
|
|
7742
7809
|
}
|
|
7743
7810
|
function generateQr(value) {
|
|
7744
|
-
return new Promise((
|
|
7745
|
-
qrcode2.generate(value, { small: true }, (qr) =>
|
|
7811
|
+
return new Promise((resolve6) => {
|
|
7812
|
+
qrcode2.generate(value, { small: true }, (qr) => resolve6(qr));
|
|
7746
7813
|
});
|
|
7747
7814
|
}
|
|
7748
7815
|
function remotionMeta(c) {
|
|
@@ -8510,8 +8577,9 @@ import { execSync, spawn as nodeSpawn } from "child_process";
|
|
|
8510
8577
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
8511
8578
|
import { copyFileSync, existsSync as existsSync5, mkdirSync as mkdirSync3, readFileSync as readFileSync5, rmSync as rmSync3, writeFileSync as writeFileSync3 } from "fs";
|
|
8512
8579
|
import { createInterface as createInterface2 } from "readline";
|
|
8580
|
+
import { createInterface as createPromptInterface } from "readline/promises";
|
|
8513
8581
|
import { homedir as homedir5 } from "os";
|
|
8514
|
-
import { join as join3, resolve as resolve3 } from "path";
|
|
8582
|
+
import { dirname, join as join3, resolve as resolve3 } from "path";
|
|
8515
8583
|
var SOURCE_COMPOSE_FILE = join3("backend", "core", "docker-compose.yml");
|
|
8516
8584
|
var IMAGE_COMPOSE_FILE = join3("backend", "core", "docker-compose.selfhost.yml");
|
|
8517
8585
|
var COMPOSE_OVERRIDE = join3("backend", "core", "docker-compose.override.yml");
|
|
@@ -8528,6 +8596,43 @@ var IMAGE_CHANNEL_TAGS = {
|
|
|
8528
8596
|
main: MAIN_BRANCH,
|
|
8529
8597
|
dev: DEV_BRANCH
|
|
8530
8598
|
};
|
|
8599
|
+
var BACKEND_CONFIG_FILE = join3("backend", "config", "backend_config.yml");
|
|
8600
|
+
var IMAGE_RUNTIME_CONFIG_FILE = join3("config", "backend_config.yml");
|
|
8601
|
+
var LOCAL_AI_MODELS_FILE = "local-ai-models.yml";
|
|
8602
|
+
var OFF_BY_DEFAULT_FEATURES = /* @__PURE__ */ new Map([
|
|
8603
|
+
["embed:code:application", "Application previews are still unstable"],
|
|
8604
|
+
["platform:projects", "Projects workspace is not ready by default"],
|
|
8605
|
+
["platform:workflows", "Workflows workspace is not implemented yet"],
|
|
8606
|
+
["platform:tasks", "Tasks workspace is not implemented yet"]
|
|
8607
|
+
]);
|
|
8608
|
+
var LOCAL_MODEL_RUNTIME_DEFAULTS = {
|
|
8609
|
+
ollama: {
|
|
8610
|
+
label: "Ollama",
|
|
8611
|
+
serverId: "ollama",
|
|
8612
|
+
baseUrl: "http://host.docker.internal:11434/v1",
|
|
8613
|
+
apiKey: "ollama"
|
|
8614
|
+
},
|
|
8615
|
+
"lm-studio": {
|
|
8616
|
+
label: "LM Studio",
|
|
8617
|
+
serverId: "lm_studio",
|
|
8618
|
+
baseUrl: "http://host.docker.internal:1234/v1",
|
|
8619
|
+
apiKey: "lm-studio"
|
|
8620
|
+
},
|
|
8621
|
+
custom: {
|
|
8622
|
+
label: "Custom OpenAI-compatible API",
|
|
8623
|
+
serverId: "custom_openai_compatible",
|
|
8624
|
+
baseUrl: "",
|
|
8625
|
+
apiKey: "local"
|
|
8626
|
+
}
|
|
8627
|
+
};
|
|
8628
|
+
var MODEL_CREATOR_OPTIONS = [
|
|
8629
|
+
{ id: "alibaba", name: "Alibaba / Qwen", match: /(^|[-_:/])qwen/i },
|
|
8630
|
+
{ id: "google", name: "Google / Gemma", match: /(^|[-_:/])gemma/i },
|
|
8631
|
+
{ id: "mistral", name: "Mistral", match: /(^|[-_:/])(mistral|mixtral|ministral)/i },
|
|
8632
|
+
{ id: "openai", name: "OpenAI", match: /(^|[-_:/])gpt-oss/i },
|
|
8633
|
+
{ id: "deepseek", name: "DeepSeek", match: /(^|[-_:/])deepseek/i },
|
|
8634
|
+
{ id: "custom", name: "Custom", match: null }
|
|
8635
|
+
];
|
|
8531
8636
|
var MINIMAL_ENV_TEMPLATE = `# OpenMates self-host image-mode environment
|
|
8532
8637
|
SECRET__MISTRAL_AI__API_KEY=
|
|
8533
8638
|
SECRET__CEREBRAS__API_KEY=
|
|
@@ -8597,9 +8702,9 @@ function exec(cmd, cwd) {
|
|
|
8597
8702
|
return execSync(cmd, { cwd, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
|
|
8598
8703
|
}
|
|
8599
8704
|
function runInteractive(cmd, args, cwd) {
|
|
8600
|
-
return new Promise((
|
|
8705
|
+
return new Promise((resolve6, reject) => {
|
|
8601
8706
|
const child = nodeSpawn(cmd, args, { cwd, stdio: "inherit", shell: false });
|
|
8602
|
-
child.on("close", (code) =>
|
|
8707
|
+
child.on("close", (code) => resolve6(code ?? 1));
|
|
8603
8708
|
child.on("error", reject);
|
|
8604
8709
|
});
|
|
8605
8710
|
}
|
|
@@ -8615,6 +8720,65 @@ function getInstallMode(installPath, config = loadConfigForInstallPath(installPa
|
|
|
8615
8720
|
function shouldPullImages() {
|
|
8616
8721
|
return process.env.OPENMATES_SKIP_IMAGE_PULL !== "1";
|
|
8617
8722
|
}
|
|
8723
|
+
function normalizeFeatureList(items) {
|
|
8724
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8725
|
+
const normalized = [];
|
|
8726
|
+
for (const item of items) {
|
|
8727
|
+
const value = item.trim();
|
|
8728
|
+
if (!value || seen.has(value)) continue;
|
|
8729
|
+
seen.add(value);
|
|
8730
|
+
normalized.push(value);
|
|
8731
|
+
}
|
|
8732
|
+
return normalized;
|
|
8733
|
+
}
|
|
8734
|
+
function parseListBlock(content, key) {
|
|
8735
|
+
const match = content.match(new RegExp(`^${key}:\\n((?:[ \\t]+.*\\n?)*)`, "m"));
|
|
8736
|
+
if (!match) return [];
|
|
8737
|
+
const block = match[1] ?? "";
|
|
8738
|
+
return normalizeFeatureList(
|
|
8739
|
+
[...block.matchAll(/^\s*-\s*["']?([^"'\n#]+)["']?/gm)].map((item) => item[1] ?? "")
|
|
8740
|
+
);
|
|
8741
|
+
}
|
|
8742
|
+
function parseFeatureOverrides(content) {
|
|
8743
|
+
const overridesMatch = content.match(/^feature_overrides:\n((?:[ \t]+.*\n?)*)/m);
|
|
8744
|
+
const overridesBlock = overridesMatch?.[1] ?? "";
|
|
8745
|
+
const enabled = parseListBlock(overridesBlock.replace(/^ {2}/gm, ""), "enabled");
|
|
8746
|
+
const disabled = parseListBlock(overridesBlock.replace(/^ {2}/gm, ""), "disabled");
|
|
8747
|
+
const legacyDisabledApps = parseListBlock(content, "disabled_apps").map(
|
|
8748
|
+
(appId) => appId.startsWith("app:") ? appId : `app:${appId}`
|
|
8749
|
+
);
|
|
8750
|
+
return {
|
|
8751
|
+
enabled: normalizeFeatureList(enabled),
|
|
8752
|
+
disabled: normalizeFeatureList([...disabled, ...legacyDisabledApps])
|
|
8753
|
+
};
|
|
8754
|
+
}
|
|
8755
|
+
function renderFeatureOverrides(overrides) {
|
|
8756
|
+
const renderList = (key, items) => {
|
|
8757
|
+
if (!items.length) return ` ${key}: []`;
|
|
8758
|
+
return [` ${key}:`, ...items.map((item) => ` - "${item}"`)].join("\n");
|
|
8759
|
+
};
|
|
8760
|
+
return [
|
|
8761
|
+
"# Admin feature overrides. Changes require a server restart.",
|
|
8762
|
+
"feature_overrides:",
|
|
8763
|
+
renderList("enabled", overrides.enabled),
|
|
8764
|
+
renderList("disabled", overrides.disabled),
|
|
8765
|
+
""
|
|
8766
|
+
].join("\n");
|
|
8767
|
+
}
|
|
8768
|
+
function removeConfigBlock(content, key) {
|
|
8769
|
+
return content.replace(new RegExp(`(?:^|\\n)#.*\\n${key}:\\n(?:[ \\t]+.*\\n?)*`, "m"), "\n").replace(new RegExp(`^${key}:\\n(?:[ \\t]+.*\\n?)*`, "m"), "");
|
|
8770
|
+
}
|
|
8771
|
+
function updateFeatureOverridesContent(content, overrides) {
|
|
8772
|
+
let next = removeConfigBlock(content, "feature_overrides");
|
|
8773
|
+
next = removeConfigBlock(next, "disabled_apps");
|
|
8774
|
+
next = next.trimEnd();
|
|
8775
|
+
return `${next}
|
|
8776
|
+
|
|
8777
|
+
${renderFeatureOverrides(overrides)}`;
|
|
8778
|
+
}
|
|
8779
|
+
function featureKind(featureId) {
|
|
8780
|
+
return featureId.split(":", 1)[0] || "unknown";
|
|
8781
|
+
}
|
|
8618
8782
|
function composeArgs(installPath, withOverrides, installMode = getInstallMode(installPath)) {
|
|
8619
8783
|
const composeFile = installMode === "image" ? IMAGE_COMPOSE_FILE : SOURCE_COMPOSE_FILE;
|
|
8620
8784
|
const args = ["compose", "--env-file", ".env", "-f", composeFile];
|
|
@@ -8765,8 +8929,10 @@ async function writeImageModeRuntimeFiles(installPath, imageTag) {
|
|
|
8765
8929
|
const coreDir = join3(installPath, "backend", "core");
|
|
8766
8930
|
const vaultConfigDir = join3(coreDir, "vault", "config");
|
|
8767
8931
|
mkdirSync3(vaultConfigDir, { recursive: true });
|
|
8932
|
+
mkdirSync3(join3(installPath, "config", "providers"), { recursive: true });
|
|
8768
8933
|
writeFileSync3(join3(coreDir, "docker-compose.selfhost.yml"), await loadSelfHostComposeTemplate(templateRefForImageTag(imageTag, getPackageVersion())));
|
|
8769
8934
|
writeFileSync3(join3(vaultConfigDir, "vault.hcl"), VAULT_CONFIG_TEMPLATE);
|
|
8935
|
+
ensureImageRuntimeConfig(installPath);
|
|
8770
8936
|
const envPath = join3(installPath, ".env");
|
|
8771
8937
|
let envContent = existsSync5(envPath) ? readFileSync5(envPath, "utf-8") : MINIMAL_ENV_TEMPLATE;
|
|
8772
8938
|
envContent = setEnvIfEmpty(envContent, "DATABASE_ADMIN_PASSWORD", randomHex(12));
|
|
@@ -8830,6 +8996,7 @@ function defaultCloneBranchForVersion(version) {
|
|
|
8830
8996
|
}
|
|
8831
8997
|
function hasLlmCredentials(envPath) {
|
|
8832
8998
|
if (!existsSync5(envPath)) return false;
|
|
8999
|
+
if (hasLocalAiModels(dirname(envPath))) return true;
|
|
8833
9000
|
const content = readFileSync5(envPath, "utf-8");
|
|
8834
9001
|
for (const line of content.split("\n")) {
|
|
8835
9002
|
const trimmed = line.trim();
|
|
@@ -8854,22 +9021,180 @@ function warnIfMissingLlmCredentials(installPath) {
|
|
|
8854
9021
|
}
|
|
8855
9022
|
if (!hasLlmCredentials(envPath)) {
|
|
8856
9023
|
console.error(
|
|
8857
|
-
"No LLM provider API key found in .env.\nOpenMates will start, but AI chat/model processing will stay unavailable until you add one.\n\
|
|
9024
|
+
"No LLM provider API key found in .env.\nOpenMates will start, but AI chat/model processing will stay unavailable until you add one.\n\nRun 'openmates server ai models add' to add a local Ollama/LM Studio model, or add at least one of these to your .env file:\n SECRET__OPENAI__API_KEY=sk-...\n SECRET__ANTHROPIC__API_KEY=sk-ant-...\n SECRET__GOOGLE_AI_STUDIO__API_KEY=...\n\nAfter updating .env, run 'openmates server restart'."
|
|
8858
9025
|
);
|
|
8859
9026
|
}
|
|
8860
9027
|
}
|
|
8861
9028
|
async function confirmDestructive(phrase) {
|
|
8862
9029
|
const rl = createInterface2({ input: process.stdin, output: process.stderr });
|
|
8863
|
-
return new Promise((
|
|
9030
|
+
return new Promise((resolve6) => {
|
|
8864
9031
|
rl.question(`Type "${phrase}" to confirm: `, (answer) => {
|
|
8865
9032
|
rl.close();
|
|
8866
|
-
|
|
9033
|
+
resolve6(answer.trim() === phrase);
|
|
8867
9034
|
});
|
|
8868
9035
|
});
|
|
8869
9036
|
}
|
|
8870
9037
|
function printJson(data) {
|
|
8871
9038
|
console.log(JSON.stringify(data, null, 2));
|
|
8872
9039
|
}
|
|
9040
|
+
function localModelsOverlayPath(installPath, installMode = getInstallMode(installPath)) {
|
|
9041
|
+
if (installMode === "source") return join3(installPath, "backend", "providers", LOCAL_AI_MODELS_FILE);
|
|
9042
|
+
return join3(installPath, "config", "providers", LOCAL_AI_MODELS_FILE);
|
|
9043
|
+
}
|
|
9044
|
+
function imageBackendConfigPath(installPath) {
|
|
9045
|
+
return join3(installPath, IMAGE_RUNTIME_CONFIG_FILE);
|
|
9046
|
+
}
|
|
9047
|
+
function ensureImageRuntimeConfig(installPath) {
|
|
9048
|
+
const configPath = imageBackendConfigPath(installPath);
|
|
9049
|
+
if (existsSync5(configPath)) return;
|
|
9050
|
+
mkdirSync3(dirname(configPath), { recursive: true });
|
|
9051
|
+
writeFileSync3(configPath, renderFeatureOverrides({ enabled: [], disabled: [] }));
|
|
9052
|
+
}
|
|
9053
|
+
function readLocalModelOverlay(path) {
|
|
9054
|
+
if (!existsSync5(path)) return { providers: [] };
|
|
9055
|
+
try {
|
|
9056
|
+
const parsed = JSON.parse(readFileSync5(path, "utf-8"));
|
|
9057
|
+
return { providers: Array.isArray(parsed.providers) ? parsed.providers : [] };
|
|
9058
|
+
} catch (error) {
|
|
9059
|
+
throw new Error(
|
|
9060
|
+
`Could not parse ${path}. This file is managed by the OpenMates CLI and must remain JSON-compatible YAML. ${error instanceof Error ? error.message : String(error)}`
|
|
9061
|
+
);
|
|
9062
|
+
}
|
|
9063
|
+
}
|
|
9064
|
+
function writeLocalModelOverlay(path, overlay) {
|
|
9065
|
+
mkdirSync3(dirname(path), { recursive: true });
|
|
9066
|
+
writeFileSync3(path, `${JSON.stringify(overlay, null, 2)}
|
|
9067
|
+
`);
|
|
9068
|
+
}
|
|
9069
|
+
function hasLocalAiModels(installPath) {
|
|
9070
|
+
const imagePath = join3(installPath, "config", "providers", LOCAL_AI_MODELS_FILE);
|
|
9071
|
+
const sourcePath = join3(installPath, "backend", "providers", LOCAL_AI_MODELS_FILE);
|
|
9072
|
+
return [imagePath, sourcePath].some((path) => {
|
|
9073
|
+
if (!existsSync5(path)) return false;
|
|
9074
|
+
try {
|
|
9075
|
+
return readLocalModelOverlay(path).providers.some((provider) => provider.models.length > 0);
|
|
9076
|
+
} catch {
|
|
9077
|
+
return false;
|
|
9078
|
+
}
|
|
9079
|
+
});
|
|
9080
|
+
}
|
|
9081
|
+
function sanitizeModelId(raw) {
|
|
9082
|
+
return raw.trim().toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "local-model";
|
|
9083
|
+
}
|
|
9084
|
+
function inferCreatorId(modelId) {
|
|
9085
|
+
return MODEL_CREATOR_OPTIONS.find((option) => option.match?.test(modelId))?.id ?? "custom";
|
|
9086
|
+
}
|
|
9087
|
+
function creatorDisplayName(creatorId) {
|
|
9088
|
+
return MODEL_CREATOR_OPTIONS.find((option) => option.id === creatorId)?.name ?? creatorId;
|
|
9089
|
+
}
|
|
9090
|
+
function normalizeCreatorId(value) {
|
|
9091
|
+
const normalized = value.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "_").replace(/^_+|_+$/g, "");
|
|
9092
|
+
return normalized || "custom_local";
|
|
9093
|
+
}
|
|
9094
|
+
function normalizeRuntimeKey(value) {
|
|
9095
|
+
const normalized = value.trim().toLowerCase().replace(/_/g, "-");
|
|
9096
|
+
if (normalized === "lmstudio" || normalized === "lm-studio") return "lm-studio";
|
|
9097
|
+
if (normalized === "ollama") return "ollama";
|
|
9098
|
+
return "custom";
|
|
9099
|
+
}
|
|
9100
|
+
function boolFromFlag(value, defaultValue = false) {
|
|
9101
|
+
if (value === void 0) return defaultValue;
|
|
9102
|
+
if (value === true) return true;
|
|
9103
|
+
if (value === false) return false;
|
|
9104
|
+
const normalized = value.toLowerCase();
|
|
9105
|
+
return ["1", "true", "yes", "y", "on"].includes(normalized);
|
|
9106
|
+
}
|
|
9107
|
+
async function promptText(question, defaultValue = "") {
|
|
9108
|
+
const rl = createPromptInterface({ input: process.stdin, output: process.stderr });
|
|
9109
|
+
try {
|
|
9110
|
+
const suffix = defaultValue ? ` (${defaultValue})` : "";
|
|
9111
|
+
const answer = await rl.question(`${question}${suffix}: `);
|
|
9112
|
+
return answer.trim() || defaultValue;
|
|
9113
|
+
} finally {
|
|
9114
|
+
rl.close();
|
|
9115
|
+
}
|
|
9116
|
+
}
|
|
9117
|
+
async function promptChoice(question, choices, defaultValue) {
|
|
9118
|
+
console.error(question);
|
|
9119
|
+
choices.forEach((choice, index) => {
|
|
9120
|
+
const marker = choice.value === defaultValue ? " [default]" : "";
|
|
9121
|
+
console.error(` ${index + 1}. ${choice.label}${marker}`);
|
|
9122
|
+
});
|
|
9123
|
+
const answer = await promptText("Choose number or value", defaultValue);
|
|
9124
|
+
const numeric = Number.parseInt(answer, 10);
|
|
9125
|
+
if (Number.isInteger(numeric) && numeric >= 1 && numeric <= choices.length) {
|
|
9126
|
+
return choices[numeric - 1].value;
|
|
9127
|
+
}
|
|
9128
|
+
const direct = choices.find((choice) => choice.value === answer || choice.label.toLowerCase() === answer.toLowerCase());
|
|
9129
|
+
return direct?.value ?? answer;
|
|
9130
|
+
}
|
|
9131
|
+
async function fetchLocalModelIds(baseUrl, apiKey) {
|
|
9132
|
+
const response = await fetch(`${baseUrl.replace(/\/+$/, "")}/models`, {
|
|
9133
|
+
headers: { Authorization: `Bearer ${apiKey || "local"}` }
|
|
9134
|
+
});
|
|
9135
|
+
if (!response.ok) throw new Error(`Failed to fetch local models: HTTP ${response.status}`);
|
|
9136
|
+
const body = await response.json();
|
|
9137
|
+
return (body.data ?? []).map((model) => model.id).filter((id) => Boolean(id));
|
|
9138
|
+
}
|
|
9139
|
+
async function testLocalModel(baseUrl, apiKey, modelId) {
|
|
9140
|
+
const controller = new AbortController();
|
|
9141
|
+
const timeout = setTimeout(() => controller.abort(), 3e4);
|
|
9142
|
+
try {
|
|
9143
|
+
const response = await fetch(`${baseUrl.replace(/\/+$/, "")}/chat/completions`, {
|
|
9144
|
+
method: "POST",
|
|
9145
|
+
signal: controller.signal,
|
|
9146
|
+
headers: {
|
|
9147
|
+
Authorization: `Bearer ${apiKey || "local"}`,
|
|
9148
|
+
"Content-Type": "application/json"
|
|
9149
|
+
},
|
|
9150
|
+
body: JSON.stringify({
|
|
9151
|
+
model: modelId,
|
|
9152
|
+
messages: [
|
|
9153
|
+
{ role: "system", content: "Answer with only the number." },
|
|
9154
|
+
{ role: "user", content: "1+2?" }
|
|
9155
|
+
],
|
|
9156
|
+
stream: false,
|
|
9157
|
+
temperature: 0
|
|
9158
|
+
})
|
|
9159
|
+
});
|
|
9160
|
+
if (!response.ok) throw new Error(`HTTP ${response.status}: ${await response.text()}`);
|
|
9161
|
+
const body = await response.json();
|
|
9162
|
+
return body.choices?.[0]?.message?.content?.trim() ?? "";
|
|
9163
|
+
} finally {
|
|
9164
|
+
clearTimeout(timeout);
|
|
9165
|
+
}
|
|
9166
|
+
}
|
|
9167
|
+
function upsertLocalModel(path, providerId, providerName, model) {
|
|
9168
|
+
const overlay = readLocalModelOverlay(path);
|
|
9169
|
+
let provider = overlay.providers.find((item) => item.provider_id === providerId);
|
|
9170
|
+
if (!provider) {
|
|
9171
|
+
provider = {
|
|
9172
|
+
provider_id: providerId,
|
|
9173
|
+
name: providerName,
|
|
9174
|
+
description: `Local self-hosted models for ${providerName}.`,
|
|
9175
|
+
models: []
|
|
9176
|
+
};
|
|
9177
|
+
overlay.providers.push(provider);
|
|
9178
|
+
}
|
|
9179
|
+
const modelId = model.id;
|
|
9180
|
+
provider.models = provider.models.filter((existing) => existing.id !== modelId);
|
|
9181
|
+
provider.models.push(model);
|
|
9182
|
+
writeLocalModelOverlay(path, overlay);
|
|
9183
|
+
}
|
|
9184
|
+
function removeLocalModel(path, fullModelId) {
|
|
9185
|
+
const [providerId, modelId] = fullModelId.split("/", 2);
|
|
9186
|
+
if (!providerId || !modelId) throw new Error("Use provider/model-id format.");
|
|
9187
|
+
const overlay = readLocalModelOverlay(path);
|
|
9188
|
+
const provider = overlay.providers.find((item) => item.provider_id === providerId);
|
|
9189
|
+
if (!provider) return false;
|
|
9190
|
+
const before = provider.models.length;
|
|
9191
|
+
provider.models = provider.models.filter((model) => model.id !== modelId);
|
|
9192
|
+
writeLocalModelOverlay(path, overlay);
|
|
9193
|
+
return provider.models.length !== before;
|
|
9194
|
+
}
|
|
9195
|
+
function localModelsFromOverlay(overlay) {
|
|
9196
|
+
return overlay.providers.flatMap((provider) => provider.models.map((model) => ({ providerId: provider.provider_id, model })));
|
|
9197
|
+
}
|
|
8873
9198
|
async function serverStatus(flags) {
|
|
8874
9199
|
requireDocker();
|
|
8875
9200
|
const installPath = resolveServerPath(flags);
|
|
@@ -9324,6 +9649,245 @@ async function serverMakeAdmin(rest, flags) {
|
|
|
9324
9649
|
console.log(`Admin privileges granted to ${email}.`);
|
|
9325
9650
|
}
|
|
9326
9651
|
}
|
|
9652
|
+
async function serverFeatures(rest, flags) {
|
|
9653
|
+
const action = rest[0] ?? "list";
|
|
9654
|
+
const featureId = rest[1];
|
|
9655
|
+
const installPath = resolveServerPath(flags);
|
|
9656
|
+
const installMode = getInstallMode(installPath);
|
|
9657
|
+
if (installMode === "image") ensureImageRuntimeConfig(installPath);
|
|
9658
|
+
const configPath = installMode === "image" ? imageBackendConfigPath(installPath) : join3(installPath, BACKEND_CONFIG_FILE);
|
|
9659
|
+
if (!existsSync5(configPath)) {
|
|
9660
|
+
throw new Error(`Backend config not found at ${configPath}. Run 'openmates server install' first or pass --path <dir>.`);
|
|
9661
|
+
}
|
|
9662
|
+
const content = readFileSync5(configPath, "utf-8");
|
|
9663
|
+
const overrides = parseFeatureOverrides(content);
|
|
9664
|
+
const writeOverrides = (nextOverrides) => {
|
|
9665
|
+
writeFileSync3(configPath, updateFeatureOverridesContent(content, nextOverrides));
|
|
9666
|
+
console.log(`Updated ${configPath}`);
|
|
9667
|
+
console.log("Restart the server for feature changes to take effect: openmates server restart");
|
|
9668
|
+
};
|
|
9669
|
+
if (action === "list") {
|
|
9670
|
+
console.log("Feature overrides:");
|
|
9671
|
+
console.log(` enabled: ${overrides.enabled.length ? overrides.enabled.join(", ") : "none"}`);
|
|
9672
|
+
console.log(` disabled: ${overrides.disabled.length ? overrides.disabled.join(", ") : "none"}`);
|
|
9673
|
+
console.log("\nKnown off-by-default features:");
|
|
9674
|
+
for (const [id, reason] of OFF_BY_DEFAULT_FEATURES.entries()) {
|
|
9675
|
+
const override = overrides.enabled.includes(id) ? "enabled override" : overrides.disabled.includes(id) ? "disabled override" : "default off";
|
|
9676
|
+
console.log(` ${id} (${featureKind(id)}): ${override} - ${reason}`);
|
|
9677
|
+
}
|
|
9678
|
+
return;
|
|
9679
|
+
}
|
|
9680
|
+
if (!featureId) {
|
|
9681
|
+
throw new Error(`Usage: openmates server features ${action} <feature-id>`);
|
|
9682
|
+
}
|
|
9683
|
+
if (action === "enable") {
|
|
9684
|
+
writeOverrides({
|
|
9685
|
+
enabled: normalizeFeatureList([...overrides.enabled, featureId]),
|
|
9686
|
+
disabled: overrides.disabled.filter((id) => id !== featureId)
|
|
9687
|
+
});
|
|
9688
|
+
return;
|
|
9689
|
+
}
|
|
9690
|
+
if (action === "disable") {
|
|
9691
|
+
writeOverrides({
|
|
9692
|
+
enabled: overrides.enabled.filter((id) => id !== featureId),
|
|
9693
|
+
disabled: normalizeFeatureList([...overrides.disabled, featureId])
|
|
9694
|
+
});
|
|
9695
|
+
return;
|
|
9696
|
+
}
|
|
9697
|
+
if (action === "reset") {
|
|
9698
|
+
writeOverrides({
|
|
9699
|
+
enabled: overrides.enabled.filter((id) => id !== featureId),
|
|
9700
|
+
disabled: overrides.disabled.filter((id) => id !== featureId)
|
|
9701
|
+
});
|
|
9702
|
+
return;
|
|
9703
|
+
}
|
|
9704
|
+
if (action === "explain") {
|
|
9705
|
+
const defaultReason = OFF_BY_DEFAULT_FEATURES.get(featureId);
|
|
9706
|
+
const override = overrides.enabled.includes(featureId) ? "enabled" : overrides.disabled.includes(featureId) ? "disabled" : "none";
|
|
9707
|
+
const defaultState = defaultReason ? "off" : "on";
|
|
9708
|
+
const effective = override === "enabled" ? "enabled" : override === "disabled" ? "disabled" : defaultState === "on" ? "enabled" : "disabled";
|
|
9709
|
+
console.log(`Feature: ${featureId}`);
|
|
9710
|
+
console.log(`Kind: ${featureKind(featureId)}`);
|
|
9711
|
+
console.log(`Default: ${defaultState}${defaultReason ? ` (${defaultReason})` : ""}`);
|
|
9712
|
+
console.log(`Override: ${override}`);
|
|
9713
|
+
console.log(`Effective after restart: ${effective}`);
|
|
9714
|
+
return;
|
|
9715
|
+
}
|
|
9716
|
+
throw new Error(`Unknown server features command '${action}'. Use list, enable, disable, reset, or explain.`);
|
|
9717
|
+
}
|
|
9718
|
+
async function serverAiModelsAdd(flags) {
|
|
9719
|
+
const installPath = resolveServerPath(flags);
|
|
9720
|
+
const installMode = getInstallMode(installPath);
|
|
9721
|
+
const overlayPath = localModelsOverlayPath(installPath, installMode);
|
|
9722
|
+
const runtimeInput = typeof flags.runtime === "string" ? flags.runtime : await promptChoice(
|
|
9723
|
+
"Which local runtime should OpenMates use?",
|
|
9724
|
+
[
|
|
9725
|
+
{ value: "ollama", label: "Ollama" },
|
|
9726
|
+
{ value: "lm-studio", label: "LM Studio" },
|
|
9727
|
+
{ value: "custom", label: "Custom OpenAI-compatible API" }
|
|
9728
|
+
],
|
|
9729
|
+
"ollama"
|
|
9730
|
+
);
|
|
9731
|
+
const runtimeKey = normalizeRuntimeKey(runtimeInput);
|
|
9732
|
+
const runtime = LOCAL_MODEL_RUNTIME_DEFAULTS[runtimeKey];
|
|
9733
|
+
const baseUrl = typeof flags["base-url"] === "string" ? flags["base-url"] : await promptText("OpenAI-compatible base URL", runtime.baseUrl);
|
|
9734
|
+
const apiKey = typeof flags["api-key"] === "string" ? flags["api-key"] : runtimeKey === "custom" ? await promptText("API key", runtime.apiKey) : runtime.apiKey;
|
|
9735
|
+
let availableModels = [];
|
|
9736
|
+
try {
|
|
9737
|
+
availableModels = await fetchLocalModelIds(baseUrl, apiKey);
|
|
9738
|
+
} catch (error) {
|
|
9739
|
+
console.error(`Could not fetch /v1/models: ${error instanceof Error ? error.message : String(error)}`);
|
|
9740
|
+
}
|
|
9741
|
+
const rawModelId = typeof flags.model === "string" ? flags.model : availableModels.length ? await promptChoice(
|
|
9742
|
+
"Which installed model should OpenMates add?",
|
|
9743
|
+
[...availableModels.map((id) => ({ value: id, label: id })), { value: "manual", label: "Enter manually" }],
|
|
9744
|
+
availableModels[0]
|
|
9745
|
+
) : await promptText("Local model ID");
|
|
9746
|
+
const serverModelId = rawModelId === "manual" ? await promptText("Local model ID") : rawModelId;
|
|
9747
|
+
if (!serverModelId) throw new Error("Model ID is required.");
|
|
9748
|
+
const inferredCreator = inferCreatorId(serverModelId);
|
|
9749
|
+
const creatorInput = typeof flags.creator === "string" ? flags.creator : await promptChoice(
|
|
9750
|
+
"Who created the model?",
|
|
9751
|
+
MODEL_CREATOR_OPTIONS.map((option) => ({ value: option.id, label: option.name })),
|
|
9752
|
+
inferredCreator
|
|
9753
|
+
);
|
|
9754
|
+
const providerId = creatorInput === "custom" ? normalizeCreatorId(await promptText("Custom creator/provider ID", "custom_local")) : normalizeCreatorId(creatorInput);
|
|
9755
|
+
const providerName = providerId === creatorInput ? creatorDisplayName(providerId) : providerId;
|
|
9756
|
+
const internalModelId = typeof flags.id === "string" ? sanitizeModelId(flags.id) : `${sanitizeModelId(serverModelId)}-local`;
|
|
9757
|
+
const displayName = typeof flags.name === "string" ? flags.name : await promptText("Display name", serverModelId);
|
|
9758
|
+
const supportsImages = boolFromFlag(flags.images ?? flags.image ?? flags.vision, false);
|
|
9759
|
+
const supportsTools = boolFromFlag(flags.tools ?? flags["tool-use"], false);
|
|
9760
|
+
const contextWindowInput = typeof flags["context-window"] === "string" ? flags["context-window"] : await promptText("Context window tokens", "32768");
|
|
9761
|
+
const contextWindow = Number.parseInt(contextWindowInput, 10) || 32768;
|
|
9762
|
+
if (flags["skip-test"] !== true) {
|
|
9763
|
+
console.error(`Testing ${runtime.label} model '${serverModelId}'...`);
|
|
9764
|
+
const testOutput = await testLocalModel(baseUrl, apiKey, serverModelId);
|
|
9765
|
+
if (!testOutput) throw new Error("Local model test returned no content. Not saving model.");
|
|
9766
|
+
console.error(`Test response: ${testOutput}`);
|
|
9767
|
+
}
|
|
9768
|
+
const model = {
|
|
9769
|
+
id: internalModelId,
|
|
9770
|
+
name: displayName,
|
|
9771
|
+
description: `${displayName} served locally through ${runtime.label}.`,
|
|
9772
|
+
country_origin: "local",
|
|
9773
|
+
for_app_skill: "ai.ask",
|
|
9774
|
+
allow_auto_select: false,
|
|
9775
|
+
local: true,
|
|
9776
|
+
self_hosted: true,
|
|
9777
|
+
input_types: supportsImages ? ["text", "image"] : ["text"],
|
|
9778
|
+
output_types: ["text"],
|
|
9779
|
+
default_server: runtime.serverId,
|
|
9780
|
+
servers: [
|
|
9781
|
+
{
|
|
9782
|
+
id: runtime.serverId,
|
|
9783
|
+
name: runtime.label,
|
|
9784
|
+
model_id: serverModelId,
|
|
9785
|
+
region: "local",
|
|
9786
|
+
base_url: baseUrl.replace(/\/+$/, ""),
|
|
9787
|
+
api_key: apiKey,
|
|
9788
|
+
supports_tools: supportsTools
|
|
9789
|
+
}
|
|
9790
|
+
],
|
|
9791
|
+
pricing: { fixed: { credits: 0 } },
|
|
9792
|
+
costs: {
|
|
9793
|
+
input_per_million_token: { price: 0, currency: "USD", max_context: contextWindow },
|
|
9794
|
+
output_per_million_token: { price: 0, currency: "USD", max_context: contextWindow }
|
|
9795
|
+
},
|
|
9796
|
+
features: {
|
|
9797
|
+
streaming: true,
|
|
9798
|
+
tool_use: supportsTools,
|
|
9799
|
+
max_context: contextWindow
|
|
9800
|
+
}
|
|
9801
|
+
};
|
|
9802
|
+
upsertLocalModel(overlayPath, providerId, providerName, model);
|
|
9803
|
+
if (installMode === "image") ensureImageRuntimeConfig(installPath);
|
|
9804
|
+
if (flags.json === true) {
|
|
9805
|
+
printJson({ command: "server ai models add", status: "success", model: `${providerId}/${internalModelId}`, overlayPath });
|
|
9806
|
+
} else {
|
|
9807
|
+
console.log(`Added local model: ${providerId}/${internalModelId}`);
|
|
9808
|
+
console.log(`Updated ${overlayPath}`);
|
|
9809
|
+
console.log("Restart the server for model changes to take effect: openmates server restart");
|
|
9810
|
+
console.log("Self-hosted local models charge 0 credits; token usage may still be recorded in usage history.");
|
|
9811
|
+
}
|
|
9812
|
+
}
|
|
9813
|
+
async function serverAiModelsList(flags) {
|
|
9814
|
+
const installPath = resolveServerPath(flags);
|
|
9815
|
+
const overlayPath = localModelsOverlayPath(installPath);
|
|
9816
|
+
const overlay = readLocalModelOverlay(overlayPath);
|
|
9817
|
+
const models = localModelsFromOverlay(overlay).map(({ providerId, model }) => ({
|
|
9818
|
+
id: `${providerId}/${String(model.id ?? "")}`,
|
|
9819
|
+
name: model.name ?? model.id,
|
|
9820
|
+
server: Array.isArray(model.servers) ? model.servers[0]?.id : void 0,
|
|
9821
|
+
serverModelId: Array.isArray(model.servers) ? model.servers[0]?.model_id : void 0
|
|
9822
|
+
}));
|
|
9823
|
+
if (flags.json === true) {
|
|
9824
|
+
printJson({ overlayPath, models });
|
|
9825
|
+
return;
|
|
9826
|
+
}
|
|
9827
|
+
if (!models.length) {
|
|
9828
|
+
console.log("No local AI models configured. Add one with: openmates server ai models add");
|
|
9829
|
+
return;
|
|
9830
|
+
}
|
|
9831
|
+
console.log("Local AI models:");
|
|
9832
|
+
for (const model of models) {
|
|
9833
|
+
console.log(` ${model.id} (${model.server}: ${model.serverModelId})`);
|
|
9834
|
+
}
|
|
9835
|
+
}
|
|
9836
|
+
async function serverAiModelsTest(rest, flags) {
|
|
9837
|
+
const installPath = resolveServerPath(flags);
|
|
9838
|
+
const overlayPath = localModelsOverlayPath(installPath);
|
|
9839
|
+
const fullModelId = rest[0];
|
|
9840
|
+
if (!fullModelId || !fullModelId.includes("/")) throw new Error("Usage: openmates server ai models test <provider/model-id>");
|
|
9841
|
+
const [providerId, modelId] = fullModelId.split("/", 2);
|
|
9842
|
+
const overlay = readLocalModelOverlay(overlayPath);
|
|
9843
|
+
const provider = overlay.providers.find((item) => item.provider_id === providerId);
|
|
9844
|
+
const model = provider?.models.find((item) => item.id === modelId);
|
|
9845
|
+
const server = Array.isArray(model?.servers) ? model.servers[0] : void 0;
|
|
9846
|
+
if (!server) throw new Error(`Local model not found in ${overlayPath}: ${fullModelId}`);
|
|
9847
|
+
const baseUrl = String(server.base_url ?? "");
|
|
9848
|
+
const apiKey = String(server.api_key ?? "local");
|
|
9849
|
+
const serverModelId = String(server.model_id ?? "");
|
|
9850
|
+
const output = await testLocalModel(baseUrl, apiKey, serverModelId);
|
|
9851
|
+
if (flags.json === true) {
|
|
9852
|
+
printJson({ model: fullModelId, status: "success", output });
|
|
9853
|
+
} else {
|
|
9854
|
+
console.log(`Model test succeeded: ${fullModelId}`);
|
|
9855
|
+
console.log(`Response: ${output}`);
|
|
9856
|
+
}
|
|
9857
|
+
}
|
|
9858
|
+
async function serverAiModelsRemove(rest, flags) {
|
|
9859
|
+
const installPath = resolveServerPath(flags);
|
|
9860
|
+
const overlayPath = localModelsOverlayPath(installPath);
|
|
9861
|
+
const fullModelId = rest[0];
|
|
9862
|
+
if (!fullModelId) throw new Error("Usage: openmates server ai models remove <provider/model-id>");
|
|
9863
|
+
const removed = removeLocalModel(overlayPath, fullModelId);
|
|
9864
|
+
if (flags.json === true) {
|
|
9865
|
+
printJson({ command: "server ai models remove", status: removed ? "success" : "not_found", model: fullModelId });
|
|
9866
|
+
} else if (removed) {
|
|
9867
|
+
console.log(`Removed local model: ${fullModelId}`);
|
|
9868
|
+
console.log("Restart the server for model changes to take effect: openmates server restart");
|
|
9869
|
+
} else {
|
|
9870
|
+
console.log(`Local model not found: ${fullModelId}`);
|
|
9871
|
+
}
|
|
9872
|
+
}
|
|
9873
|
+
async function serverAi(rest, flags) {
|
|
9874
|
+
const area = rest[0];
|
|
9875
|
+
const action = rest[1] ?? "list";
|
|
9876
|
+
const args = rest.slice(2);
|
|
9877
|
+
if (area !== "models") throw new Error("Usage: openmates server ai models <add|list|test|remove>");
|
|
9878
|
+
switch (action) {
|
|
9879
|
+
case "add":
|
|
9880
|
+
return serverAiModelsAdd(flags);
|
|
9881
|
+
case "list":
|
|
9882
|
+
return serverAiModelsList(flags);
|
|
9883
|
+
case "test":
|
|
9884
|
+
return serverAiModelsTest(args, flags);
|
|
9885
|
+
case "remove":
|
|
9886
|
+
return serverAiModelsRemove(args, flags);
|
|
9887
|
+
default:
|
|
9888
|
+
throw new Error(`Unknown server ai models command '${action}'. Use add, list, test, or remove.`);
|
|
9889
|
+
}
|
|
9890
|
+
}
|
|
9327
9891
|
async function serverUninstall(flags) {
|
|
9328
9892
|
requireDocker();
|
|
9329
9893
|
const installPath = resolveServerPath(flags);
|
|
@@ -9391,6 +9955,7 @@ Commands:
|
|
|
9391
9955
|
logs Display server logs
|
|
9392
9956
|
update Update to latest version (pull images, or git pull + rebuild for source installs)
|
|
9393
9957
|
make-admin Grant admin privileges to a user
|
|
9958
|
+
ai Manage self-hosted local AI models
|
|
9394
9959
|
reset Reset server data (requires confirmation)
|
|
9395
9960
|
uninstall Completely remove OpenMates (requires confirmation)
|
|
9396
9961
|
|
|
@@ -9435,11 +10000,27 @@ Command Options:
|
|
|
9435
10000
|
make-admin:
|
|
9436
10001
|
openmates server make-admin <email>
|
|
9437
10002
|
|
|
10003
|
+
features:
|
|
10004
|
+
openmates server features list
|
|
10005
|
+
openmates server features enable <feature-id>
|
|
10006
|
+
openmates server features disable <feature-id>
|
|
10007
|
+
openmates server features reset <feature-id>
|
|
10008
|
+
openmates server features explain <feature-id>
|
|
10009
|
+
|
|
10010
|
+
ai models:
|
|
10011
|
+
openmates server ai models add
|
|
10012
|
+
openmates server ai models list
|
|
10013
|
+
openmates server ai models test <provider/model-id>
|
|
10014
|
+
openmates server ai models remove <provider/model-id>
|
|
10015
|
+
|
|
9438
10016
|
Examples:
|
|
9439
10017
|
openmates server install
|
|
9440
10018
|
openmates server start --with-overrides
|
|
9441
10019
|
openmates server logs --container api --follow
|
|
9442
10020
|
openmates server make-admin user@example.com
|
|
10021
|
+
openmates server features disable app:videos
|
|
10022
|
+
openmates server ai models add
|
|
10023
|
+
openmates server features enable embed:code:application
|
|
9443
10024
|
openmates server update
|
|
9444
10025
|
openmates server update --dry-run
|
|
9445
10026
|
openmates server update --image-tag v0.12.0-alpha.1
|
|
@@ -9471,6 +10052,10 @@ async function handleServer(subcommand, rest, flags) {
|
|
|
9471
10052
|
return serverReset(flags);
|
|
9472
10053
|
case "make-admin":
|
|
9473
10054
|
return serverMakeAdmin(rest, flags);
|
|
10055
|
+
case "ai":
|
|
10056
|
+
return serverAi(rest, flags);
|
|
10057
|
+
case "features":
|
|
10058
|
+
return serverFeatures(rest, flags);
|
|
9474
10059
|
case "uninstall":
|
|
9475
10060
|
return serverUninstall(flags);
|
|
9476
10061
|
default:
|
|
@@ -20671,7 +21256,7 @@ line_count: 37`,
|
|
|
20671
21256
|
metadata: {
|
|
20672
21257
|
featured: true,
|
|
20673
21258
|
order: 2,
|
|
20674
|
-
content_embed_examples: [
|
|
21259
|
+
content_embed_examples: []
|
|
20675
21260
|
}
|
|
20676
21261
|
};
|
|
20677
21262
|
|
|
@@ -26232,6 +26817,9 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
|
|
|
26232
26817
|
anonymous_free_usage: {
|
|
26233
26818
|
feature_notice: {
|
|
26234
26819
|
text: "You are using free anonymous credits. File uploads, memories, chat sync, and some other features require creating an account."
|
|
26820
|
+
},
|
|
26821
|
+
daily_credits_exhausted: {
|
|
26822
|
+
text: "You used up your free daily credits. Sign up & buy credits to make full use of OpenMates."
|
|
26235
26823
|
}
|
|
26236
26824
|
},
|
|
26237
26825
|
interactive_question_failed: {
|
|
@@ -27900,6 +28488,50 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
|
|
|
27900
28488
|
account_created: {
|
|
27901
28489
|
text: "Account created"
|
|
27902
28490
|
},
|
|
28491
|
+
account_created_second_login_title: {
|
|
28492
|
+
text: "Add a second login method"
|
|
28493
|
+
},
|
|
28494
|
+
account_created_second_login_info: {
|
|
28495
|
+
text: "If you signed up with a passkey, add password plus 2FA as a backup. If you signed up with password plus 2FA, add a passkey for faster secure login."
|
|
28496
|
+
},
|
|
28497
|
+
existing_account: {
|
|
28498
|
+
subject: {
|
|
28499
|
+
text: "An OpenMates account already exists for this email"
|
|
28500
|
+
},
|
|
28501
|
+
title: {
|
|
28502
|
+
text: "An account with your email already exists"
|
|
28503
|
+
},
|
|
28504
|
+
intro: {
|
|
28505
|
+
text: "Someone tried to create a new OpenMates account with this email address, but this email is already connected to an existing account."
|
|
28506
|
+
},
|
|
28507
|
+
saved_logins_title: {
|
|
28508
|
+
text: "Check your saved logins"
|
|
28509
|
+
},
|
|
28510
|
+
saved_logins_body: {
|
|
28511
|
+
text: "Your browser, password manager, or device may already have the login saved. Look for OpenMates in saved passwords or passkeys before creating a new account."
|
|
28512
|
+
},
|
|
28513
|
+
login_methods_title: {
|
|
28514
|
+
text: "Try your login methods"
|
|
28515
|
+
},
|
|
28516
|
+
login_methods_body: {
|
|
28517
|
+
text: "You can log in with a passkey, or with your password plus 2FA. If you set up backup codes, keep them ready for the 2FA step."
|
|
28518
|
+
},
|
|
28519
|
+
login_button: {
|
|
28520
|
+
text: "Log in to OpenMates"
|
|
28521
|
+
},
|
|
28522
|
+
recovery_key_title: {
|
|
28523
|
+
text: "Have your recovery key?"
|
|
28524
|
+
},
|
|
28525
|
+
recovery_key_body: {
|
|
28526
|
+
text: "If you do not remember your password or passkey but still have your recovery key, use recovery key login. It preserves your encrypted chats and account data."
|
|
28527
|
+
},
|
|
28528
|
+
recovery_button: {
|
|
28529
|
+
text: "Open login and use recovery key"
|
|
28530
|
+
},
|
|
28531
|
+
reset_warning: {
|
|
28532
|
+
text: "If you lost every login method and your recovery key, account reset is the last resort. Because OpenMates encrypts your data, resetting the account can permanently delete encrypted chats, memories, app settings, embeds, passkeys, and API keys."
|
|
28533
|
+
}
|
|
28534
|
+
},
|
|
27903
28535
|
password_security_reminder: {
|
|
27904
28536
|
subject: {
|
|
27905
28537
|
text: "Action needed to secure your OpenMates account"
|
|
@@ -28129,10 +28761,7 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
|
|
|
28129
28761
|
text: "Welcome to OpenMates!"
|
|
28130
28762
|
},
|
|
28131
28763
|
complete_signup_info: {
|
|
28132
|
-
text: "
|
|
28133
|
-
},
|
|
28134
|
-
auto_delete_warning: {
|
|
28135
|
-
text: "Please note: Accounts that haven't completed the signup process will be automatically deleted after 7 days."
|
|
28764
|
+
text: "Your account is ready. Here are a few helpful next steps to protect your access and keep a copy of your data."
|
|
28136
28765
|
},
|
|
28137
28766
|
want_to_delete_account: {
|
|
28138
28767
|
text: "Want to delete your account?"
|
|
@@ -28920,6 +29549,9 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
|
|
|
28920
29549
|
}
|
|
28921
29550
|
},
|
|
28922
29551
|
embeds: {
|
|
29552
|
+
learning_mode_shortened_notice: {
|
|
29553
|
+
text: "Shortened since Learning Mode is active."
|
|
29554
|
+
},
|
|
28923
29555
|
weather: {
|
|
28924
29556
|
rain_radar: {
|
|
28925
29557
|
no_rain: {
|
|
@@ -29262,6 +29894,15 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
|
|
|
29262
29894
|
copy_failed: {
|
|
29263
29895
|
text: "Failed to copy to clipboard"
|
|
29264
29896
|
},
|
|
29897
|
+
code_file_downloaded: {
|
|
29898
|
+
text: "Code file downloaded successfully"
|
|
29899
|
+
},
|
|
29900
|
+
code_file_download_failed: {
|
|
29901
|
+
text: "Failed to download code file"
|
|
29902
|
+
},
|
|
29903
|
+
action_failed: {
|
|
29904
|
+
text: "Failed to perform action"
|
|
29905
|
+
},
|
|
29265
29906
|
download_itinerary: {
|
|
29266
29907
|
text: "Download itinerary"
|
|
29267
29908
|
},
|
|
@@ -37618,6 +38259,99 @@ As of mid-2026, the severe supply shocks from the 2024\u20132025 avian flu have
|
|
|
37618
38259
|
incognito: {
|
|
37619
38260
|
text: "Incognito"
|
|
37620
38261
|
},
|
|
38262
|
+
learning_mode: {
|
|
38263
|
+
text: "Learning"
|
|
38264
|
+
},
|
|
38265
|
+
learning_mode_active: {
|
|
38266
|
+
text: "Active for all chats on this account"
|
|
38267
|
+
},
|
|
38268
|
+
learning_mode_inactive: {
|
|
38269
|
+
text: "Teach step-by-step instead of giving complete answers"
|
|
38270
|
+
},
|
|
38271
|
+
learning_mode_load_error: {
|
|
38272
|
+
text: "Could not load Learning Mode status."
|
|
38273
|
+
},
|
|
38274
|
+
learning_mode_save_error: {
|
|
38275
|
+
text: "Could not update Learning Mode."
|
|
38276
|
+
},
|
|
38277
|
+
learning_mode_enabled: {
|
|
38278
|
+
text: "Learning Mode enabled."
|
|
38279
|
+
},
|
|
38280
|
+
learning_mode_disabled: {
|
|
38281
|
+
text: "Learning Mode disabled."
|
|
38282
|
+
},
|
|
38283
|
+
learning_mode_age_group_prompt: {
|
|
38284
|
+
text: "Learner age group: under_10, 10_12, 13_15, 16_18, or adult"
|
|
38285
|
+
},
|
|
38286
|
+
learning_mode_enable_passcode_prompt: {
|
|
38287
|
+
text: "Set a Learning Mode passcode."
|
|
38288
|
+
},
|
|
38289
|
+
learning_mode_disable_passcode_prompt: {
|
|
38290
|
+
text: "Enter the Learning Mode passcode to disable it."
|
|
38291
|
+
},
|
|
38292
|
+
learning_mode_invalid_age_group: {
|
|
38293
|
+
text: "Invalid age group. Use under_10, 10_12, 13_15, 16_18, or adult."
|
|
38294
|
+
},
|
|
38295
|
+
learning_mode_enable_description: {
|
|
38296
|
+
text: "Choose the learner age group and set a passcode. Learning Mode will apply to every chat on this account."
|
|
38297
|
+
},
|
|
38298
|
+
learning_mode_disable_description: {
|
|
38299
|
+
text: "Enter the Learning Mode passcode to turn it off for this account."
|
|
38300
|
+
},
|
|
38301
|
+
learning_mode_guest_description: {
|
|
38302
|
+
text: "Choose the learner age group. For guests, Learning Mode lasts only for this browser session and has no passcode."
|
|
38303
|
+
},
|
|
38304
|
+
learning_mode_active_detail: {
|
|
38305
|
+
text: "Learning Mode is active for every chat on this account. Answers stay teaching-first and selected tools are restricted."
|
|
38306
|
+
},
|
|
38307
|
+
learning_mode_inactive_detail: {
|
|
38308
|
+
text: "When enabled, OpenMates guides the learner step by step instead of giving complete answers."
|
|
38309
|
+
},
|
|
38310
|
+
learning_mode_guest_active_detail: {
|
|
38311
|
+
text: "Learning Mode is active for anonymous chats in this browser session. Create an account to lock it with a passcode across devices."
|
|
38312
|
+
},
|
|
38313
|
+
learning_mode_guest_inactive_detail: {
|
|
38314
|
+
text: "Guests can try Learning Mode for anonymous chats in this browser session. Create an account to lock it with a passcode."
|
|
38315
|
+
},
|
|
38316
|
+
learning_mode_age_group_label: {
|
|
38317
|
+
text: "Learner age group"
|
|
38318
|
+
},
|
|
38319
|
+
learning_mode_age_under_10: {
|
|
38320
|
+
text: "Under 10"
|
|
38321
|
+
},
|
|
38322
|
+
learning_mode_age_10_12: {
|
|
38323
|
+
text: "10 to 12"
|
|
38324
|
+
},
|
|
38325
|
+
learning_mode_age_13_15: {
|
|
38326
|
+
text: "13 to 15"
|
|
38327
|
+
},
|
|
38328
|
+
learning_mode_age_16_18: {
|
|
38329
|
+
text: "16 to 18"
|
|
38330
|
+
},
|
|
38331
|
+
learning_mode_age_adult: {
|
|
38332
|
+
text: "Adult"
|
|
38333
|
+
},
|
|
38334
|
+
learning_mode_enable_passcode_label: {
|
|
38335
|
+
text: "Set passcode"
|
|
38336
|
+
},
|
|
38337
|
+
learning_mode_disable_passcode_label: {
|
|
38338
|
+
text: "Enter passcode"
|
|
38339
|
+
},
|
|
38340
|
+
learning_mode_enable_passcode_placeholder: {
|
|
38341
|
+
text: "Create a passcode"
|
|
38342
|
+
},
|
|
38343
|
+
learning_mode_disable_passcode_placeholder: {
|
|
38344
|
+
text: "Learning Mode passcode"
|
|
38345
|
+
},
|
|
38346
|
+
learning_mode_passcode_required: {
|
|
38347
|
+
text: "Enter a passcode to continue."
|
|
38348
|
+
},
|
|
38349
|
+
learning_mode_enable_button: {
|
|
38350
|
+
text: "Start Learning Mode"
|
|
38351
|
+
},
|
|
38352
|
+
learning_mode_disable_button: {
|
|
38353
|
+
text: "Turn off Learning Mode"
|
|
38354
|
+
},
|
|
37621
38355
|
incognito_explainer_description: {
|
|
37622
38356
|
text: "Incognito mode applies only to new chats you create. New chats created while incognito mode is active are not synced across devices, not stored on the server, and not cached. These chats exist only in your current browser session. Existing chats remain unchanged."
|
|
37623
38357
|
},
|
|
@@ -41455,7 +42189,12 @@ function isInteractiveQuestionPayload(value) {
|
|
|
41455
42189
|
if (!isQuestionType(payload.type)) return false;
|
|
41456
42190
|
if (typeof payload.id !== "string" || payload.id.trim().length === 0) return false;
|
|
41457
42191
|
if (payload.type === "choice") {
|
|
41458
|
-
|
|
42192
|
+
if (typeof payload.question !== "string" || payload.question.trim().length === 0) return false;
|
|
42193
|
+
if (!Array.isArray(payload.options) || payload.options.length === 0) return false;
|
|
42194
|
+
if (payload.custom_option_id !== void 0) {
|
|
42195
|
+
return typeof payload.custom_option_id === "string" && payload.options.some((option) => option.id === payload.custom_option_id);
|
|
42196
|
+
}
|
|
42197
|
+
return true;
|
|
41459
42198
|
}
|
|
41460
42199
|
if (payload.type === "input") return Array.isArray(payload.fields) && payload.fields.length > 0;
|
|
41461
42200
|
if (payload.type === "slider") {
|
|
@@ -41463,13 +42202,27 @@ function isInteractiveQuestionPayload(value) {
|
|
|
41463
42202
|
}
|
|
41464
42203
|
if (payload.type === "swipe") return Array.isArray(payload.cards) && payload.cards.length > 0;
|
|
41465
42204
|
if (payload.type === "rating") {
|
|
41466
|
-
return typeof payload.question === "string" && payload.question.trim().length > 0 && (typeof payload.max === "number" || typeof payload.scale === "number");
|
|
42205
|
+
return typeof payload.question === "string" && payload.question.trim().length > 0 && (typeof payload.max_stars === "number" || typeof payload.max === "number" || typeof payload.scale === "number");
|
|
41467
42206
|
}
|
|
41468
42207
|
return false;
|
|
41469
42208
|
}
|
|
41470
42209
|
function isQuestionType(value) {
|
|
41471
42210
|
return value === "choice" || value === "input" || value === "slider" || value === "swipe" || value === "rating";
|
|
41472
42211
|
}
|
|
42212
|
+
function formatInteractiveQuestionAnswer(question, answer) {
|
|
42213
|
+
const responsePayload = buildResponsePayload(question, answer);
|
|
42214
|
+
const displayText = buildDisplayText(question, answer);
|
|
42215
|
+
const protocol = JSON.stringify(responsePayload, null, 2);
|
|
42216
|
+
return {
|
|
42217
|
+
displayText,
|
|
42218
|
+
messageContent: `${displayText}
|
|
42219
|
+
|
|
42220
|
+
\`\`\`interactive_response
|
|
42221
|
+
${protocol}
|
|
42222
|
+
\`\`\``,
|
|
42223
|
+
responsePayload
|
|
42224
|
+
};
|
|
42225
|
+
}
|
|
41473
42226
|
function toWaitingForUserResult(params) {
|
|
41474
42227
|
return {
|
|
41475
42228
|
status: "waiting_for_user",
|
|
@@ -41479,6 +42232,50 @@ function toWaitingForUserResult(params) {
|
|
|
41479
42232
|
question: params.question
|
|
41480
42233
|
};
|
|
41481
42234
|
}
|
|
42235
|
+
function buildResponsePayload(question, answer) {
|
|
42236
|
+
return {
|
|
42237
|
+
id: question.id,
|
|
42238
|
+
...answer
|
|
42239
|
+
};
|
|
42240
|
+
}
|
|
42241
|
+
function buildDisplayText(question, answer) {
|
|
42242
|
+
if (question.type === "choice") {
|
|
42243
|
+
const selection = Array.isArray(answer.selection) ? answer.selection.map(String) : [];
|
|
42244
|
+
const optionsById = new Map((question.options ?? []).map((option) => [option.id, option.text]));
|
|
42245
|
+
const customAnswer = answer.custom_answer == null ? "" : String(answer.custom_answer).trim();
|
|
42246
|
+
return selection.map((id) => customAnswer && isCustomChoiceOption(question, id) ? customAnswer : optionsById.get(id) ?? id).join(", ");
|
|
42247
|
+
}
|
|
42248
|
+
if (question.type === "input") {
|
|
42249
|
+
const values = answer.inputs && typeof answer.inputs === "object" ? answer.inputs : answer.values && typeof answer.values === "object" ? answer.values : answer;
|
|
42250
|
+
return Object.entries(values).filter(([key]) => key !== "id").map(([, value]) => String(value)).filter(Boolean).join("\n");
|
|
42251
|
+
}
|
|
42252
|
+
if (question.type === "slider") {
|
|
42253
|
+
return answer.value == null ? "" : String(answer.value);
|
|
42254
|
+
}
|
|
42255
|
+
if (question.type === "swipe") {
|
|
42256
|
+
const liked = Array.isArray(answer.liked) ? answer.liked.map(String) : [];
|
|
42257
|
+
const cardsById = new Map((question.cards ?? []).map((card) => [card.id, card.text]));
|
|
42258
|
+
return liked.map((id) => cardsById.get(id) ?? id).join(", ");
|
|
42259
|
+
}
|
|
42260
|
+
if (question.type === "rating") {
|
|
42261
|
+
const rating = answer.rating == null ? "" : String(answer.rating);
|
|
42262
|
+
const comment = answer.comment == null ? "" : String(answer.comment).trim();
|
|
42263
|
+
return [rating, comment].filter(Boolean).join("\n");
|
|
42264
|
+
}
|
|
42265
|
+
return "";
|
|
42266
|
+
}
|
|
42267
|
+
function isCustomChoiceOption(question, optionId) {
|
|
42268
|
+
if (question.custom_option_id) return optionId === question.custom_option_id;
|
|
42269
|
+
const optionText = (question.options ?? []).find((option) => option.id === optionId)?.text.trim().toLowerCase() ?? "";
|
|
42270
|
+
return [
|
|
42271
|
+
"i give you my own answer",
|
|
42272
|
+
"my own answer",
|
|
42273
|
+
"own answer",
|
|
42274
|
+
"custom answer",
|
|
42275
|
+
"something else",
|
|
42276
|
+
"other"
|
|
42277
|
+
].some((pattern) => optionText === pattern || optionText.includes(pattern));
|
|
42278
|
+
}
|
|
41482
42279
|
|
|
41483
42280
|
// src/feedback.ts
|
|
41484
42281
|
var ASSISTANT_FEEDBACK_THANKS = "Thanks for the feedback!";
|
|
@@ -41502,6 +42299,1022 @@ function buildAssistantFeedbackDecision(rating) {
|
|
|
41502
42299
|
};
|
|
41503
42300
|
}
|
|
41504
42301
|
|
|
42302
|
+
// src/benchmark.ts
|
|
42303
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
42304
|
+
import { existsSync as existsSync6, mkdtempSync, readFileSync as readFileSync6, readdirSync, writeFileSync as writeFileSync4 } from "fs";
|
|
42305
|
+
import { tmpdir } from "os";
|
|
42306
|
+
import { dirname as dirname2, join as join4, resolve as resolve5 } from "path";
|
|
42307
|
+
import { fileURLToPath } from "url";
|
|
42308
|
+
var DEFAULT_JUDGE_MODEL = "google/gemini-3-flash-preview";
|
|
42309
|
+
var DEFAULT_EXTENSIVE_SIZE = 10;
|
|
42310
|
+
var DEFAULT_PARALLEL = 4;
|
|
42311
|
+
var FIXTURE_IMAGE_SVG = `<?xml version="1.0" encoding="UTF-8"?>
|
|
42312
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="800" viewBox="0 0 1200 800">
|
|
42313
|
+
<rect width="1200" height="800" fill="#d8ecff"/>
|
|
42314
|
+
<rect y="560" width="1200" height="240" fill="#d7c39a"/>
|
|
42315
|
+
<text x="600" y="88" text-anchor="middle" font-family="Arial, sans-serif" font-size="44" font-weight="700" fill="#23344d">Brandenburger Tor, Berlin</text>
|
|
42316
|
+
<g transform="translate(160 170)" fill="#c9aa6a" stroke="#5d4522" stroke-width="8">
|
|
42317
|
+
<rect x="80" y="160" width="800" height="58"/>
|
|
42318
|
+
<rect x="120" y="218" width="720" height="48"/>
|
|
42319
|
+
<rect x="150" y="266" width="660" height="42"/>
|
|
42320
|
+
<g fill="#d9bd7d">
|
|
42321
|
+
<rect x="170" y="308" width="54" height="250"/>
|
|
42322
|
+
<rect x="285" y="308" width="54" height="250"/>
|
|
42323
|
+
<rect x="400" y="308" width="54" height="250"/>
|
|
42324
|
+
<rect x="515" y="308" width="54" height="250"/>
|
|
42325
|
+
<rect x="630" y="308" width="54" height="250"/>
|
|
42326
|
+
<rect x="745" y="308" width="54" height="250"/>
|
|
42327
|
+
</g>
|
|
42328
|
+
<rect x="130" y="558" width="700" height="50"/>
|
|
42329
|
+
<path d="M480 30 C530 72 620 88 682 48 L720 84 C652 142 530 124 456 78 Z" fill="#3e6f5f"/>
|
|
42330
|
+
<circle cx="510" cy="92" r="22" fill="#3e6f5f"/>
|
|
42331
|
+
<circle cx="625" cy="92" r="22" fill="#3e6f5f"/>
|
|
42332
|
+
<path d="M565 38 l26 78 h-52 z" fill="#3e6f5f"/>
|
|
42333
|
+
</g>
|
|
42334
|
+
<text x="600" y="740" text-anchor="middle" font-family="Arial, sans-serif" font-size="32" fill="#23344d">Neoclassical gate with Quadriga on top</text>
|
|
42335
|
+
</svg>
|
|
42336
|
+
`;
|
|
42337
|
+
var QUICK_CASES = [
|
|
42338
|
+
{
|
|
42339
|
+
id: "quick-exact-token",
|
|
42340
|
+
suite: "quick",
|
|
42341
|
+
title: "Exact token smoke test",
|
|
42342
|
+
prompt: "Reply with exactly this token and no extra text: BENCHMARK_SMOKE_OK",
|
|
42343
|
+
complexity: "basic",
|
|
42344
|
+
category: "smoke",
|
|
42345
|
+
expectedIncludes: "BENCHMARK_SMOKE_OK",
|
|
42346
|
+
judge: true,
|
|
42347
|
+
estimatedInputTokens: 12e3,
|
|
42348
|
+
estimatedOutputTokens: 64
|
|
42349
|
+
},
|
|
42350
|
+
{
|
|
42351
|
+
id: "quick-arithmetic",
|
|
42352
|
+
suite: "quick",
|
|
42353
|
+
title: "Arithmetic direct answer",
|
|
42354
|
+
prompt: "Compute 19 * 23. Reply with only the integer result.",
|
|
42355
|
+
complexity: "basic",
|
|
42356
|
+
category: "math",
|
|
42357
|
+
expectedIncludes: "437",
|
|
42358
|
+
judge: true,
|
|
42359
|
+
estimatedInputTokens: 12e3,
|
|
42360
|
+
estimatedOutputTokens: 64
|
|
42361
|
+
},
|
|
42362
|
+
{
|
|
42363
|
+
id: "quick-code",
|
|
42364
|
+
suite: "quick",
|
|
42365
|
+
title: "Small code generation",
|
|
42366
|
+
prompt: "Write a TypeScript function isPalindrome(input: string): boolean that ignores spaces, punctuation, and case. Include only the function and one short usage example.",
|
|
42367
|
+
complexity: "medium",
|
|
42368
|
+
category: "coding",
|
|
42369
|
+
judge: true,
|
|
42370
|
+
estimatedInputTokens: 12200,
|
|
42371
|
+
estimatedOutputTokens: 650
|
|
42372
|
+
},
|
|
42373
|
+
{
|
|
42374
|
+
id: "quick-image-brandenburger-tor",
|
|
42375
|
+
suite: "quick",
|
|
42376
|
+
title: "Default image understanding",
|
|
42377
|
+
prompt: "Look at the attached image. What landmark is shown, when was it built, and who designed it? Answer in three concise bullet points.",
|
|
42378
|
+
complexity: "medium",
|
|
42379
|
+
category: "image",
|
|
42380
|
+
image: "default",
|
|
42381
|
+
expectedIncludes: "Brandenburg",
|
|
42382
|
+
judge: true,
|
|
42383
|
+
estimatedInputTokens: 13500,
|
|
42384
|
+
estimatedOutputTokens: 350
|
|
42385
|
+
},
|
|
42386
|
+
{
|
|
42387
|
+
id: "quick-followup-continuity",
|
|
42388
|
+
suite: "quick",
|
|
42389
|
+
title: "Short multi-turn continuity",
|
|
42390
|
+
prompt: "Create a three-step plan for evaluating whether a new AI model is ready for production use.",
|
|
42391
|
+
complexity: "medium",
|
|
42392
|
+
category: "multi_turn",
|
|
42393
|
+
judge: true,
|
|
42394
|
+
estimatedInputTokens: 14e3,
|
|
42395
|
+
estimatedOutputTokens: 900,
|
|
42396
|
+
followUps: [
|
|
42397
|
+
{ prompt: "Now make step 2 more concrete with two measurable checks." },
|
|
42398
|
+
{ prompt: "Summarize the final plan in one sentence." }
|
|
42399
|
+
]
|
|
42400
|
+
}
|
|
42401
|
+
];
|
|
42402
|
+
var EXTENSIVE_CASES = [
|
|
42403
|
+
...QUICK_CASES,
|
|
42404
|
+
{
|
|
42405
|
+
id: "extensive-coding-debug",
|
|
42406
|
+
suite: "extensive",
|
|
42407
|
+
title: "Debug a JavaScript bug",
|
|
42408
|
+
prompt: "A JavaScript function returns NaN when summing prices from [{price: '12.50'}, {price: undefined}]. Explain the bug and write a corrected function.",
|
|
42409
|
+
complexity: "medium",
|
|
42410
|
+
category: "coding",
|
|
42411
|
+
judge: true,
|
|
42412
|
+
estimatedInputTokens: 12300,
|
|
42413
|
+
estimatedOutputTokens: 850
|
|
42414
|
+
},
|
|
42415
|
+
{
|
|
42416
|
+
id: "extensive-coding-api-design",
|
|
42417
|
+
suite: "extensive",
|
|
42418
|
+
title: "Design a small API contract",
|
|
42419
|
+
prompt: "Design a minimal JSON API for creating and listing benchmark runs. Include request/response examples and one validation error.",
|
|
42420
|
+
complexity: "advanced",
|
|
42421
|
+
category: "coding",
|
|
42422
|
+
judge: true,
|
|
42423
|
+
estimatedInputTokens: 12300,
|
|
42424
|
+
estimatedOutputTokens: 1e3
|
|
42425
|
+
},
|
|
42426
|
+
{
|
|
42427
|
+
id: "extensive-reasoning-tradeoffs",
|
|
42428
|
+
suite: "extensive",
|
|
42429
|
+
title: "Reason about benchmark tradeoffs",
|
|
42430
|
+
prompt: "Compare deterministic assertions and LLM-as-judge evaluation for model benchmarks. Give two strengths and two risks for each.",
|
|
42431
|
+
complexity: "medium",
|
|
42432
|
+
category: "reasoning",
|
|
42433
|
+
judge: true,
|
|
42434
|
+
estimatedInputTokens: 12200,
|
|
42435
|
+
estimatedOutputTokens: 800
|
|
42436
|
+
},
|
|
42437
|
+
{
|
|
42438
|
+
id: "extensive-planning",
|
|
42439
|
+
suite: "extensive",
|
|
42440
|
+
title: "Operational rollout plan",
|
|
42441
|
+
prompt: "Create a rollout checklist for switching a production chatbot from one model to another. Include monitoring, rollback, and user-visible risk checks.",
|
|
42442
|
+
complexity: "advanced",
|
|
42443
|
+
category: "synthesis",
|
|
42444
|
+
judge: true,
|
|
42445
|
+
estimatedInputTokens: 12300,
|
|
42446
|
+
estimatedOutputTokens: 950
|
|
42447
|
+
},
|
|
42448
|
+
{
|
|
42449
|
+
id: "extensive-long-context-followup",
|
|
42450
|
+
suite: "extensive",
|
|
42451
|
+
title: "Prebuilt 20-message long chat follow-up",
|
|
42452
|
+
prompt: "Based on the earlier discussion, choose the best launch strategy and explain why in five bullets.",
|
|
42453
|
+
complexity: "advanced",
|
|
42454
|
+
category: "long_context",
|
|
42455
|
+
longContext: true,
|
|
42456
|
+
judge: true,
|
|
42457
|
+
estimatedInputTokens: 18500,
|
|
42458
|
+
estimatedOutputTokens: 900
|
|
42459
|
+
},
|
|
42460
|
+
{
|
|
42461
|
+
id: "extensive-policy-summary",
|
|
42462
|
+
suite: "extensive",
|
|
42463
|
+
title: "Policy summarization",
|
|
42464
|
+
prompt: "Summarize why privacy-preserving benchmark logs should avoid raw user prompts. Include a concrete safer alternative.",
|
|
42465
|
+
complexity: "medium",
|
|
42466
|
+
category: "reasoning",
|
|
42467
|
+
judge: true,
|
|
42468
|
+
estimatedInputTokens: 12200,
|
|
42469
|
+
estimatedOutputTokens: 650
|
|
42470
|
+
},
|
|
42471
|
+
{
|
|
42472
|
+
id: "extensive-structured-output",
|
|
42473
|
+
suite: "extensive",
|
|
42474
|
+
title: "Structured JSON output",
|
|
42475
|
+
prompt: "Return only JSON with keys risk, mitigation, and confidence for the risk: benchmark results are biased by prompt wording.",
|
|
42476
|
+
complexity: "medium",
|
|
42477
|
+
category: "synthesis",
|
|
42478
|
+
judge: true,
|
|
42479
|
+
estimatedInputTokens: 12200,
|
|
42480
|
+
estimatedOutputTokens: 350
|
|
42481
|
+
},
|
|
42482
|
+
{
|
|
42483
|
+
id: "extensive-creative-constraint",
|
|
42484
|
+
suite: "extensive",
|
|
42485
|
+
title: "Creative constrained response",
|
|
42486
|
+
prompt: "Write a six-line product note announcing model comparisons. Each line must be under 70 characters and avoid hype words like revolutionary or magical.",
|
|
42487
|
+
complexity: "medium",
|
|
42488
|
+
category: "synthesis",
|
|
42489
|
+
judge: true,
|
|
42490
|
+
estimatedInputTokens: 12200,
|
|
42491
|
+
estimatedOutputTokens: 500
|
|
42492
|
+
},
|
|
42493
|
+
{
|
|
42494
|
+
id: "extensive-data-reasoning",
|
|
42495
|
+
suite: "extensive",
|
|
42496
|
+
title: "Interpret metrics",
|
|
42497
|
+
prompt: "A benchmark has pass rates 8/10, 7/10, and 9/10 across three runs. Explain what you can and cannot conclude from this sample.",
|
|
42498
|
+
complexity: "medium",
|
|
42499
|
+
category: "reasoning",
|
|
42500
|
+
judge: true,
|
|
42501
|
+
estimatedInputTokens: 12200,
|
|
42502
|
+
estimatedOutputTokens: 600
|
|
42503
|
+
},
|
|
42504
|
+
{
|
|
42505
|
+
id: "extensive-security-review",
|
|
42506
|
+
suite: "extensive",
|
|
42507
|
+
title: "Security review",
|
|
42508
|
+
prompt: "Review this benchmark design for security risks: it logs prompts, outputs, model ids, and usage costs to a shared file. List risks and safer defaults.",
|
|
42509
|
+
complexity: "advanced",
|
|
42510
|
+
category: "reasoning",
|
|
42511
|
+
judge: true,
|
|
42512
|
+
estimatedInputTokens: 12300,
|
|
42513
|
+
estimatedOutputTokens: 850
|
|
42514
|
+
},
|
|
42515
|
+
{
|
|
42516
|
+
id: "extensive-followup-requirements",
|
|
42517
|
+
suite: "extensive",
|
|
42518
|
+
title: "Three-turn requirements refinement",
|
|
42519
|
+
prompt: "Draft acceptance criteria for a CLI benchmark comparison feature.",
|
|
42520
|
+
complexity: "advanced",
|
|
42521
|
+
category: "multi_turn",
|
|
42522
|
+
judge: true,
|
|
42523
|
+
estimatedInputTokens: 14500,
|
|
42524
|
+
estimatedOutputTokens: 1100,
|
|
42525
|
+
followUps: [
|
|
42526
|
+
{ prompt: "Add one criterion about cost estimation before live runs." },
|
|
42527
|
+
{ prompt: "Add one criterion about partial results after interruption." },
|
|
42528
|
+
{ prompt: "Now compress the criteria to five bullets total." }
|
|
42529
|
+
]
|
|
42530
|
+
},
|
|
42531
|
+
{
|
|
42532
|
+
id: "extensive-coding-tests",
|
|
42533
|
+
suite: "extensive",
|
|
42534
|
+
title: "Write tests for parser behavior",
|
|
42535
|
+
prompt: "Write Node.js test cases for a function parseSuites(value) that accepts quick, extensive, all, and comma-separated lists, and rejects unknown suites.",
|
|
42536
|
+
complexity: "medium",
|
|
42537
|
+
category: "coding",
|
|
42538
|
+
judge: true,
|
|
42539
|
+
estimatedInputTokens: 12300,
|
|
42540
|
+
estimatedOutputTokens: 950
|
|
42541
|
+
},
|
|
42542
|
+
{
|
|
42543
|
+
id: "extensive-coding-refactor",
|
|
42544
|
+
suite: "extensive",
|
|
42545
|
+
title: "Refactor duplicated code",
|
|
42546
|
+
prompt: "Given two duplicated TypeScript loops that build arrays of result objects, explain when to extract a helper and write the helper signature.",
|
|
42547
|
+
complexity: "medium",
|
|
42548
|
+
category: "coding",
|
|
42549
|
+
judge: true,
|
|
42550
|
+
estimatedInputTokens: 12300,
|
|
42551
|
+
estimatedOutputTokens: 750
|
|
42552
|
+
},
|
|
42553
|
+
{
|
|
42554
|
+
id: "extensive-comparison-analysis",
|
|
42555
|
+
suite: "extensive",
|
|
42556
|
+
title: "Compare two model outputs",
|
|
42557
|
+
prompt: "Explain how you would compare two model outputs when one is concise but misses caveats and the other is verbose but complete.",
|
|
42558
|
+
complexity: "medium",
|
|
42559
|
+
category: "reasoning",
|
|
42560
|
+
judge: true,
|
|
42561
|
+
estimatedInputTokens: 12200,
|
|
42562
|
+
estimatedOutputTokens: 650
|
|
42563
|
+
},
|
|
42564
|
+
{
|
|
42565
|
+
id: "extensive-failure-mode",
|
|
42566
|
+
suite: "extensive",
|
|
42567
|
+
title: "Failure-mode analysis",
|
|
42568
|
+
prompt: "List five failure modes for image-understanding benchmarks and one mitigation for each.",
|
|
42569
|
+
complexity: "advanced",
|
|
42570
|
+
category: "image",
|
|
42571
|
+
judge: true,
|
|
42572
|
+
estimatedInputTokens: 12300,
|
|
42573
|
+
estimatedOutputTokens: 900
|
|
42574
|
+
}
|
|
42575
|
+
];
|
|
42576
|
+
async function handleBenchmark(client, subcommand, rest, flags) {
|
|
42577
|
+
if (!subcommand || subcommand === "help" || flags.help === true) {
|
|
42578
|
+
printBenchmarkHelp();
|
|
42579
|
+
return;
|
|
42580
|
+
}
|
|
42581
|
+
if (subcommand !== "model") {
|
|
42582
|
+
throw new Error(`Unknown benchmark command '${subcommand}'. Run 'openmates benchmark --help'.`);
|
|
42583
|
+
}
|
|
42584
|
+
const targetModels = rest.filter((arg) => !arg.startsWith("--"));
|
|
42585
|
+
if (targetModels.length === 0) {
|
|
42586
|
+
throw new Error("Missing target model. Usage: openmates benchmark model <provider/model> [model-b] --confirm-spend-credits");
|
|
42587
|
+
}
|
|
42588
|
+
const compare = flags.compare === true;
|
|
42589
|
+
if (targetModels.length > 1 && !compare) {
|
|
42590
|
+
throw new Error("Multiple target models require --compare.");
|
|
42591
|
+
}
|
|
42592
|
+
if (compare && targetModels.length < 2) {
|
|
42593
|
+
throw new Error("--compare requires at least two target models.");
|
|
42594
|
+
}
|
|
42595
|
+
const judgeModel = typeof flags["judge-model"] === "string" ? flags["judge-model"] : DEFAULT_JUDGE_MODEL;
|
|
42596
|
+
const suites = parseSuites(flags.suite);
|
|
42597
|
+
const runs = parseRuns(flags.runs);
|
|
42598
|
+
const extensiveSize = parseExtensiveSize(flags["extensive-size"]);
|
|
42599
|
+
const parallel = parseParallel(flags.parallel);
|
|
42600
|
+
const caseIds = parseCaseIds(flags.case);
|
|
42601
|
+
const dryRun = flags["dry-run"] === true;
|
|
42602
|
+
const output = typeof flags.output === "string" ? flags.output : void 0;
|
|
42603
|
+
const runId = typeof flags["run-id"] === "string" ? flags["run-id"] : randomUUID3();
|
|
42604
|
+
const imagePath = typeof flags.image === "string" ? resolve5(flags.image) : defaultImageFixturePath();
|
|
42605
|
+
if (!dryRun && flags["confirm-spend-credits"] !== true) {
|
|
42606
|
+
throw new Error(
|
|
42607
|
+
"Benchmark runs spend real credits from the logged-in account. Rerun with --confirm-spend-credits, or use --dry-run to preview the plan."
|
|
42608
|
+
);
|
|
42609
|
+
}
|
|
42610
|
+
const cases = filterCases(expandCases(suites, runs, extensiveSize), caseIds);
|
|
42611
|
+
const pricing = loadPricingForModels([...targetModels, judgeModel]);
|
|
42612
|
+
const estimate = estimateCredits(cases, targetModels, judgeModel, pricing);
|
|
42613
|
+
const result = makeBaseResult({
|
|
42614
|
+
runId,
|
|
42615
|
+
targetModels,
|
|
42616
|
+
judgeModel,
|
|
42617
|
+
suites,
|
|
42618
|
+
runs,
|
|
42619
|
+
compare,
|
|
42620
|
+
parallel,
|
|
42621
|
+
extensiveSize,
|
|
42622
|
+
dryRun,
|
|
42623
|
+
estimate,
|
|
42624
|
+
totalJobs: cases.length * targetModels.length
|
|
42625
|
+
});
|
|
42626
|
+
if (dryRun) {
|
|
42627
|
+
writeBenchmarkResult(result, flags, output);
|
|
42628
|
+
return;
|
|
42629
|
+
}
|
|
42630
|
+
if (!client.hasSession()) {
|
|
42631
|
+
throw new Error("Benchmark runs require login. Run 'openmates login' first.");
|
|
42632
|
+
}
|
|
42633
|
+
let interrupted = false;
|
|
42634
|
+
const onInterrupt = () => {
|
|
42635
|
+
interrupted = true;
|
|
42636
|
+
};
|
|
42637
|
+
process.once("SIGINT", onInterrupt);
|
|
42638
|
+
try {
|
|
42639
|
+
const jobs = cases.flatMap((benchmarkCase) => targetModels.map((model) => ({ model, benchmarkCase })));
|
|
42640
|
+
await runPool(jobs, parallel, async (job) => {
|
|
42641
|
+
if (interrupted) return;
|
|
42642
|
+
const caseResult = await runCaseJob({ client, job, judgeModel, runId, imagePath });
|
|
42643
|
+
result.cases.push(caseResult);
|
|
42644
|
+
recomputeResult(result, jobs.length, interrupted);
|
|
42645
|
+
});
|
|
42646
|
+
} finally {
|
|
42647
|
+
process.off("SIGINT", onInterrupt);
|
|
42648
|
+
}
|
|
42649
|
+
recomputeResult(result, cases.length * targetModels.length, interrupted);
|
|
42650
|
+
writeBenchmarkResult(result, flags, output);
|
|
42651
|
+
}
|
|
42652
|
+
function printBenchmarkHelp() {
|
|
42653
|
+
console.log(`Benchmark commands:
|
|
42654
|
+
openmates benchmark model <provider/model> [provider/model...] --confirm-spend-credits [--compare] [--suite quick|extensive|all] [--json]
|
|
42655
|
+
|
|
42656
|
+
Runs real incognito chat requests through the OpenMates product path. Live runs
|
|
42657
|
+
spend the logged-in user's credits and usage entries are grouped as benchmark spend.
|
|
42658
|
+
|
|
42659
|
+
Options:
|
|
42660
|
+
--confirm-spend-credits Required for live benchmark runs
|
|
42661
|
+
--dry-run Preview the benchmark plan without inference or spend
|
|
42662
|
+
--compare Compare two or more target models
|
|
42663
|
+
--suite <list> Comma-separated suites: quick, extensive, all (default: quick)
|
|
42664
|
+
--case <id[,id...]> Run only specific case id(s) from the selected suites
|
|
42665
|
+
--extensive-size <n> Extensive cases to run: 5, 10, or 20 (default: ${DEFAULT_EXTENSIVE_SIZE})
|
|
42666
|
+
--runs <n> Repeat each selected case (default: 1)
|
|
42667
|
+
--parallel <n> Concurrent target case requests (default: ${DEFAULT_PARALLEL})
|
|
42668
|
+
--judge-model <provider/model> Judge for evaluated cases (default: ${DEFAULT_JUDGE_MODEL})
|
|
42669
|
+
--image <path> Override default Brandenburger Tor image fixture
|
|
42670
|
+
--run-id <id> Reuse a benchmark run id for grouping
|
|
42671
|
+
--output <path> Save JSON result to a file
|
|
42672
|
+
--json Print JSON result`);
|
|
42673
|
+
}
|
|
42674
|
+
function parseSuites(value) {
|
|
42675
|
+
if (value === void 0 || value === false) return ["quick"];
|
|
42676
|
+
if (value === true) throw new Error("--suite requires a value");
|
|
42677
|
+
const suites = value.split(",").map((suite) => suite.trim()).filter(Boolean);
|
|
42678
|
+
if (suites.includes("all")) return ["quick", "extensive"];
|
|
42679
|
+
const allowed = /* @__PURE__ */ new Set(["quick", "extensive"]);
|
|
42680
|
+
const invalid = suites.filter((suite) => !allowed.has(suite));
|
|
42681
|
+
if (invalid.length > 0 || suites.length === 0) {
|
|
42682
|
+
throw new Error("Invalid --suite. Use quick, extensive, or all.");
|
|
42683
|
+
}
|
|
42684
|
+
return [...new Set(suites)];
|
|
42685
|
+
}
|
|
42686
|
+
function parseRuns(value) {
|
|
42687
|
+
if (value === void 0 || value === false) return 1;
|
|
42688
|
+
if (value === true) throw new Error("--runs requires a value");
|
|
42689
|
+
const parsed = Number.parseInt(value, 10);
|
|
42690
|
+
if (!Number.isInteger(parsed) || parsed < 1 || parsed > 20) {
|
|
42691
|
+
throw new Error("--runs must be an integer from 1 to 20");
|
|
42692
|
+
}
|
|
42693
|
+
return parsed;
|
|
42694
|
+
}
|
|
42695
|
+
function parseExtensiveSize(value) {
|
|
42696
|
+
if (value === void 0 || value === false) return DEFAULT_EXTENSIVE_SIZE;
|
|
42697
|
+
if (value === true) throw new Error("--extensive-size requires a value");
|
|
42698
|
+
const parsed = Number.parseInt(value, 10);
|
|
42699
|
+
if (![5, 10, 20].includes(parsed)) {
|
|
42700
|
+
throw new Error("--extensive-size must be 5, 10, or 20");
|
|
42701
|
+
}
|
|
42702
|
+
return parsed;
|
|
42703
|
+
}
|
|
42704
|
+
function parseParallel(value) {
|
|
42705
|
+
if (value === void 0 || value === false) return DEFAULT_PARALLEL;
|
|
42706
|
+
if (value === true) throw new Error("--parallel requires a value");
|
|
42707
|
+
const parsed = Number.parseInt(value, 10);
|
|
42708
|
+
if (!Number.isInteger(parsed) || parsed < 1 || parsed > 20) {
|
|
42709
|
+
throw new Error("--parallel must be an integer from 1 to 20");
|
|
42710
|
+
}
|
|
42711
|
+
return parsed;
|
|
42712
|
+
}
|
|
42713
|
+
function parseCaseIds(value) {
|
|
42714
|
+
if (value === void 0 || value === false) return [];
|
|
42715
|
+
if (value === true) throw new Error("--case requires a case id");
|
|
42716
|
+
const caseIds = value.split(",").map((caseId) => caseId.trim()).filter(Boolean);
|
|
42717
|
+
if (caseIds.length === 0) throw new Error("--case requires at least one case id");
|
|
42718
|
+
return [...new Set(caseIds)];
|
|
42719
|
+
}
|
|
42720
|
+
function filterCases(cases, caseIds) {
|
|
42721
|
+
if (caseIds.length === 0) return cases;
|
|
42722
|
+
const availableIds = new Set(cases.map((benchmarkCase) => benchmarkCase.id));
|
|
42723
|
+
const missing = caseIds.filter((caseId) => !availableIds.has(caseId));
|
|
42724
|
+
if (missing.length > 0) {
|
|
42725
|
+
throw new Error(
|
|
42726
|
+
`Unknown benchmark case id(s): ${missing.join(", ")}. Available in selected suite(s): ${[...availableIds].sort().join(", ")}`
|
|
42727
|
+
);
|
|
42728
|
+
}
|
|
42729
|
+
return cases.filter((benchmarkCase) => caseIds.includes(benchmarkCase.id));
|
|
42730
|
+
}
|
|
42731
|
+
function expandCases(suites, runs, extensiveSize) {
|
|
42732
|
+
const selected = [];
|
|
42733
|
+
if (suites.includes("quick")) selected.push(...QUICK_CASES);
|
|
42734
|
+
if (suites.includes("extensive")) selected.push(...selectExtensiveCases(extensiveSize));
|
|
42735
|
+
const uniqueSelected = dedupeCases(selected);
|
|
42736
|
+
const expanded = [];
|
|
42737
|
+
for (let run = 1; run <= runs; run += 1) {
|
|
42738
|
+
for (const benchmarkCase of uniqueSelected) expanded.push({ ...benchmarkCase, run });
|
|
42739
|
+
}
|
|
42740
|
+
return expanded;
|
|
42741
|
+
}
|
|
42742
|
+
function selectExtensiveCases(size) {
|
|
42743
|
+
const cases = dedupeCases(EXTENSIVE_CASES).slice(0, size);
|
|
42744
|
+
const minimumCoding = Math.ceil(size * 0.15);
|
|
42745
|
+
const codingCount = cases.filter((benchmarkCase) => benchmarkCase.category === "coding").length;
|
|
42746
|
+
if (codingCount >= minimumCoding) return cases;
|
|
42747
|
+
const selectedIds = new Set(cases.map((benchmarkCase) => benchmarkCase.id));
|
|
42748
|
+
const codingBackfill = EXTENSIVE_CASES.filter(
|
|
42749
|
+
(benchmarkCase) => benchmarkCase.category === "coding" && !selectedIds.has(benchmarkCase.id)
|
|
42750
|
+
);
|
|
42751
|
+
const result = [...cases];
|
|
42752
|
+
for (const codingCase of codingBackfill) {
|
|
42753
|
+
let replaceIndex = -1;
|
|
42754
|
+
for (let index = result.length - 1; index >= 0; index -= 1) {
|
|
42755
|
+
if (result[index]?.category !== "coding") {
|
|
42756
|
+
replaceIndex = index;
|
|
42757
|
+
break;
|
|
42758
|
+
}
|
|
42759
|
+
}
|
|
42760
|
+
if (replaceIndex === -1) break;
|
|
42761
|
+
result[replaceIndex] = codingCase;
|
|
42762
|
+
if (result.filter((benchmarkCase) => benchmarkCase.category === "coding").length >= minimumCoding) break;
|
|
42763
|
+
}
|
|
42764
|
+
return result;
|
|
42765
|
+
}
|
|
42766
|
+
function dedupeCases(cases) {
|
|
42767
|
+
const seen = /* @__PURE__ */ new Set();
|
|
42768
|
+
const result = [];
|
|
42769
|
+
for (const benchmarkCase of cases) {
|
|
42770
|
+
if (seen.has(benchmarkCase.id)) continue;
|
|
42771
|
+
seen.add(benchmarkCase.id);
|
|
42772
|
+
result.push(benchmarkCase);
|
|
42773
|
+
}
|
|
42774
|
+
return result;
|
|
42775
|
+
}
|
|
42776
|
+
async function runCaseJob(params) {
|
|
42777
|
+
const { client, job, judgeModel, runId, imagePath } = params;
|
|
42778
|
+
const { model, benchmarkCase } = job;
|
|
42779
|
+
const startedAt = Date.now();
|
|
42780
|
+
const turns = [];
|
|
42781
|
+
const history = benchmarkCase.longContext ? buildLongContextHistory() : [];
|
|
42782
|
+
let chatId;
|
|
42783
|
+
try {
|
|
42784
|
+
const initialPrompt = await buildPromptWithAttachments(client, benchmarkCase, model, imagePath);
|
|
42785
|
+
const targetResponse = await sendBenchmarkTurn({
|
|
42786
|
+
client,
|
|
42787
|
+
model,
|
|
42788
|
+
judgeModel,
|
|
42789
|
+
runId,
|
|
42790
|
+
benchmarkCase,
|
|
42791
|
+
prompt: initialPrompt.message,
|
|
42792
|
+
chatId,
|
|
42793
|
+
history,
|
|
42794
|
+
preparedEmbeds: initialPrompt.embeds,
|
|
42795
|
+
caseId: benchmarkCase.id
|
|
42796
|
+
});
|
|
42797
|
+
chatId = targetResponse.chatId;
|
|
42798
|
+
turns.push(targetResponse.turn);
|
|
42799
|
+
appendHistory(history, "user", initialPrompt.message);
|
|
42800
|
+
appendHistory(history, "assistant", targetResponse.turn.assistant);
|
|
42801
|
+
for (const [index, followUp] of (benchmarkCase.followUps ?? []).entries()) {
|
|
42802
|
+
const response = await sendBenchmarkTurn({
|
|
42803
|
+
client,
|
|
42804
|
+
model,
|
|
42805
|
+
judgeModel,
|
|
42806
|
+
runId,
|
|
42807
|
+
benchmarkCase,
|
|
42808
|
+
prompt: `${modelMention(model)} ${followUp.prompt}`,
|
|
42809
|
+
chatId,
|
|
42810
|
+
history,
|
|
42811
|
+
caseId: `${benchmarkCase.id}:followup-${index + 1}`
|
|
42812
|
+
});
|
|
42813
|
+
chatId = response.chatId;
|
|
42814
|
+
turns.push(response.turn);
|
|
42815
|
+
appendHistory(history, "user", response.rawPrompt);
|
|
42816
|
+
appendHistory(history, "assistant", response.turn.assistant);
|
|
42817
|
+
}
|
|
42818
|
+
const assistant = turns.at(-1)?.assistant ?? "";
|
|
42819
|
+
const caseResult = {
|
|
42820
|
+
id: benchmarkCase.id,
|
|
42821
|
+
suite: benchmarkCase.suite,
|
|
42822
|
+
title: benchmarkCase.title,
|
|
42823
|
+
model,
|
|
42824
|
+
run: benchmarkCase.run,
|
|
42825
|
+
complexity: benchmarkCase.complexity,
|
|
42826
|
+
category: benchmarkCase.category,
|
|
42827
|
+
prompt: benchmarkCase.prompt,
|
|
42828
|
+
assistant,
|
|
42829
|
+
modelName: turns.at(-1)?.modelName ?? null,
|
|
42830
|
+
passed: benchmarkCase.expectedIncludes ? assistant.includes(benchmarkCase.expectedIncludes) : true,
|
|
42831
|
+
durationMs: Date.now() - startedAt,
|
|
42832
|
+
expectedIncludes: benchmarkCase.expectedIncludes,
|
|
42833
|
+
turns
|
|
42834
|
+
};
|
|
42835
|
+
if (benchmarkCase.judge) {
|
|
42836
|
+
caseResult.judge = await judgeCase({ client, judgeModel, targetModel: model, benchmarkCase, caseResult, runId });
|
|
42837
|
+
caseResult.passed = caseResult.judge.score !== null && caseResult.judge.score >= 4 && caseResult.passed;
|
|
42838
|
+
}
|
|
42839
|
+
return caseResult;
|
|
42840
|
+
} catch (error) {
|
|
42841
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
42842
|
+
return {
|
|
42843
|
+
id: benchmarkCase.id,
|
|
42844
|
+
suite: benchmarkCase.suite,
|
|
42845
|
+
title: benchmarkCase.title,
|
|
42846
|
+
model,
|
|
42847
|
+
run: benchmarkCase.run,
|
|
42848
|
+
complexity: benchmarkCase.complexity,
|
|
42849
|
+
category: benchmarkCase.category,
|
|
42850
|
+
prompt: benchmarkCase.prompt,
|
|
42851
|
+
assistant: turns.at(-1)?.assistant ?? "",
|
|
42852
|
+
modelName: turns.at(-1)?.modelName ?? null,
|
|
42853
|
+
passed: false,
|
|
42854
|
+
durationMs: Date.now() - startedAt,
|
|
42855
|
+
expectedIncludes: benchmarkCase.expectedIncludes,
|
|
42856
|
+
turns,
|
|
42857
|
+
error: message
|
|
42858
|
+
};
|
|
42859
|
+
}
|
|
42860
|
+
}
|
|
42861
|
+
async function sendBenchmarkTurn(params) {
|
|
42862
|
+
const startedAt = Date.now();
|
|
42863
|
+
const response = await params.client.sendMessage({
|
|
42864
|
+
message: params.prompt,
|
|
42865
|
+
chatId: params.chatId,
|
|
42866
|
+
incognito: true,
|
|
42867
|
+
autoApproveSubChats: true,
|
|
42868
|
+
benchmarkMetadata: benchmarkMetadata({
|
|
42869
|
+
runId: params.runId,
|
|
42870
|
+
suite: params.benchmarkCase.suite,
|
|
42871
|
+
caseId: params.caseId,
|
|
42872
|
+
targetModel: params.model,
|
|
42873
|
+
judgeModel: params.judgeModel
|
|
42874
|
+
}),
|
|
42875
|
+
messageHistory: params.history,
|
|
42876
|
+
preparedEmbeds: params.preparedEmbeds,
|
|
42877
|
+
precollectResponse: true
|
|
42878
|
+
});
|
|
42879
|
+
return {
|
|
42880
|
+
chatId: response.chatId,
|
|
42881
|
+
rawPrompt: params.prompt,
|
|
42882
|
+
turn: {
|
|
42883
|
+
prompt: params.prompt,
|
|
42884
|
+
assistant: response.assistant,
|
|
42885
|
+
modelName: response.modelName,
|
|
42886
|
+
durationMs: Date.now() - startedAt
|
|
42887
|
+
}
|
|
42888
|
+
};
|
|
42889
|
+
}
|
|
42890
|
+
async function buildPromptWithAttachments(client, benchmarkCase, model, imagePath) {
|
|
42891
|
+
const baseMessage = `${modelMention(model)} ${benchmarkCase.prompt}`;
|
|
42892
|
+
if (benchmarkCase.image !== "default") return { message: baseMessage };
|
|
42893
|
+
const attachment = await prepareImageAttachment(client, imagePath);
|
|
42894
|
+
return { message: `${baseMessage}
|
|
42895
|
+
|
|
42896
|
+
${attachment.messageSuffix}`, embeds: attachment.embeds };
|
|
42897
|
+
}
|
|
42898
|
+
async function prepareImageAttachment(client, imagePath) {
|
|
42899
|
+
if (!existsSync6(imagePath)) throw new Error(`Benchmark image not found: ${imagePath}`);
|
|
42900
|
+
const processed = processFiles([imagePath], null);
|
|
42901
|
+
if (processed.blocked.length > 0 || processed.errors.length > 0 || processed.embeds.length === 0) {
|
|
42902
|
+
const reason = [...processed.blocked, ...processed.errors].map((entry) => entry.error).join("; ") || "no image embed produced";
|
|
42903
|
+
throw new Error(`Failed to prepare benchmark image: ${reason}`);
|
|
42904
|
+
}
|
|
42905
|
+
const fileEmbed = processed.embeds[0];
|
|
42906
|
+
if (!fileEmbed.requiresUpload || !fileEmbed.localPath) {
|
|
42907
|
+
return { messageSuffix: fileEmbed.referenceBlock, embeds: [fileEmbed.embed] };
|
|
42908
|
+
}
|
|
42909
|
+
await uploadBenchmarkImage(client, fileEmbed);
|
|
42910
|
+
return { messageSuffix: fileEmbed.referenceBlock, embeds: [fileEmbed.embed] };
|
|
42911
|
+
}
|
|
42912
|
+
async function uploadBenchmarkImage(client, fileEmbed) {
|
|
42913
|
+
if (!fileEmbed.localPath) return;
|
|
42914
|
+
const uploadResult = await uploadFile(fileEmbed.localPath, client.getSession());
|
|
42915
|
+
const embedRef = fileEmbed.embed.embedRef ?? `benchmark-image-${uploadResult.embed_id.slice(0, 8)}`;
|
|
42916
|
+
fileEmbed.embed.embedRef = embedRef;
|
|
42917
|
+
fileEmbed.embed.content = toonEncodeContent({
|
|
42918
|
+
type: "image",
|
|
42919
|
+
app_id: "images",
|
|
42920
|
+
skill_id: "upload",
|
|
42921
|
+
status: "finished",
|
|
42922
|
+
filename: fileEmbed.displayName,
|
|
42923
|
+
embed_ref: embedRef,
|
|
42924
|
+
content_hash: uploadResult.content_hash,
|
|
42925
|
+
s3_base_url: uploadResult.s3_base_url,
|
|
42926
|
+
files: uploadResult.files,
|
|
42927
|
+
aes_key: uploadResult.aes_key,
|
|
42928
|
+
aes_nonce: uploadResult.aes_nonce,
|
|
42929
|
+
vault_wrapped_aes_key: uploadResult.vault_wrapped_aes_key,
|
|
42930
|
+
ai_detection: uploadResult.ai_detection
|
|
42931
|
+
});
|
|
42932
|
+
fileEmbed.embed.status = "finished";
|
|
42933
|
+
fileEmbed.embed.contentHash = uploadResult.content_hash;
|
|
42934
|
+
fileEmbed.embed.embedId = uploadResult.embed_id;
|
|
42935
|
+
fileEmbed.referenceBlock = createBenchmarkEmbedReferenceBlock(fileEmbed.embed.embedId, fileEmbed.embed.type);
|
|
42936
|
+
}
|
|
42937
|
+
function createBenchmarkEmbedReferenceBlock(embedId, embedType) {
|
|
42938
|
+
return `
|
|
42939
|
+
|
|
42940
|
+
\`\`\`json
|
|
42941
|
+
${JSON.stringify({ type: embedType, embed_id: embedId })}
|
|
42942
|
+
\`\`\``;
|
|
42943
|
+
}
|
|
42944
|
+
async function judgeCase(params) {
|
|
42945
|
+
const startedAt = Date.now();
|
|
42946
|
+
const judgeResponse = await params.client.sendMessage({
|
|
42947
|
+
message: `${modelMention(params.judgeModel)} ${judgePrompt(params.targetModel, params.benchmarkCase, params.caseResult)}`,
|
|
42948
|
+
incognito: true,
|
|
42949
|
+
autoApproveSubChats: true,
|
|
42950
|
+
benchmarkMetadata: benchmarkMetadata({
|
|
42951
|
+
runId: params.runId,
|
|
42952
|
+
suite: params.benchmarkCase.suite,
|
|
42953
|
+
caseId: `${params.benchmarkCase.id}:judge:${params.targetModel}`,
|
|
42954
|
+
targetModel: params.targetModel,
|
|
42955
|
+
judgeModel: params.judgeModel
|
|
42956
|
+
}),
|
|
42957
|
+
precollectResponse: true
|
|
42958
|
+
});
|
|
42959
|
+
const judgment = parseJudgment(judgeResponse.assistant);
|
|
42960
|
+
return {
|
|
42961
|
+
model: params.judgeModel,
|
|
42962
|
+
score: judgment.score,
|
|
42963
|
+
reason: judgment.reason,
|
|
42964
|
+
raw: judgeResponse.assistant,
|
|
42965
|
+
durationMs: Date.now() - startedAt
|
|
42966
|
+
};
|
|
42967
|
+
}
|
|
42968
|
+
async function runPool(items, parallel, worker) {
|
|
42969
|
+
let index = 0;
|
|
42970
|
+
const workers = Array.from({ length: Math.min(parallel, items.length) }, async () => {
|
|
42971
|
+
while (index < items.length) {
|
|
42972
|
+
const item = items[index];
|
|
42973
|
+
index += 1;
|
|
42974
|
+
await worker(item);
|
|
42975
|
+
}
|
|
42976
|
+
});
|
|
42977
|
+
await Promise.all(workers);
|
|
42978
|
+
}
|
|
42979
|
+
function buildLongContextHistory() {
|
|
42980
|
+
const now = Math.floor(Date.now() / 1e3) - 2e3;
|
|
42981
|
+
const topics = [
|
|
42982
|
+
["user", "We need to launch a CLI benchmark for model comparisons."],
|
|
42983
|
+
["assistant", "The first goal should be a quick suite with deterministic checks."],
|
|
42984
|
+
["user", "The benchmark also needs image inference."],
|
|
42985
|
+
["assistant", "Use a public fixture image and ask a factual visual question."],
|
|
42986
|
+
["user", "We should avoid wasting credits."],
|
|
42987
|
+
["assistant", "Run a pricing preflight and require explicit spend confirmation."],
|
|
42988
|
+
["user", "What about longer conversations?"],
|
|
42989
|
+
["assistant", "Add a 20-message predefined history and a dependent follow-up."],
|
|
42990
|
+
["user", "The extensive suite should not be too small."],
|
|
42991
|
+
["assistant", "Default to 10 cases and allow 5 or 20 as alternatives."],
|
|
42992
|
+
["user", "Coding quality matters."],
|
|
42993
|
+
["assistant", "Reserve at least 15 percent of extensive cases for coding prompts."],
|
|
42994
|
+
["user", "We also need comparison mode."],
|
|
42995
|
+
["assistant", "Accept multiple models with --compare and run target jobs in parallel."],
|
|
42996
|
+
["user", "How should judging work?"],
|
|
42997
|
+
["assistant", "Judge each completed case immediately with Gemini so partial results remain useful."],
|
|
42998
|
+
["user", "What if the process is interrupted?"],
|
|
42999
|
+
["assistant", "Print or write a partial summary with completed judgments and skipped counts."],
|
|
43000
|
+
["user", "What is the best launch strategy?"],
|
|
43001
|
+
["assistant", "Ship quick and comparison first, then use extensive for slower releases."]
|
|
43002
|
+
];
|
|
43003
|
+
return topics.map(([role, content], index) => ({
|
|
43004
|
+
message_id: `benchmark-history-${index + 1}`,
|
|
43005
|
+
role,
|
|
43006
|
+
sender_name: role === "user" ? "User" : "Assistant",
|
|
43007
|
+
content,
|
|
43008
|
+
created_at: now + index * 30
|
|
43009
|
+
}));
|
|
43010
|
+
}
|
|
43011
|
+
function appendHistory(history, role, content) {
|
|
43012
|
+
history.push({
|
|
43013
|
+
message_id: randomUUID3(),
|
|
43014
|
+
role,
|
|
43015
|
+
sender_name: role === "user" ? "User" : "Assistant",
|
|
43016
|
+
content,
|
|
43017
|
+
created_at: Math.floor(Date.now() / 1e3)
|
|
43018
|
+
});
|
|
43019
|
+
}
|
|
43020
|
+
function modelMention(model) {
|
|
43021
|
+
const separator = model.indexOf("/");
|
|
43022
|
+
if (separator === -1) return `@ai-model:${model}`;
|
|
43023
|
+
const provider = model.slice(0, separator);
|
|
43024
|
+
const modelId = model.slice(separator + 1);
|
|
43025
|
+
if (!provider || !modelId) return `@ai-model:${model}`;
|
|
43026
|
+
return `@ai-model:${modelId}:${provider}`;
|
|
43027
|
+
}
|
|
43028
|
+
function benchmarkMetadata(params) {
|
|
43029
|
+
return {
|
|
43030
|
+
source: "benchmark",
|
|
43031
|
+
benchmark_run_id: params.runId,
|
|
43032
|
+
benchmark_suite: params.suite,
|
|
43033
|
+
benchmark_case: params.caseId,
|
|
43034
|
+
benchmark_target_model: params.targetModel,
|
|
43035
|
+
benchmark_judge_model: params.judgeModel
|
|
43036
|
+
};
|
|
43037
|
+
}
|
|
43038
|
+
function judgePrompt(targetModel, benchmarkCase, result) {
|
|
43039
|
+
return [
|
|
43040
|
+
"You are judging a real OpenMates model benchmark response.",
|
|
43041
|
+
"Return exactly two plain-text lines, with no markdown, no code block, and no tool use.",
|
|
43042
|
+
"Line 1 format: BENCHMARK_SCORE=<integer from 1 to 5>",
|
|
43043
|
+
"Line 2 format: BENCHMARK_REASON=<one short sentence>",
|
|
43044
|
+
"Score for correctness, instruction-following, usefulness, and continuity where relevant.",
|
|
43045
|
+
`Target model: ${targetModel}`,
|
|
43046
|
+
`Benchmark case: ${benchmarkCase.id} (${benchmarkCase.category}, ${benchmarkCase.complexity})`,
|
|
43047
|
+
`Initial prompt: ${JSON.stringify(benchmarkCase.prompt)}`,
|
|
43048
|
+
`Turns: ${JSON.stringify(result.turns.map((turn) => ({ prompt: turn.prompt, assistant: turn.assistant })))}`
|
|
43049
|
+
].join("\n");
|
|
43050
|
+
}
|
|
43051
|
+
function parseJudgment(answer) {
|
|
43052
|
+
const markerScore = answer.match(/BENCHMARK_SCORE\s*=\s*([1-5])/i);
|
|
43053
|
+
if (markerScore) {
|
|
43054
|
+
const reasonMatch = answer.match(/BENCHMARK_REASON\s*=\s*(.+)/i);
|
|
43055
|
+
return {
|
|
43056
|
+
score: Number.parseInt(markerScore[1], 10),
|
|
43057
|
+
reason: reasonMatch?.[1]?.trim() ?? null
|
|
43058
|
+
};
|
|
43059
|
+
}
|
|
43060
|
+
const jsonText = extractJsonObject(answer);
|
|
43061
|
+
if (!jsonText) return { score: null, reason: null };
|
|
43062
|
+
try {
|
|
43063
|
+
const parsed = JSON.parse(jsonText);
|
|
43064
|
+
const score = typeof parsed.score === "number" && Number.isFinite(parsed.score) ? parsed.score : null;
|
|
43065
|
+
const reason = typeof parsed.reason === "string" ? parsed.reason : null;
|
|
43066
|
+
return { score, reason };
|
|
43067
|
+
} catch {
|
|
43068
|
+
return { score: null, reason: null };
|
|
43069
|
+
}
|
|
43070
|
+
}
|
|
43071
|
+
function extractJsonObject(text) {
|
|
43072
|
+
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
|
|
43073
|
+
if (fenced) return fenced[1];
|
|
43074
|
+
const start = text.indexOf("{");
|
|
43075
|
+
const end = text.lastIndexOf("}");
|
|
43076
|
+
if (start === -1 || end === -1 || end <= start) return null;
|
|
43077
|
+
return text.slice(start, end + 1);
|
|
43078
|
+
}
|
|
43079
|
+
function loadPricingForModels(models) {
|
|
43080
|
+
const availablePricing = loadProviderPricing();
|
|
43081
|
+
const pricing = /* @__PURE__ */ new Map();
|
|
43082
|
+
const missing = [];
|
|
43083
|
+
for (const model of [...new Set(models)]) {
|
|
43084
|
+
const key = normalizeModelKey(model);
|
|
43085
|
+
const modelPricing = availablePricing.get(key);
|
|
43086
|
+
if (!modelPricing) {
|
|
43087
|
+
missing.push(model);
|
|
43088
|
+
continue;
|
|
43089
|
+
}
|
|
43090
|
+
pricing.set(model, modelPricing);
|
|
43091
|
+
}
|
|
43092
|
+
if (missing.length > 0) {
|
|
43093
|
+
throw new Error(
|
|
43094
|
+
`Cannot estimate benchmark cost because pricing metadata is unavailable for: ${missing.join(", ")}. Use provider/model ids with backend provider pricing metadata.`
|
|
43095
|
+
);
|
|
43096
|
+
}
|
|
43097
|
+
return pricing;
|
|
43098
|
+
}
|
|
43099
|
+
function loadProviderPricing() {
|
|
43100
|
+
const providersDir = findProvidersDir();
|
|
43101
|
+
const pricing = /* @__PURE__ */ new Map();
|
|
43102
|
+
if (!providersDir) return pricing;
|
|
43103
|
+
for (const fileName of readdirSync(providersDir)) {
|
|
43104
|
+
if (!fileName.endsWith(".yml")) continue;
|
|
43105
|
+
const filePath = join4(providersDir, fileName);
|
|
43106
|
+
const text = readFileSync6(filePath, "utf-8");
|
|
43107
|
+
const provider = parseProviderId(text) ?? fileName.replace(/\.yml$/, "");
|
|
43108
|
+
for (const modelPricing of parseModelPricing(text, provider)) {
|
|
43109
|
+
pricing.set(`${modelPricing.provider}/${modelPricing.modelId}`, modelPricing);
|
|
43110
|
+
pricing.set(modelPricing.modelId, modelPricing);
|
|
43111
|
+
}
|
|
43112
|
+
}
|
|
43113
|
+
return pricing;
|
|
43114
|
+
}
|
|
43115
|
+
function parseProviderId(text) {
|
|
43116
|
+
const match = text.match(/^provider_id:\s*["']?([^"'\n]+)["']?/m);
|
|
43117
|
+
return match?.[1]?.trim() ?? null;
|
|
43118
|
+
}
|
|
43119
|
+
function parseModelPricing(text, provider) {
|
|
43120
|
+
const lines = text.split("\n");
|
|
43121
|
+
const results = [];
|
|
43122
|
+
let modelId = null;
|
|
43123
|
+
let inModel = false;
|
|
43124
|
+
let inputTokensPerCredit = null;
|
|
43125
|
+
let outputTokensPerCredit = null;
|
|
43126
|
+
for (const line of lines) {
|
|
43127
|
+
const modelMatch = line.match(/^\s{2}-\s+id:\s*["']?([^"'\n#]+)["']?/);
|
|
43128
|
+
if (modelMatch) {
|
|
43129
|
+
if (inModel && modelId && inputTokensPerCredit && outputTokensPerCredit) {
|
|
43130
|
+
results.push({ provider, modelId, inputTokensPerCredit, outputTokensPerCredit });
|
|
43131
|
+
}
|
|
43132
|
+
inModel = true;
|
|
43133
|
+
modelId = modelMatch[1].trim();
|
|
43134
|
+
inputTokensPerCredit = null;
|
|
43135
|
+
outputTokensPerCredit = null;
|
|
43136
|
+
continue;
|
|
43137
|
+
}
|
|
43138
|
+
if (!inModel) continue;
|
|
43139
|
+
const inputMatch = line.match(/^\s{10}per_credit_unit:\s*(\d+)/);
|
|
43140
|
+
if (inputMatch && inputTokensPerCredit === null) {
|
|
43141
|
+
inputTokensPerCredit = Number.parseInt(inputMatch[1], 10);
|
|
43142
|
+
continue;
|
|
43143
|
+
}
|
|
43144
|
+
if (inputMatch && inputTokensPerCredit !== null && outputTokensPerCredit === null) {
|
|
43145
|
+
outputTokensPerCredit = Number.parseInt(inputMatch[1], 10);
|
|
43146
|
+
}
|
|
43147
|
+
}
|
|
43148
|
+
if (inModel && modelId && inputTokensPerCredit && outputTokensPerCredit) {
|
|
43149
|
+
results.push({ provider, modelId, inputTokensPerCredit, outputTokensPerCredit });
|
|
43150
|
+
}
|
|
43151
|
+
return results;
|
|
43152
|
+
}
|
|
43153
|
+
function normalizeModelKey(model) {
|
|
43154
|
+
return model.includes("/") ? model : model;
|
|
43155
|
+
}
|
|
43156
|
+
function findProvidersDir() {
|
|
43157
|
+
const currentFile = fileURLToPath(import.meta.url);
|
|
43158
|
+
let current = dirname2(currentFile);
|
|
43159
|
+
for (let index = 0; index < 8; index += 1) {
|
|
43160
|
+
const candidate = join4(current, "backend", "providers");
|
|
43161
|
+
if (existsSync6(candidate)) return candidate;
|
|
43162
|
+
const parentCandidate = join4(current, "..", "..", "backend", "providers");
|
|
43163
|
+
if (existsSync6(parentCandidate)) return resolve5(parentCandidate);
|
|
43164
|
+
const next = dirname2(current);
|
|
43165
|
+
if (next === current) break;
|
|
43166
|
+
current = next;
|
|
43167
|
+
}
|
|
43168
|
+
return null;
|
|
43169
|
+
}
|
|
43170
|
+
function estimateCredits(cases, targetModels, judgeModel, pricing) {
|
|
43171
|
+
let targetCredits = 0;
|
|
43172
|
+
let judgeCredits = 0;
|
|
43173
|
+
let targetInputTokens = 0;
|
|
43174
|
+
let targetOutputTokens = 0;
|
|
43175
|
+
let judgeInputTokens = 0;
|
|
43176
|
+
let judgeOutputTokens = 0;
|
|
43177
|
+
for (const benchmarkCase of cases) {
|
|
43178
|
+
const turnCount = 1 + (benchmarkCase.followUps?.length ?? 0);
|
|
43179
|
+
for (const model of targetModels) {
|
|
43180
|
+
const modelPricing = pricing.get(model);
|
|
43181
|
+
if (!modelPricing) continue;
|
|
43182
|
+
const input = benchmarkCase.estimatedInputTokens * turnCount;
|
|
43183
|
+
const output = benchmarkCase.estimatedOutputTokens * turnCount;
|
|
43184
|
+
targetInputTokens += input;
|
|
43185
|
+
targetOutputTokens += output;
|
|
43186
|
+
targetCredits += creditsFor(modelPricing, input, output);
|
|
43187
|
+
if (benchmarkCase.judge) {
|
|
43188
|
+
const judgePricing = pricing.get(judgeModel);
|
|
43189
|
+
if (!judgePricing) continue;
|
|
43190
|
+
const judgeInput = Math.max(2e3, Math.ceil(output * 1.5));
|
|
43191
|
+
const judgeOutput = 350;
|
|
43192
|
+
judgeInputTokens += judgeInput;
|
|
43193
|
+
judgeOutputTokens += judgeOutput;
|
|
43194
|
+
judgeCredits += creditsFor(judgePricing, judgeInput, judgeOutput);
|
|
43195
|
+
}
|
|
43196
|
+
}
|
|
43197
|
+
}
|
|
43198
|
+
return {
|
|
43199
|
+
targetCredits,
|
|
43200
|
+
judgeCredits,
|
|
43201
|
+
totalCredits: targetCredits + judgeCredits,
|
|
43202
|
+
assumptions: { targetInputTokens, targetOutputTokens, judgeInputTokens, judgeOutputTokens }
|
|
43203
|
+
};
|
|
43204
|
+
}
|
|
43205
|
+
function creditsFor(pricing, inputTokens, outputTokens) {
|
|
43206
|
+
return Math.ceil(inputTokens / pricing.inputTokensPerCredit) + Math.ceil(outputTokens / pricing.outputTokensPerCredit);
|
|
43207
|
+
}
|
|
43208
|
+
function makeBaseResult(params) {
|
|
43209
|
+
return {
|
|
43210
|
+
command: "benchmark model",
|
|
43211
|
+
status: params.dryRun ? "planned" : "completed",
|
|
43212
|
+
runId: params.runId,
|
|
43213
|
+
targetModel: params.targetModels[0],
|
|
43214
|
+
targetModels: params.targetModels,
|
|
43215
|
+
judgeModel: params.judgeModel,
|
|
43216
|
+
suites: params.suites,
|
|
43217
|
+
runs: params.runs,
|
|
43218
|
+
compare: params.compare,
|
|
43219
|
+
parallel: params.parallel,
|
|
43220
|
+
extensiveSize: params.extensiveSize,
|
|
43221
|
+
spendsCredits: !params.dryRun,
|
|
43222
|
+
estimatedCredits: params.estimate,
|
|
43223
|
+
cases: [],
|
|
43224
|
+
modelSummaries: params.targetModels.map((model) => ({
|
|
43225
|
+
model,
|
|
43226
|
+
total: 0,
|
|
43227
|
+
passed: 0,
|
|
43228
|
+
failed: 0,
|
|
43229
|
+
averageJudgeScore: null,
|
|
43230
|
+
averageDurationMs: null
|
|
43231
|
+
})),
|
|
43232
|
+
summary: {
|
|
43233
|
+
total: params.totalJobs,
|
|
43234
|
+
completed: 0,
|
|
43235
|
+
passed: 0,
|
|
43236
|
+
failed: 0,
|
|
43237
|
+
skipped: params.dryRun ? params.totalJobs : 0,
|
|
43238
|
+
interrupted: false
|
|
43239
|
+
}
|
|
43240
|
+
};
|
|
43241
|
+
}
|
|
43242
|
+
function recomputeResult(result, totalJobs, interrupted) {
|
|
43243
|
+
const completed = result.cases.length;
|
|
43244
|
+
const passed = result.cases.filter((caseResult) => caseResult.passed).length;
|
|
43245
|
+
const failed = result.cases.filter((caseResult) => !caseResult.passed).length;
|
|
43246
|
+
result.summary = {
|
|
43247
|
+
total: totalJobs,
|
|
43248
|
+
completed,
|
|
43249
|
+
passed,
|
|
43250
|
+
failed,
|
|
43251
|
+
skipped: Math.max(0, totalJobs - completed),
|
|
43252
|
+
interrupted
|
|
43253
|
+
};
|
|
43254
|
+
result.status = interrupted || completed < totalJobs ? "partial" : "completed";
|
|
43255
|
+
result.modelSummaries = result.targetModels.map((model) => summarizeModel(model, result.cases));
|
|
43256
|
+
if (result.compare) result.comparison = buildComparison(result.modelSummaries);
|
|
43257
|
+
}
|
|
43258
|
+
function summarizeModel(model, cases) {
|
|
43259
|
+
const modelCases = cases.filter((caseResult) => caseResult.model === model);
|
|
43260
|
+
const scores = modelCases.map((caseResult) => caseResult.judge?.score).filter((score) => typeof score === "number" && Number.isFinite(score));
|
|
43261
|
+
const durations = modelCases.map((caseResult) => caseResult.durationMs).filter((value) => value > 0);
|
|
43262
|
+
return {
|
|
43263
|
+
model,
|
|
43264
|
+
total: modelCases.length,
|
|
43265
|
+
passed: modelCases.filter((caseResult) => caseResult.passed).length,
|
|
43266
|
+
failed: modelCases.filter((caseResult) => !caseResult.passed).length,
|
|
43267
|
+
averageJudgeScore: scores.length > 0 ? round2(scores.reduce((sum, score) => sum + score, 0) / scores.length) : null,
|
|
43268
|
+
averageDurationMs: durations.length > 0 ? Math.round(durations.reduce((sum, value) => sum + value, 0) / durations.length) : null
|
|
43269
|
+
};
|
|
43270
|
+
}
|
|
43271
|
+
function buildComparison(summaries) {
|
|
43272
|
+
const ranking = [...summaries].sort((a, b) => (b.averageJudgeScore ?? -1) - (a.averageJudgeScore ?? -1) || b.passed - a.passed).map((summary) => ({
|
|
43273
|
+
model: summary.model,
|
|
43274
|
+
averageJudgeScore: summary.averageJudgeScore,
|
|
43275
|
+
passed: summary.passed,
|
|
43276
|
+
total: summary.total
|
|
43277
|
+
}));
|
|
43278
|
+
const notes = ranking.length > 0 ? [`Top model so far: ${ranking[0].model} (${ranking[0].passed}/${ranking[0].total} passed).`] : [];
|
|
43279
|
+
return { ranking, notes };
|
|
43280
|
+
}
|
|
43281
|
+
function round2(value) {
|
|
43282
|
+
return Math.round(value * 100) / 100;
|
|
43283
|
+
}
|
|
43284
|
+
function defaultImageFixturePath() {
|
|
43285
|
+
const fixtureDir = join4(dirname2(fileURLToPath(import.meta.url)), "..", "fixtures");
|
|
43286
|
+
const fixturePath = join4(fixtureDir, "brandenburger-tor.png");
|
|
43287
|
+
if (existsSync6(fixturePath)) return fixturePath;
|
|
43288
|
+
const tempDir = mkdtempSync(join4(tmpdir(), "openmates-benchmark-"));
|
|
43289
|
+
const tempPath = join4(tempDir, "brandenburger-tor.svg");
|
|
43290
|
+
writeFileSync4(tempPath, FIXTURE_IMAGE_SVG, "utf-8");
|
|
43291
|
+
return tempPath;
|
|
43292
|
+
}
|
|
43293
|
+
function writeBenchmarkResult(result, flags, output) {
|
|
43294
|
+
const json = `${JSON.stringify(result, null, 2)}
|
|
43295
|
+
`;
|
|
43296
|
+
if (output) writeFileSync4(output, json, "utf-8");
|
|
43297
|
+
if (flags.json === true || output) {
|
|
43298
|
+
process.stdout.write(json);
|
|
43299
|
+
return;
|
|
43300
|
+
}
|
|
43301
|
+
console.log(`Benchmark ${result.status}: ${result.targetModels.join(", ")}`);
|
|
43302
|
+
console.log(`Run ID: ${result.runId}`);
|
|
43303
|
+
console.log(`Suites: ${result.suites.join(", ")}`);
|
|
43304
|
+
console.log(`Judge: ${result.judgeModel}`);
|
|
43305
|
+
console.log(`Estimated credits: ${result.estimatedCredits.totalCredits}`);
|
|
43306
|
+
console.log(`Spend credits: ${result.spendsCredits ? "yes" : "no"}`);
|
|
43307
|
+
if (result.status !== "planned") {
|
|
43308
|
+
console.log(`Passed: ${result.summary.passed}/${result.summary.completed} completed (${result.summary.skipped} skipped)`);
|
|
43309
|
+
for (const benchmarkCase of result.cases) {
|
|
43310
|
+
const mark = benchmarkCase.passed ? "PASS" : "FAIL";
|
|
43311
|
+
const judge = benchmarkCase.judge ? ` judge=${benchmarkCase.judge.score ?? "unparsed"}` : "";
|
|
43312
|
+
const error = benchmarkCase.error ? ` error=${benchmarkCase.error}` : "";
|
|
43313
|
+
console.log(`${mark} ${benchmarkCase.model} ${benchmarkCase.suite}/${benchmarkCase.id} (${benchmarkCase.durationMs}ms)${judge}${error}`);
|
|
43314
|
+
}
|
|
43315
|
+
}
|
|
43316
|
+
}
|
|
43317
|
+
|
|
41505
43318
|
// src/cli.ts
|
|
41506
43319
|
async function main() {
|
|
41507
43320
|
const parsed = parseArgs(process.argv.slice(2));
|
|
@@ -41536,6 +43349,10 @@ async function main() {
|
|
|
41536
43349
|
printSettingsHelp(client);
|
|
41537
43350
|
return;
|
|
41538
43351
|
}
|
|
43352
|
+
if (command === "learning-mode") {
|
|
43353
|
+
printLearningModeHelp();
|
|
43354
|
+
return;
|
|
43355
|
+
}
|
|
41539
43356
|
if (command === "signup") {
|
|
41540
43357
|
printSignupHelp();
|
|
41541
43358
|
return;
|
|
@@ -41572,6 +43389,10 @@ async function main() {
|
|
|
41572
43389
|
printDocsHelp();
|
|
41573
43390
|
return;
|
|
41574
43391
|
}
|
|
43392
|
+
if (command === "benchmark") {
|
|
43393
|
+
printBenchmarkHelp();
|
|
43394
|
+
return;
|
|
43395
|
+
}
|
|
41575
43396
|
printHelp();
|
|
41576
43397
|
return;
|
|
41577
43398
|
}
|
|
@@ -41630,6 +43451,10 @@ async function main() {
|
|
|
41630
43451
|
await handleSettings(client, subcommand, rest, parsed.flags);
|
|
41631
43452
|
return;
|
|
41632
43453
|
}
|
|
43454
|
+
if (command === "learning-mode") {
|
|
43455
|
+
await handleLearningMode(client, subcommand, parsed.flags);
|
|
43456
|
+
return;
|
|
43457
|
+
}
|
|
41633
43458
|
if (command === "inspirations") {
|
|
41634
43459
|
await handleInspirations(client, parsed.flags);
|
|
41635
43460
|
return;
|
|
@@ -41642,10 +43467,22 @@ async function main() {
|
|
|
41642
43467
|
handleFeedback(subcommand, rest, parsed.flags);
|
|
41643
43468
|
return;
|
|
41644
43469
|
}
|
|
43470
|
+
if (command === "benchmark") {
|
|
43471
|
+
await handleBenchmark(client, subcommand, rest, parsed.flags);
|
|
43472
|
+
return;
|
|
43473
|
+
}
|
|
41645
43474
|
throw new Error(`Unknown command '${command}'. Run 'openmates help'.`);
|
|
41646
43475
|
}
|
|
41647
43476
|
function shouldInitializeRedactor(command, subcommand) {
|
|
41648
|
-
return command === "chats" && ["new", "send", "incognito"].includes(subcommand ?? "");
|
|
43477
|
+
return command === "chats" && ["new", "send", "answer-interactive", "incognito"].includes(subcommand ?? "");
|
|
43478
|
+
}
|
|
43479
|
+
function parseJsonFlag(value, flagName) {
|
|
43480
|
+
try {
|
|
43481
|
+
return JSON.parse(value);
|
|
43482
|
+
} catch (error) {
|
|
43483
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
43484
|
+
throw new Error(`Invalid JSON for ${flagName}: ${message}`);
|
|
43485
|
+
}
|
|
41649
43486
|
}
|
|
41650
43487
|
async function handleChats(client, subcommand, rest, flags, redactor) {
|
|
41651
43488
|
if (!subcommand || subcommand === "help" || flags.help === true) {
|
|
@@ -41698,7 +43535,8 @@ async function handleChats(client, subcommand, rest, flags, redactor) {
|
|
|
41698
43535
|
json: flags.json === true,
|
|
41699
43536
|
autoApproveSubChats: flags["auto-approve"] === true,
|
|
41700
43537
|
autoApproveMemories: flags["auto-approve-memories"] === true,
|
|
41701
|
-
piiDetection: flags["no-pii-detection"] !== true
|
|
43538
|
+
piiDetection: flags["no-pii-detection"] !== true,
|
|
43539
|
+
anonymousLearningMode: client.hasSession() ? void 0 : parseAnonymousLearningModeFlags(flags)
|
|
41702
43540
|
},
|
|
41703
43541
|
redactor
|
|
41704
43542
|
);
|
|
@@ -41779,6 +43617,34 @@ Run 'openmates chats show ` + chatId + "' to check if suggestions have been save
|
|
|
41779
43617
|
if (flags.json === true) printJson2(result);
|
|
41780
43618
|
return;
|
|
41781
43619
|
}
|
|
43620
|
+
if (subcommand === "answer-interactive") {
|
|
43621
|
+
const chatId = typeof flags.chat === "string" ? flags.chat : void 0;
|
|
43622
|
+
const questionJson = typeof flags["question-json"] === "string" ? flags["question-json"] : void 0;
|
|
43623
|
+
const answerJson = typeof flags["answer-json"] === "string" ? flags["answer-json"] : void 0;
|
|
43624
|
+
if (!chatId || !questionJson || !answerJson) {
|
|
43625
|
+
throw new Error(
|
|
43626
|
+
"Missing interactive answer data. Usage: openmates chats answer-interactive --chat <id> --question-json '<json>' --answer-json '<json>'"
|
|
43627
|
+
);
|
|
43628
|
+
}
|
|
43629
|
+
const question = parseJsonFlag(questionJson, "--question-json");
|
|
43630
|
+
const answer = parseJsonFlag(answerJson, "--answer-json");
|
|
43631
|
+
const formatted = formatInteractiveQuestionAnswer(question, answer);
|
|
43632
|
+
const result = await sendMessageStreaming(
|
|
43633
|
+
client,
|
|
43634
|
+
{
|
|
43635
|
+
message: formatted.messageContent,
|
|
43636
|
+
chatId,
|
|
43637
|
+
incognito: false,
|
|
43638
|
+
json: flags.json === true,
|
|
43639
|
+
autoApproveSubChats: flags["auto-approve"] === true,
|
|
43640
|
+
autoApproveMemories: flags["auto-approve-memories"] === true,
|
|
43641
|
+
piiDetection: flags["no-pii-detection"] !== true
|
|
43642
|
+
},
|
|
43643
|
+
redactor
|
|
43644
|
+
);
|
|
43645
|
+
if (flags.json === true) printJson2(result);
|
|
43646
|
+
return;
|
|
43647
|
+
}
|
|
41782
43648
|
if (subcommand === "incognito") {
|
|
41783
43649
|
const message = rest.join(" ").trim();
|
|
41784
43650
|
if (!message)
|
|
@@ -41878,10 +43744,10 @@ Run 'openmates chats show ` + chatId + "' to check if suggestions have been save
|
|
|
41878
43744
|
input: process.stdin,
|
|
41879
43745
|
output: process.stdout
|
|
41880
43746
|
});
|
|
41881
|
-
const answer = await new Promise((
|
|
43747
|
+
const answer = await new Promise((resolve6) => {
|
|
41882
43748
|
iface.question(
|
|
41883
43749
|
`Delete ${resolved.length} chat(s)? This cannot be undone. [y/N] `,
|
|
41884
|
-
|
|
43750
|
+
resolve6
|
|
41885
43751
|
);
|
|
41886
43752
|
});
|
|
41887
43753
|
iface.close();
|
|
@@ -42041,16 +43907,16 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
|
|
|
42041
43907
|
}
|
|
42042
43908
|
}
|
|
42043
43909
|
const { mkdir, writeFile } = await import("fs/promises");
|
|
42044
|
-
const { join:
|
|
43910
|
+
const { join: join5 } = await import("path");
|
|
42045
43911
|
if (useZip) {
|
|
42046
|
-
const tmpDir =
|
|
43912
|
+
const tmpDir = join5(outputDir, `.${filenameBase}_tmp`);
|
|
42047
43913
|
await mkdir(tmpDir, { recursive: true });
|
|
42048
|
-
await writeFile(
|
|
42049
|
-
await writeFile(
|
|
43914
|
+
await writeFile(join5(tmpDir, `${filenameBase}.yml`), yamlContent);
|
|
43915
|
+
await writeFile(join5(tmpDir, `${filenameBase}.md`), mdContent);
|
|
42050
43916
|
if (codeEmbeds.length > 0) {
|
|
42051
43917
|
for (const ce of codeEmbeds) {
|
|
42052
43918
|
const fpath = ce.filePath ?? ce.filename ?? `${ce.embedId.slice(0, 8)}.${getExtForLang(ce.language)}`;
|
|
42053
|
-
const fullPath =
|
|
43919
|
+
const fullPath = join5(tmpDir, "code", fpath);
|
|
42054
43920
|
await mkdir(fullPath.substring(0, fullPath.lastIndexOf("/")), {
|
|
42055
43921
|
recursive: true
|
|
42056
43922
|
});
|
|
@@ -42058,13 +43924,13 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
|
|
|
42058
43924
|
}
|
|
42059
43925
|
}
|
|
42060
43926
|
if (transcriptEmbeds.length > 0) {
|
|
42061
|
-
const tDir =
|
|
43927
|
+
const tDir = join5(tmpDir, "transcripts");
|
|
42062
43928
|
await mkdir(tDir, { recursive: true });
|
|
42063
43929
|
for (const te of transcriptEmbeds) {
|
|
42064
|
-
await writeFile(
|
|
43930
|
+
await writeFile(join5(tDir, te.filename), te.content);
|
|
42065
43931
|
}
|
|
42066
43932
|
}
|
|
42067
|
-
const zipPath =
|
|
43933
|
+
const zipPath = join5(outputDir, `${filenameBase}.zip`);
|
|
42068
43934
|
const { execSync: execSync2 } = await import("child_process");
|
|
42069
43935
|
try {
|
|
42070
43936
|
execSync2(`cd "${tmpDir}" && zip -r "${zipPath}" .`, { stdio: "pipe" });
|
|
@@ -42079,17 +43945,17 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
|
|
|
42079
43945
|
);
|
|
42080
43946
|
}
|
|
42081
43947
|
} else {
|
|
42082
|
-
const chatDir =
|
|
43948
|
+
const chatDir = join5(outputDir, filenameBase);
|
|
42083
43949
|
await mkdir(chatDir, { recursive: true });
|
|
42084
43950
|
const written = [];
|
|
42085
|
-
await writeFile(
|
|
43951
|
+
await writeFile(join5(chatDir, `${filenameBase}.yml`), yamlContent);
|
|
42086
43952
|
written.push(`${filenameBase}.yml`);
|
|
42087
|
-
await writeFile(
|
|
43953
|
+
await writeFile(join5(chatDir, `${filenameBase}.md`), mdContent);
|
|
42088
43954
|
written.push(`${filenameBase}.md`);
|
|
42089
43955
|
if (codeEmbeds.length > 0) {
|
|
42090
43956
|
for (const ce of codeEmbeds) {
|
|
42091
43957
|
const fpath = ce.filePath ?? ce.filename ?? `${ce.embedId.slice(0, 8)}.${getExtForLang(ce.language)}`;
|
|
42092
|
-
const fullPath =
|
|
43958
|
+
const fullPath = join5(chatDir, "code", fpath);
|
|
42093
43959
|
await mkdir(fullPath.substring(0, fullPath.lastIndexOf("/")), {
|
|
42094
43960
|
recursive: true
|
|
42095
43961
|
});
|
|
@@ -42098,10 +43964,10 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
|
|
|
42098
43964
|
}
|
|
42099
43965
|
}
|
|
42100
43966
|
if (transcriptEmbeds.length > 0) {
|
|
42101
|
-
const tDir =
|
|
43967
|
+
const tDir = join5(chatDir, "transcripts");
|
|
42102
43968
|
await mkdir(tDir, { recursive: true });
|
|
42103
43969
|
for (const te of transcriptEmbeds) {
|
|
42104
|
-
await writeFile(
|
|
43970
|
+
await writeFile(join5(tDir, te.filename), te.content);
|
|
42105
43971
|
written.push(`transcripts/${te.filename}`);
|
|
42106
43972
|
}
|
|
42107
43973
|
}
|
|
@@ -42137,7 +44003,7 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
|
|
|
42137
44003
|
printJson2({
|
|
42138
44004
|
chat_id: chat.id,
|
|
42139
44005
|
title: chat.title,
|
|
42140
|
-
output_dir: useZip ?
|
|
44006
|
+
output_dir: useZip ? join5(outputDir, `${filenameBase}.zip`) : join5(outputDir, filenameBase),
|
|
42141
44007
|
files,
|
|
42142
44008
|
code_embeds: codeEmbeds.length,
|
|
42143
44009
|
transcript_embeds: transcriptEmbeds.length
|
|
@@ -42658,7 +44524,7 @@ async function handleCodeRun(client, flags, apiKey) {
|
|
|
42658
44524
|
}
|
|
42659
44525
|
}
|
|
42660
44526
|
async function streamCodeRunToTerminal(url, jsonMode) {
|
|
42661
|
-
return await new Promise((
|
|
44527
|
+
return await new Promise((resolve6, reject) => {
|
|
42662
44528
|
const ws = new WebSocket2(url);
|
|
42663
44529
|
let lastStatus = {};
|
|
42664
44530
|
ws.on("message", (data) => {
|
|
@@ -42677,7 +44543,7 @@ async function streamCodeRunToTerminal(url, jsonMode) {
|
|
|
42677
44543
|
const status = String(payload.status ?? "");
|
|
42678
44544
|
if (["finished", "failed", "timeout", "cancelled"].includes(status)) {
|
|
42679
44545
|
ws.close();
|
|
42680
|
-
|
|
44546
|
+
resolve6(lastStatus);
|
|
42681
44547
|
}
|
|
42682
44548
|
}
|
|
42683
44549
|
} catch (err) {
|
|
@@ -42687,7 +44553,7 @@ async function streamCodeRunToTerminal(url, jsonMode) {
|
|
|
42687
44553
|
});
|
|
42688
44554
|
ws.on("error", () => reject(new Error("Code Run stream failed.")));
|
|
42689
44555
|
ws.on("close", () => {
|
|
42690
|
-
if (Object.keys(lastStatus).length > 0)
|
|
44556
|
+
if (Object.keys(lastStatus).length > 0) resolve6(lastStatus);
|
|
42691
44557
|
});
|
|
42692
44558
|
});
|
|
42693
44559
|
}
|
|
@@ -42698,7 +44564,7 @@ async function pollCodeRunStatus(client, statusPath, apiKey, jsonMode) {
|
|
|
42698
44564
|
if (!jsonMode && value) process.stderr.write(`Code Run status: ${value}
|
|
42699
44565
|
`);
|
|
42700
44566
|
if (["finished", "failed", "timeout", "cancelled"].includes(value)) return status;
|
|
42701
|
-
await new Promise((
|
|
44567
|
+
await new Promise((resolve6) => setTimeout(resolve6, 1e3));
|
|
42702
44568
|
}
|
|
42703
44569
|
}
|
|
42704
44570
|
function buildSkillInput(flags, inlineTokens, schemaParams) {
|
|
@@ -42898,7 +44764,7 @@ async function handleEmbeds(client, subcommand, rest, flags) {
|
|
|
42898
44764
|
throw new Error("Embed version content was not available after local reconstruction.");
|
|
42899
44765
|
}
|
|
42900
44766
|
if (typeof flags.output === "string") {
|
|
42901
|
-
|
|
44767
|
+
writeFileSync5(flags.output, result.content, "utf-8");
|
|
42902
44768
|
if (flags.json === true) {
|
|
42903
44769
|
printJson2({ ...result, output: flags.output });
|
|
42904
44770
|
} else {
|
|
@@ -43039,6 +44905,25 @@ async function printSettingsMutationResult(resultPromise, flags) {
|
|
|
43039
44905
|
process.stdout.write("\x1B[32m\u2713\x1B[0m Settings updated\n");
|
|
43040
44906
|
if (result && typeof result === "object") printGenericObject(result);
|
|
43041
44907
|
}
|
|
44908
|
+
function printReportIssueCreateResult(result, flags) {
|
|
44909
|
+
if (flags.json === true) {
|
|
44910
|
+
printJson2(result);
|
|
44911
|
+
return;
|
|
44912
|
+
}
|
|
44913
|
+
process.stdout.write("\x1B[32m\u2713\x1B[0m Issue reported\n");
|
|
44914
|
+
const obj = result && typeof result === "object" ? result : {};
|
|
44915
|
+
const issueId = typeof obj.issue_id === "string" ? obj.issue_id : "";
|
|
44916
|
+
const shortIssueId = typeof obj.short_issue_id === "string" ? obj.short_issue_id : "";
|
|
44917
|
+
if (shortIssueId || issueId) {
|
|
44918
|
+
console.log(`Issue reference: ${shortIssueId || issueId}`);
|
|
44919
|
+
}
|
|
44920
|
+
if (issueId && shortIssueId && issueId !== shortIssueId) {
|
|
44921
|
+
console.log(`Internal issue ID: ${issueId}`);
|
|
44922
|
+
}
|
|
44923
|
+
if (typeof obj.message === "string") {
|
|
44924
|
+
console.log(obj.message);
|
|
44925
|
+
}
|
|
44926
|
+
}
|
|
43042
44927
|
function addQueryParam(params, key, value) {
|
|
43043
44928
|
if (typeof value === "string" && value.length > 0) params.set(key, value);
|
|
43044
44929
|
}
|
|
@@ -43182,11 +45067,11 @@ function parseYamlScalar(value) {
|
|
|
43182
45067
|
}
|
|
43183
45068
|
async function saveDownloadedDocument(document, output) {
|
|
43184
45069
|
const { mkdir, writeFile } = await import("fs/promises");
|
|
43185
|
-
const { join:
|
|
45070
|
+
const { join: join5, basename: basename4, dirname: dirname4 } = await import("path");
|
|
43186
45071
|
const target = typeof output === "string" ? output : ".";
|
|
43187
45072
|
const filename = basename4(document.filename || "document.pdf");
|
|
43188
|
-
const filePath = target.endsWith(".pdf") ? target :
|
|
43189
|
-
await mkdir(
|
|
45073
|
+
const filePath = target.endsWith(".pdf") ? target : join5(target, filename);
|
|
45074
|
+
await mkdir(dirname4(filePath), { recursive: true });
|
|
43190
45075
|
await writeFile(filePath, document.data);
|
|
43191
45076
|
return filePath;
|
|
43192
45077
|
}
|
|
@@ -43214,7 +45099,7 @@ function printMateInfo(mateId, json) {
|
|
|
43214
45099
|
async function confirmOrExit(question) {
|
|
43215
45100
|
const rl = await import("readline");
|
|
43216
45101
|
const iface = rl.createInterface({ input: process.stdin, output: process.stdout });
|
|
43217
|
-
const answer = await new Promise((
|
|
45102
|
+
const answer = await new Promise((resolve6) => iface.question(question, resolve6));
|
|
43218
45103
|
iface.close();
|
|
43219
45104
|
if (answer.trim().toLowerCase() !== "y") {
|
|
43220
45105
|
console.log("Aborted.");
|
|
@@ -43224,7 +45109,7 @@ async function confirmOrExit(question) {
|
|
|
43224
45109
|
async function promptLine(question) {
|
|
43225
45110
|
const rl = await import("readline");
|
|
43226
45111
|
const iface = rl.createInterface({ input: process.stdin, output: process.stdout });
|
|
43227
|
-
const answer = await new Promise((
|
|
45112
|
+
const answer = await new Promise((resolve6) => iface.question(question, resolve6));
|
|
43228
45113
|
iface.close();
|
|
43229
45114
|
return answer.trim();
|
|
43230
45115
|
}
|
|
@@ -43232,7 +45117,7 @@ async function promptSecret(question) {
|
|
|
43232
45117
|
if (!process.stdin.isTTY) {
|
|
43233
45118
|
return promptLine(question);
|
|
43234
45119
|
}
|
|
43235
|
-
return new Promise((
|
|
45120
|
+
return new Promise((resolve6) => {
|
|
43236
45121
|
const stdin2 = process.stdin;
|
|
43237
45122
|
const wasRaw = stdin2.isRaw;
|
|
43238
45123
|
let value = "";
|
|
@@ -43245,7 +45130,7 @@ async function promptSecret(question) {
|
|
|
43245
45130
|
stdin2.off("data", onData);
|
|
43246
45131
|
stdin2.setRawMode(wasRaw);
|
|
43247
45132
|
process.stdout.write("\n");
|
|
43248
|
-
|
|
45133
|
+
resolve6(value);
|
|
43249
45134
|
return;
|
|
43250
45135
|
}
|
|
43251
45136
|
if (char === "") {
|
|
@@ -43265,7 +45150,7 @@ async function promptSecret(question) {
|
|
|
43265
45150
|
}
|
|
43266
45151
|
async function writeSecretFile(filePath, content, force = false) {
|
|
43267
45152
|
const { mkdir, writeFile, stat: stat2 } = await import("fs/promises");
|
|
43268
|
-
const { dirname:
|
|
45153
|
+
const { dirname: dirname4 } = await import("path");
|
|
43269
45154
|
try {
|
|
43270
45155
|
await stat2(filePath);
|
|
43271
45156
|
if (!force) throw new Error(`${filePath} already exists. Use --force to overwrite.`);
|
|
@@ -43275,7 +45160,7 @@ async function writeSecretFile(filePath, content, force = false) {
|
|
|
43275
45160
|
}
|
|
43276
45161
|
if (error instanceof Error && !("code" in error)) throw error;
|
|
43277
45162
|
}
|
|
43278
|
-
await mkdir(
|
|
45163
|
+
await mkdir(dirname4(filePath), { recursive: true });
|
|
43279
45164
|
await writeFile(filePath, content, { mode: 384 });
|
|
43280
45165
|
return filePath;
|
|
43281
45166
|
}
|
|
@@ -43902,7 +45787,8 @@ async function handleSettings(client, subcommand, rest, flags) {
|
|
|
43902
45787
|
const title = typeof flags.title === "string" ? flags.title : void 0;
|
|
43903
45788
|
const body = typeof flags.body === "string" ? flags.body : void 0;
|
|
43904
45789
|
if (!title || !body) throw new Error("Provide --title and --body.");
|
|
43905
|
-
await
|
|
45790
|
+
const result = await client.settingsPost("issues", { title, description: body });
|
|
45791
|
+
printReportIssueCreateResult(result, flags);
|
|
43906
45792
|
return;
|
|
43907
45793
|
}
|
|
43908
45794
|
if (matches(tokens, ["report-issue", "status"])) {
|
|
@@ -43973,6 +45859,79 @@ async function handleSettings(client, subcommand, rest, flags) {
|
|
|
43973
45859
|
printSettingsHelp(client, [subcommand]);
|
|
43974
45860
|
process.exit(1);
|
|
43975
45861
|
}
|
|
45862
|
+
var LEARNING_MODE_AGE_GROUPS = /* @__PURE__ */ new Set([
|
|
45863
|
+
"under_10",
|
|
45864
|
+
"10_12",
|
|
45865
|
+
"13_15",
|
|
45866
|
+
"16_18",
|
|
45867
|
+
"adult"
|
|
45868
|
+
]);
|
|
45869
|
+
async function handleLearningMode(client, subcommand, flags) {
|
|
45870
|
+
if (!subcommand || subcommand === "help" || flags.help === true) {
|
|
45871
|
+
printLearningModeHelp();
|
|
45872
|
+
return;
|
|
45873
|
+
}
|
|
45874
|
+
if (subcommand === "status") {
|
|
45875
|
+
printLearningModeStatus(await client.getLearningModeStatus(), flags.json === true);
|
|
45876
|
+
return;
|
|
45877
|
+
}
|
|
45878
|
+
if (subcommand === "enable") {
|
|
45879
|
+
const ageGroup = parseLearningModeAgeGroup(flags["age-group"]);
|
|
45880
|
+
const passcode = parseRequiredStringFlag(flags.passcode, "--passcode");
|
|
45881
|
+
printLearningModeStatus(
|
|
45882
|
+
await client.activateLearningMode({ ageGroup, passcode }),
|
|
45883
|
+
flags.json === true
|
|
45884
|
+
);
|
|
45885
|
+
return;
|
|
45886
|
+
}
|
|
45887
|
+
if (subcommand === "disable") {
|
|
45888
|
+
const passcode = parseRequiredStringFlag(flags.passcode, "--passcode");
|
|
45889
|
+
printLearningModeStatus(await client.deactivateLearningMode(passcode), flags.json === true);
|
|
45890
|
+
return;
|
|
45891
|
+
}
|
|
45892
|
+
console.error(`Unknown learning-mode command '${subcommand}'.
|
|
45893
|
+
`);
|
|
45894
|
+
printLearningModeHelp();
|
|
45895
|
+
process.exit(1);
|
|
45896
|
+
}
|
|
45897
|
+
function parseLearningModeAgeGroup(value) {
|
|
45898
|
+
if (typeof value !== "string" || !LEARNING_MODE_AGE_GROUPS.has(value)) {
|
|
45899
|
+
throw new Error("Provide --age-group as one of: under_10, 10_12, 13_15, 16_18, adult.");
|
|
45900
|
+
}
|
|
45901
|
+
return value;
|
|
45902
|
+
}
|
|
45903
|
+
function parseAnonymousLearningModeFlags(flags) {
|
|
45904
|
+
if (flags["learning-mode"] !== true) return void 0;
|
|
45905
|
+
return {
|
|
45906
|
+
enabled: true,
|
|
45907
|
+
ageGroup: parseLearningModeAgeGroup(flags["age-group"]),
|
|
45908
|
+
source: "anonymous_session"
|
|
45909
|
+
};
|
|
45910
|
+
}
|
|
45911
|
+
function parseRequiredStringFlag(value, name) {
|
|
45912
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
45913
|
+
throw new Error(`Provide ${name}.`);
|
|
45914
|
+
}
|
|
45915
|
+
return value;
|
|
45916
|
+
}
|
|
45917
|
+
function printLearningModeStatus(status, json) {
|
|
45918
|
+
if (json) {
|
|
45919
|
+
printJson2(status);
|
|
45920
|
+
return;
|
|
45921
|
+
}
|
|
45922
|
+
console.log(`Learning Mode: ${status.enabled ? "enabled" : "disabled"}`);
|
|
45923
|
+
if (status.age_group) console.log(`Age group: ${status.age_group}`);
|
|
45924
|
+
if (status.failed_attempts > 0) console.log(`Failed disable attempts: ${status.failed_attempts}`);
|
|
45925
|
+
if (status.deactivation_blocked_until) {
|
|
45926
|
+
console.log(`Disable blocked until: ${new Date(status.deactivation_blocked_until * 1e3).toISOString()}`);
|
|
45927
|
+
}
|
|
45928
|
+
}
|
|
45929
|
+
function learningModeStatusToContext(status) {
|
|
45930
|
+
return {
|
|
45931
|
+
enabled: status.enabled,
|
|
45932
|
+
ageGroup: status.age_group
|
|
45933
|
+
};
|
|
45934
|
+
}
|
|
43976
45935
|
async function handleMemories(client, rest, flags) {
|
|
43977
45936
|
const action = rest[0];
|
|
43978
45937
|
if (!action || action === "help") {
|
|
@@ -44563,7 +46522,10 @@ async function sendMessageStreaming(client, params, redactor) {
|
|
|
44563
46522
|
if (!client.hasSession()) {
|
|
44564
46523
|
let result2;
|
|
44565
46524
|
try {
|
|
44566
|
-
result2 = await client.sendAnonymousMessage({
|
|
46525
|
+
result2 = await client.sendAnonymousMessage({
|
|
46526
|
+
message: finalMessage,
|
|
46527
|
+
learningMode: params.anonymousLearningMode
|
|
46528
|
+
});
|
|
44567
46529
|
} finally {
|
|
44568
46530
|
clearTyping();
|
|
44569
46531
|
}
|
|
@@ -44584,6 +46546,7 @@ async function sendMessageStreaming(client, params, redactor) {
|
|
|
44584
46546
|
const urlResult = prepareUrlEmbeds(finalMessage);
|
|
44585
46547
|
finalMessage = urlResult.message;
|
|
44586
46548
|
preparedEmbeds.push(...urlResult.embeds);
|
|
46549
|
+
const learningMode = learningModeStatusToContext(await client.getLearningModeStatus());
|
|
44587
46550
|
const result = await client.sendMessage({
|
|
44588
46551
|
message: finalMessage,
|
|
44589
46552
|
chatId: params.chatId,
|
|
@@ -44593,6 +46556,7 @@ async function sendMessageStreaming(client, params, redactor) {
|
|
|
44593
46556
|
onSubChatApprovalRequest,
|
|
44594
46557
|
autoApproveSubChats: params.autoApproveSubChats,
|
|
44595
46558
|
autoApproveMemories: params.autoApproveMemories,
|
|
46559
|
+
learningMode,
|
|
44596
46560
|
preparedEmbeds: preparedEmbeds.length > 0 ? preparedEmbeds : void 0,
|
|
44597
46561
|
piiMappings: piiResult.mappings.map((mapping) => ({
|
|
44598
46562
|
placeholder: mapping.placeholder,
|
|
@@ -45899,9 +47863,11 @@ Commands:
|
|
|
45899
47863
|
openmates mentions [--help] List available @mentions
|
|
45900
47864
|
openmates embeds [--help] Embed commands (show)
|
|
45901
47865
|
openmates settings [--help] Predefined settings commands
|
|
47866
|
+
openmates learning-mode [--help] Account-wide Learning Mode controls
|
|
45902
47867
|
openmates inspirations [--lang <code>] [--json] Daily inspirations
|
|
45903
47868
|
openmates newchatsuggestions [--limit <n>] [--json] Personalized new chat suggestions
|
|
45904
47869
|
openmates feedback [--help] Assistant response feedback helpers
|
|
47870
|
+
openmates benchmark [--help] Run real model benchmarks with usage tagged as benchmark spend
|
|
45905
47871
|
openmates server [--help] Server management (install, start, stop, ...)
|
|
45906
47872
|
openmates docs [--help] Browse, search, and download documentation
|
|
45907
47873
|
openmates e2e provision-auth-accounts Provision local E2E auth-account artifacts
|
|
@@ -45924,6 +47890,22 @@ Options:
|
|
|
45924
47890
|
--rating <1-5> Required star rating
|
|
45925
47891
|
--json Output the decision contract as JSON`);
|
|
45926
47892
|
}
|
|
47893
|
+
function printLearningModeHelp() {
|
|
47894
|
+
console.log(`Learning Mode commands:
|
|
47895
|
+
openmates learning-mode status [--json]
|
|
47896
|
+
openmates learning-mode enable --age-group <group> --passcode <passcode> [--json]
|
|
47897
|
+
openmates learning-mode disable --passcode <passcode> [--json]
|
|
47898
|
+
|
|
47899
|
+
Learning Mode is account-wide and applies to CLI, web, Apple, and API chat requests.
|
|
47900
|
+
|
|
47901
|
+
Age groups:
|
|
47902
|
+
under_10, 10_12, 13_15, 16_18, adult
|
|
47903
|
+
|
|
47904
|
+
Options:
|
|
47905
|
+
--age-group <group> Required for enable
|
|
47906
|
+
--passcode <value> Required for enable and disable
|
|
47907
|
+
--json Output backend status JSON`);
|
|
47908
|
+
}
|
|
45927
47909
|
function printSignupHelp() {
|
|
45928
47910
|
console.log(`Signup command:
|
|
45929
47911
|
openmates signup --email <email> --username <name> --invite-code <code>
|
|
@@ -45964,9 +47946,10 @@ function printChatsHelp() {
|
|
|
45964
47946
|
openmates chats show <chat-id> [--raw] [--json]
|
|
45965
47947
|
openmates chats open [<n|example-id|slug>] [--json]
|
|
45966
47948
|
openmates chats search <query> [--json]
|
|
45967
|
-
openmates chats new <message> [--json] [--auto-approve] [--auto-approve-memories] [--no-pii-detection]
|
|
47949
|
+
openmates chats new <message> [--json] [--learning-mode --age-group <group>] [--auto-approve] [--auto-approve-memories] [--no-pii-detection]
|
|
45968
47950
|
openmates chats send [--chat <id>] [--incognito] <message> [--json] [--auto-approve] [--auto-approve-memories] [--no-pii-detection]
|
|
45969
47951
|
openmates chats send --chat <id> --followup <n> [--json] [--auto-approve] [--auto-approve-memories]
|
|
47952
|
+
openmates chats answer-interactive --chat <id> --question-json '<json>' --answer-json '<json>' [--json]
|
|
45970
47953
|
openmates chats download <chat-id> [--output <path>] [--zip] [--json]
|
|
45971
47954
|
openmates chats delete <id1> [id2] [id3] ... [--yes]
|
|
45972
47955
|
openmates chats share [<chat-id>] [--expires <seconds>] [--password <pwd>] [--json]
|
|
@@ -45995,6 +47978,11 @@ Options for 'send':
|
|
|
45995
47978
|
typing the full message (requires --chat)
|
|
45996
47979
|
--incognito Send without saving to chat history
|
|
45997
47980
|
|
|
47981
|
+
Options for 'answer-interactive':
|
|
47982
|
+
--chat <id> Chat containing the interactive question
|
|
47983
|
+
--question-json The question payload returned by 'chats send --json'
|
|
47984
|
+
--answer-json Structured answer JSON, for example '{"selection":["opt_a"]}'
|
|
47985
|
+
|
|
45998
47986
|
Options for 'new', 'send', and 'incognito':
|
|
45999
47987
|
--auto-approve Automatically approve server-requested sub-chat batches.
|
|
46000
47988
|
Without this, the CLI prompts in the terminal like the web app.
|
|
@@ -46004,6 +47992,11 @@ Options for 'new', 'send', and 'incognito':
|
|
|
46004
47992
|
--no-pii-detection Send the message exactly as typed. By default, the CLI
|
|
46005
47993
|
replaces detected PII with placeholders before send.
|
|
46006
47994
|
|
|
47995
|
+
Guest-only options for logged-out 'new':
|
|
47996
|
+
--learning-mode Opt anonymous chat into request-scoped Learning Mode.
|
|
47997
|
+
--age-group <group> Required with --learning-mode: under_10, 10_12,
|
|
47998
|
+
13_15, 16_18, or adult.
|
|
47999
|
+
|
|
46007
48000
|
Options for 'download':
|
|
46008
48001
|
--output <path> Target directory (default: current directory)
|
|
46009
48002
|
--zip Create a .zip archive instead of a folder
|
|
@@ -46040,6 +48033,7 @@ Examples:
|
|
|
46040
48033
|
openmates chats show "Flight Connections Berlin to Bangkok"
|
|
46041
48034
|
openmates chats search "Madrid"
|
|
46042
48035
|
openmates chats new "Hello, what can you help me with?"
|
|
48036
|
+
openmates chats new "Help me understand fractions" --learning-mode --age-group 10_12
|
|
46043
48037
|
openmates chats send --chat d262cb68 "follow-up question"
|
|
46044
48038
|
openmates chats send --chat d262cb68 --followup 1
|
|
46045
48039
|
openmates chats send --chat d262cb68 --followup 3
|
|
@@ -46232,7 +48226,7 @@ async function handleDocs(client, subcommand, rest, flags) {
|
|
|
46232
48226
|
}
|
|
46233
48227
|
if (subcommand === "download") {
|
|
46234
48228
|
const { writeFile, mkdir } = await import("fs/promises");
|
|
46235
|
-
const { join:
|
|
48229
|
+
const { join: join5, dirname: dirname4 } = await import("path");
|
|
46236
48230
|
if (flags.all === true) {
|
|
46237
48231
|
const outputDir = typeof flags.output === "string" ? flags.output : "./openmates-docs";
|
|
46238
48232
|
const tree = await client.listDocs();
|
|
@@ -46241,8 +48235,8 @@ async function handleDocs(client, subcommand, rest, flags) {
|
|
|
46241
48235
|
let count = 0;
|
|
46242
48236
|
for (const slug2 of slugs) {
|
|
46243
48237
|
const content2 = await client.getDoc(slug2);
|
|
46244
|
-
const filePath =
|
|
46245
|
-
await mkdir(
|
|
48238
|
+
const filePath = join5(outputDir, `${slug2}.md`);
|
|
48239
|
+
await mkdir(dirname4(filePath), { recursive: true });
|
|
46246
48240
|
await writeFile(filePath, content2, "utf-8");
|
|
46247
48241
|
count++;
|
|
46248
48242
|
process.stderr.write(`\r Downloaded ${count}/${slugs.length}`);
|
|
@@ -46314,8 +48308,8 @@ function isCliEntrypoint() {
|
|
|
46314
48308
|
if (!entrypoint) return false;
|
|
46315
48309
|
try {
|
|
46316
48310
|
const invokedPath = realpathSync(entrypoint);
|
|
46317
|
-
const modulePath = realpathSync(
|
|
46318
|
-
return invokedPath === modulePath || basename3(invokedPath) === "cli.js" &&
|
|
48311
|
+
const modulePath = realpathSync(fileURLToPath2(import.meta.url));
|
|
48312
|
+
return invokedPath === modulePath || basename3(invokedPath) === "cli.js" && dirname3(invokedPath) === dirname3(modulePath);
|
|
46319
48313
|
} catch {
|
|
46320
48314
|
return false;
|
|
46321
48315
|
}
|