openmates 0.15.0-alpha.74 → 0.15.0-alpha.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-PR4HA5JZ.js → chunk-YZ75YWCS.js} +946 -73
- package/dist/cli.d.ts +2 -1
- package/dist/cli.js +3 -1
- package/dist/index.d.ts +174 -9
- package/dist/index.js +1 -1
- package/package.json +1 -1
|
@@ -1356,6 +1356,9 @@ function websocketProtocolError(envelope) {
|
|
|
1356
1356
|
}
|
|
1357
1357
|
return null;
|
|
1358
1358
|
}
|
|
1359
|
+
function numberMetric(value) {
|
|
1360
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
1361
|
+
}
|
|
1359
1362
|
function errorFrameBelongsToAiResponse(envelope, userMessageId, chatId, recoveryTurnId) {
|
|
1360
1363
|
if (envelope.type === "client_update_required") return true;
|
|
1361
1364
|
if (envelope.type !== "error") return true;
|
|
@@ -1722,6 +1725,8 @@ var OpenMatesWsClient = class {
|
|
|
1722
1725
|
let taskId = null;
|
|
1723
1726
|
let category = null;
|
|
1724
1727
|
let modelName = null;
|
|
1728
|
+
let tokenUsage = null;
|
|
1729
|
+
let promptBudget = null;
|
|
1725
1730
|
let recoveryJobId = null;
|
|
1726
1731
|
const compressionCheckpoints = [];
|
|
1727
1732
|
let followUpSuggestions = [];
|
|
@@ -1767,6 +1772,24 @@ var OpenMatesWsClient = class {
|
|
|
1767
1772
|
if (typeof p.category === "string" && p.category) category = p.category;
|
|
1768
1773
|
if (typeof p.model_name === "string" && p.model_name)
|
|
1769
1774
|
modelName = p.model_name;
|
|
1775
|
+
const promptTokens = numberMetric(p.prompt_tokens);
|
|
1776
|
+
const completionTokens = numberMetric(p.completion_tokens);
|
|
1777
|
+
const userInputTokens = numberMetric(p.user_input_tokens);
|
|
1778
|
+
const systemPromptTokens = numberMetric(p.system_prompt_tokens);
|
|
1779
|
+
const totalCredits = numberMetric(p.total_credits);
|
|
1780
|
+
const hasTokenUsage = promptTokens !== null || completionTokens !== null || userInputTokens !== null || systemPromptTokens !== null || totalCredits !== null;
|
|
1781
|
+
if (hasTokenUsage) {
|
|
1782
|
+
tokenUsage = {
|
|
1783
|
+
...promptTokens !== null ? { promptTokens } : {},
|
|
1784
|
+
...completionTokens !== null ? { completionTokens } : {},
|
|
1785
|
+
...userInputTokens !== null ? { userInputTokens } : {},
|
|
1786
|
+
...systemPromptTokens !== null ? { systemPromptTokens } : {},
|
|
1787
|
+
...totalCredits !== null ? { totalCredits } : {}
|
|
1788
|
+
};
|
|
1789
|
+
}
|
|
1790
|
+
if (systemPromptTokens !== null) {
|
|
1791
|
+
promptBudget = { systemPromptTokens };
|
|
1792
|
+
}
|
|
1770
1793
|
if (typeof p.recovery_job_id === "string" && p.recovery_job_id && p.recovery_protocol_version === 1)
|
|
1771
1794
|
recoveryJobId = p.recovery_job_id;
|
|
1772
1795
|
};
|
|
@@ -1809,7 +1832,9 @@ var OpenMatesWsClient = class {
|
|
|
1809
1832
|
embeds: [...embeds.values()],
|
|
1810
1833
|
subChatEvents,
|
|
1811
1834
|
recoveryJobId,
|
|
1812
|
-
compressionCheckpoints
|
|
1835
|
+
compressionCheckpoints,
|
|
1836
|
+
tokenUsage,
|
|
1837
|
+
promptBudget
|
|
1813
1838
|
});
|
|
1814
1839
|
return;
|
|
1815
1840
|
}
|
|
@@ -1839,7 +1864,9 @@ var OpenMatesWsClient = class {
|
|
|
1839
1864
|
embeds: [...embeds.values()],
|
|
1840
1865
|
subChatEvents,
|
|
1841
1866
|
recoveryJobId,
|
|
1842
|
-
compressionCheckpoints
|
|
1867
|
+
compressionCheckpoints,
|
|
1868
|
+
tokenUsage,
|
|
1869
|
+
promptBudget
|
|
1843
1870
|
});
|
|
1844
1871
|
}, asyncEmbedWaitMs);
|
|
1845
1872
|
return;
|
|
@@ -1865,7 +1892,9 @@ var OpenMatesWsClient = class {
|
|
|
1865
1892
|
embeds: [...embeds.values()],
|
|
1866
1893
|
subChatEvents,
|
|
1867
1894
|
recoveryJobId,
|
|
1868
|
-
compressionCheckpoints
|
|
1895
|
+
compressionCheckpoints,
|
|
1896
|
+
tokenUsage,
|
|
1897
|
+
promptBudget
|
|
1869
1898
|
});
|
|
1870
1899
|
};
|
|
1871
1900
|
const handleSubChatEvent = (type, p) => {
|
|
@@ -2139,7 +2168,9 @@ var OpenMatesWsClient = class {
|
|
|
2139
2168
|
embeds: [...embeds.values()],
|
|
2140
2169
|
subChatEvents,
|
|
2141
2170
|
recoveryJobId,
|
|
2142
|
-
compressionCheckpoints
|
|
2171
|
+
compressionCheckpoints,
|
|
2172
|
+
tokenUsage,
|
|
2173
|
+
promptBudget
|
|
2143
2174
|
});
|
|
2144
2175
|
return;
|
|
2145
2176
|
}
|
|
@@ -6388,7 +6419,9 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
6388
6419
|
taskProposals: [],
|
|
6389
6420
|
taskUpdateProposals: [],
|
|
6390
6421
|
subChatEvents: [],
|
|
6391
|
-
appSettingsMemoryRequests: []
|
|
6422
|
+
appSettingsMemoryRequests: [],
|
|
6423
|
+
tokenUsage: null,
|
|
6424
|
+
promptBudget: null
|
|
6392
6425
|
};
|
|
6393
6426
|
}
|
|
6394
6427
|
async createTurnTokenRefs(params) {
|
|
@@ -6574,7 +6607,7 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
6574
6607
|
if (!userId) {
|
|
6575
6608
|
throw new Error("Could not resolve current user id for local connected-account connector.");
|
|
6576
6609
|
}
|
|
6577
|
-
const { createHash:
|
|
6610
|
+
const { createHash: createHash15 } = await import("crypto");
|
|
6578
6611
|
const accountId = randomUUID5();
|
|
6579
6612
|
const masterKey = this.getMasterKeyBytes();
|
|
6580
6613
|
const encryptLocalConnectorValue = async (value) => {
|
|
@@ -6583,9 +6616,9 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
6583
6616
|
};
|
|
6584
6617
|
const payload = {
|
|
6585
6618
|
id: accountId,
|
|
6586
|
-
hashed_user_id:
|
|
6619
|
+
hashed_user_id: createHash15("sha256").update(userId).digest("hex"),
|
|
6587
6620
|
encrypted_provider_type: await encryptLocalConnectorValue(input.provider_id),
|
|
6588
|
-
provider_type_hash:
|
|
6621
|
+
provider_type_hash: createHash15("sha256").update(input.provider_id).digest("hex"),
|
|
6589
6622
|
encrypted_account_label: await encryptLocalConnectorValue(input.label),
|
|
6590
6623
|
encrypted_capabilities: await encryptLocalConnectorValue(input.capabilities),
|
|
6591
6624
|
encrypted_app_permissions: await encryptLocalConnectorValue({
|
|
@@ -7970,8 +8003,8 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
7970
8003
|
);
|
|
7971
8004
|
}
|
|
7972
8005
|
const embedId = String(embed.embed_id ?? embed.id ?? "");
|
|
7973
|
-
const { createHash:
|
|
7974
|
-
const hashedEmbedId =
|
|
8006
|
+
const { createHash: createHash15 } = await import("crypto");
|
|
8007
|
+
const hashedEmbedId = createHash15("sha256").update(embedId).digest("hex");
|
|
7975
8008
|
const embedKeyBytes = await this.resolveEmbedKey(
|
|
7976
8009
|
cache,
|
|
7977
8010
|
masterKey,
|
|
@@ -8079,7 +8112,7 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
8079
8112
|
async resolveEmbedKey(cache, masterKey, embed, embedId, hashedEmbedId, visited = /* @__PURE__ */ new Set()) {
|
|
8080
8113
|
if (visited.has(embedId)) return null;
|
|
8081
8114
|
visited.add(embedId);
|
|
8082
|
-
const { createHash:
|
|
8115
|
+
const { createHash: createHash15 } = await import("crypto");
|
|
8083
8116
|
const masterKeyEntry = cache.embedKeys.find(
|
|
8084
8117
|
(ek) => ek.hashed_embed_id === hashedEmbedId && String(ek.key_type) === "master"
|
|
8085
8118
|
);
|
|
@@ -8096,7 +8129,7 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
8096
8129
|
if (chatKeyEntry && typeof chatKeyEntry.encrypted_embed_key === "string") {
|
|
8097
8130
|
const hashedChatId = String(chatKeyEntry.hashed_chat_id ?? "");
|
|
8098
8131
|
const owningChat = cache.chats.find((c) => {
|
|
8099
|
-
const chatHash =
|
|
8132
|
+
const chatHash = createHash15("sha256").update(String(c.details.id ?? "")).digest("hex");
|
|
8100
8133
|
return chatHash === hashedChatId;
|
|
8101
8134
|
});
|
|
8102
8135
|
if (owningChat) {
|
|
@@ -8125,7 +8158,7 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
8125
8158
|
const parentFullId = String(
|
|
8126
8159
|
parentEmbed2.embed_id ?? parentEmbed2.id ?? ""
|
|
8127
8160
|
);
|
|
8128
|
-
const parentHashedId =
|
|
8161
|
+
const parentHashedId = createHash15("sha256").update(parentFullId).digest("hex");
|
|
8129
8162
|
const parentKey = await this.resolveEmbedKey(
|
|
8130
8163
|
cache,
|
|
8131
8164
|
masterKey,
|
|
@@ -8555,6 +8588,8 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
8555
8588
|
let assistantMessageId = null;
|
|
8556
8589
|
let category = null;
|
|
8557
8590
|
let modelName = null;
|
|
8591
|
+
let tokenUsage = null;
|
|
8592
|
+
let promptBudget = null;
|
|
8558
8593
|
let followUpSuggestions = [];
|
|
8559
8594
|
let taskProposals = [];
|
|
8560
8595
|
let taskUpdateProposals = [];
|
|
@@ -8786,6 +8821,8 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
8786
8821
|
assistant = resp.content;
|
|
8787
8822
|
category = resp.category;
|
|
8788
8823
|
modelName = resp.modelName;
|
|
8824
|
+
tokenUsage = resp.tokenUsage;
|
|
8825
|
+
promptBudget = resp.promptBudget;
|
|
8789
8826
|
taskEvents = resp.taskEvents;
|
|
8790
8827
|
pendingTaskUpdateJobs = taskUpdateJobsEnabled ? resp.pendingTaskUpdateJobs : [];
|
|
8791
8828
|
subChatEvents = resp.subChatEvents;
|
|
@@ -8805,7 +8842,9 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
8805
8842
|
taskEvents,
|
|
8806
8843
|
pendingTaskUpdateJobs,
|
|
8807
8844
|
subChatEvents,
|
|
8808
|
-
appSettingsMemoryRequests
|
|
8845
|
+
appSettingsMemoryRequests,
|
|
8846
|
+
tokenUsage,
|
|
8847
|
+
promptBudget
|
|
8809
8848
|
};
|
|
8810
8849
|
}
|
|
8811
8850
|
} finally {
|
|
@@ -8822,6 +8861,8 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
8822
8861
|
assistant = resp.content;
|
|
8823
8862
|
category = resp.category;
|
|
8824
8863
|
modelName = resp.modelName;
|
|
8864
|
+
tokenUsage = resp.tokenUsage;
|
|
8865
|
+
promptBudget = resp.promptBudget;
|
|
8825
8866
|
followUpSuggestions = resp.followUpSuggestions;
|
|
8826
8867
|
taskProposals = resp.taskProposals;
|
|
8827
8868
|
taskUpdateProposals = resp.taskUpdateProposals;
|
|
@@ -8844,7 +8885,9 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
8844
8885
|
taskEvents,
|
|
8845
8886
|
pendingTaskUpdateJobs,
|
|
8846
8887
|
subChatEvents,
|
|
8847
|
-
appSettingsMemoryRequests
|
|
8888
|
+
appSettingsMemoryRequests,
|
|
8889
|
+
tokenUsage,
|
|
8890
|
+
promptBudget
|
|
8848
8891
|
};
|
|
8849
8892
|
}
|
|
8850
8893
|
if (chatKeyBytes) {
|
|
@@ -8898,7 +8941,9 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
8898
8941
|
taskEvents,
|
|
8899
8942
|
pendingTaskUpdateJobs,
|
|
8900
8943
|
subChatEvents,
|
|
8901
|
-
appSettingsMemoryRequests
|
|
8944
|
+
appSettingsMemoryRequests,
|
|
8945
|
+
tokenUsage,
|
|
8946
|
+
promptBudget
|
|
8902
8947
|
};
|
|
8903
8948
|
}
|
|
8904
8949
|
if (claim.state !== "LEASED" || !leaseToken || leaseGeneration === null || leaseGeneration < 1 || !assistantId || keyVersion === null || keyVersion !== 1 || claim.chat_id !== chatId || claim.turn_id !== savedTurnId || typeof claim.sealed_payload !== "string") {
|
|
@@ -9072,7 +9117,9 @@ var OpenMatesClient = class _OpenMatesClient {
|
|
|
9072
9117
|
taskEvents,
|
|
9073
9118
|
pendingTaskUpdateJobs,
|
|
9074
9119
|
subChatEvents,
|
|
9075
|
-
appSettingsMemoryRequests
|
|
9120
|
+
appSettingsMemoryRequests,
|
|
9121
|
+
tokenUsage,
|
|
9122
|
+
promptBudget
|
|
9076
9123
|
};
|
|
9077
9124
|
}
|
|
9078
9125
|
async persistEncryptedSystemMessage(ws, systemMessage, targetChatId) {
|
|
@@ -11752,7 +11799,7 @@ Required: ${schema.required.join(", ")}`
|
|
|
11752
11799
|
const session = this.requireSession();
|
|
11753
11800
|
const masterKey = base64ToBytes(session.masterKeyExportedB64);
|
|
11754
11801
|
const cache = await this.ensureSynced();
|
|
11755
|
-
const { createHash:
|
|
11802
|
+
const { createHash: createHash15 } = await import("crypto");
|
|
11756
11803
|
const embed = cache.embeds.find(
|
|
11757
11804
|
(e) => String(e.embed_id ?? "").startsWith(embedIdOrShort) || String(e.id ?? "").startsWith(embedIdOrShort)
|
|
11758
11805
|
);
|
|
@@ -11760,7 +11807,7 @@ Required: ${schema.required.join(", ")}`
|
|
|
11760
11807
|
throw new Error(`Embed '${embedIdOrShort}' not found in local cache.`);
|
|
11761
11808
|
}
|
|
11762
11809
|
const embedId = String(embed.embed_id ?? embed.id ?? "");
|
|
11763
|
-
const hashedEmbedId =
|
|
11810
|
+
const hashedEmbedId = createHash15("sha256").update(embedId).digest("hex");
|
|
11764
11811
|
const embedKeyBytes = await this.resolveEmbedKey(
|
|
11765
11812
|
cache,
|
|
11766
11813
|
masterKey,
|
|
@@ -11868,8 +11915,8 @@ Required: ${schema.required.join(", ")}`
|
|
|
11868
11915
|
if (!embed) {
|
|
11869
11916
|
throw new Error(`Embed '${embedId}' not found in local cache. Run 'openmates chats list' to sync first.`);
|
|
11870
11917
|
}
|
|
11871
|
-
const { createHash:
|
|
11872
|
-
const hashedEmbedId =
|
|
11918
|
+
const { createHash: createHash15 } = await import("crypto");
|
|
11919
|
+
const hashedEmbedId = createHash15("sha256").update(embedId).digest("hex");
|
|
11873
11920
|
const embedKey = await this.resolveEmbedKey(
|
|
11874
11921
|
cache,
|
|
11875
11922
|
masterKey,
|
|
@@ -11985,8 +12032,8 @@ Required: ${schema.required.join(", ")}`
|
|
|
11985
12032
|
if (!embed) {
|
|
11986
12033
|
throw new Error(`Embed '${embedId}' not found in local cache. Run 'openmates chats list' to sync first.`);
|
|
11987
12034
|
}
|
|
11988
|
-
const { createHash:
|
|
11989
|
-
const hashedEmbedId =
|
|
12035
|
+
const { createHash: createHash15 } = await import("crypto");
|
|
12036
|
+
const hashedEmbedId = createHash15("sha256").update(embedId).digest("hex");
|
|
11990
12037
|
const embedKey = await this.resolveEmbedKey(
|
|
11991
12038
|
cache,
|
|
11992
12039
|
masterKey,
|
|
@@ -12425,9 +12472,9 @@ Required: ${schema.required.join(", ")}`
|
|
|
12425
12472
|
chats.splice(0, chats.length, ...reconciledChats);
|
|
12426
12473
|
if (totalChatCount === 0) totalChatCount = chats.length;
|
|
12427
12474
|
if (reconciliation.deleted_chat_ids?.length) {
|
|
12428
|
-
const { createHash:
|
|
12475
|
+
const { createHash: createHash15 } = await import("crypto");
|
|
12429
12476
|
const deletedHashes = new Set(
|
|
12430
|
-
reconciliation.deleted_chat_ids.map((id) =>
|
|
12477
|
+
reconciliation.deleted_chat_ids.map((id) => createHash15("sha256").update(id).digest("hex"))
|
|
12431
12478
|
);
|
|
12432
12479
|
const keptEmbeds = embeds.filter((embed) => !deletedHashes.has(String(embed.hashed_chat_id ?? "")));
|
|
12433
12480
|
const keptKeys = embedKeys.filter((key) => !deletedHashes.has(String(key.hashed_chat_id ?? "")));
|
|
@@ -12904,7 +12951,7 @@ var APP_SKILL_METADATA = [
|
|
|
12904
12951
|
"app_namespace_py": "code",
|
|
12905
12952
|
"skill_method_py": "image_to_html",
|
|
12906
12953
|
"description_key": "app_skills.code.image_to_html.description",
|
|
12907
|
-
"description": "Turn an uploaded screenshot into one standalone index.html code embed with inline CSS and optional inline JavaScript. Use this when the user asks to convert a screenshot, mockup, UI image, or app screen into HTML. Do not use remote image URLs;
|
|
12954
|
+
"description": "Turn an uploaded screenshot into one standalone index.html code embed with inline CSS and optional inline JavaScript. Use this when the user asks to convert a screenshot, mockup, UI image, or app screen into HTML. Do not use remote image URLs; pass source_image with the uploaded image embed_ref in chat, or image_base64 for REST/API calls.",
|
|
12908
12955
|
"schema": {
|
|
12909
12956
|
"type": "object",
|
|
12910
12957
|
"properties": {
|
|
@@ -12916,7 +12963,11 @@ var APP_SKILL_METADATA = [
|
|
|
12916
12963
|
"properties": {
|
|
12917
12964
|
"image_base64": {
|
|
12918
12965
|
"type": "string",
|
|
12919
|
-
"description": "Base64-encoded PNG, JPEG, or WEBP image bytes."
|
|
12966
|
+
"description": "Base64-encoded PNG, JPEG, or WEBP image bytes. Use this for REST/API calls."
|
|
12967
|
+
},
|
|
12968
|
+
"source_image": {
|
|
12969
|
+
"type": "string",
|
|
12970
|
+
"description": "Uploaded image filename/embed_ref from the conversation. Use this in chat/CLI when the user references an attached screenshot or mockup."
|
|
12920
12971
|
},
|
|
12921
12972
|
"mime_type": {
|
|
12922
12973
|
"type": "string",
|
|
@@ -12925,7 +12976,7 @@ var APP_SKILL_METADATA = [
|
|
|
12925
12976
|
"image/jpeg",
|
|
12926
12977
|
"image/webp"
|
|
12927
12978
|
],
|
|
12928
|
-
"description": "MIME type for image_base64."
|
|
12979
|
+
"description": "MIME type for image_base64. Optional when using source_image because the server resolves it from the uploaded image."
|
|
12929
12980
|
},
|
|
12930
12981
|
"filename": {
|
|
12931
12982
|
"type": "string",
|
|
@@ -12938,11 +12989,7 @@ var APP_SKILL_METADATA = [
|
|
|
12938
12989
|
"default": 2,
|
|
12939
12990
|
"description": "Maximum E2B render-feedback correction passes."
|
|
12940
12991
|
}
|
|
12941
|
-
}
|
|
12942
|
-
"required": [
|
|
12943
|
-
"image_base64",
|
|
12944
|
-
"mime_type"
|
|
12945
|
-
]
|
|
12992
|
+
}
|
|
12946
12993
|
}
|
|
12947
12994
|
}
|
|
12948
12995
|
},
|
|
@@ -14529,6 +14576,106 @@ Note: The 'id' field is auto-generated if not provided - you don't need to inclu
|
|
|
14529
14576
|
]
|
|
14530
14577
|
}
|
|
14531
14578
|
},
|
|
14579
|
+
{
|
|
14580
|
+
"app_id": "plans",
|
|
14581
|
+
"skill_id": "create",
|
|
14582
|
+
"app_namespace_ts": "plans",
|
|
14583
|
+
"skill_method_ts": "create",
|
|
14584
|
+
"app_namespace_py": "plans",
|
|
14585
|
+
"skill_method_py": "create",
|
|
14586
|
+
"description_key": "plans.skills.create.description",
|
|
14587
|
+
"description": "Request creation of a user-visible plan. Durable plan content is client-side encrypted, so a capable connected client must apply the change.",
|
|
14588
|
+
"schema": {
|
|
14589
|
+
"type": "object",
|
|
14590
|
+
"properties": {
|
|
14591
|
+
"title": {
|
|
14592
|
+
"type": "string",
|
|
14593
|
+
"description": "Short user-facing plan title."
|
|
14594
|
+
},
|
|
14595
|
+
"summary": {
|
|
14596
|
+
"type": "string",
|
|
14597
|
+
"description": "Optional plan summary."
|
|
14598
|
+
},
|
|
14599
|
+
"goal": {
|
|
14600
|
+
"type": "string",
|
|
14601
|
+
"description": "Optional plan goal."
|
|
14602
|
+
}
|
|
14603
|
+
},
|
|
14604
|
+
"required": [
|
|
14605
|
+
"title"
|
|
14606
|
+
]
|
|
14607
|
+
}
|
|
14608
|
+
},
|
|
14609
|
+
{
|
|
14610
|
+
"app_id": "plans",
|
|
14611
|
+
"skill_id": "search",
|
|
14612
|
+
"app_namespace_ts": "plans",
|
|
14613
|
+
"skill_method_ts": "search",
|
|
14614
|
+
"app_namespace_py": "plans",
|
|
14615
|
+
"skill_method_py": "search",
|
|
14616
|
+
"description_key": "plans.skills.search.description",
|
|
14617
|
+
"description": "Search the user's encrypted plans through a connected capable client. Do not use server-visible metadata as a private plan-content fallback.",
|
|
14618
|
+
"schema": {
|
|
14619
|
+
"type": "object",
|
|
14620
|
+
"properties": {
|
|
14621
|
+
"query": {
|
|
14622
|
+
"type": "string",
|
|
14623
|
+
"description": "Private plan text to search for on a connected client."
|
|
14624
|
+
}
|
|
14625
|
+
},
|
|
14626
|
+
"required": [
|
|
14627
|
+
"query"
|
|
14628
|
+
]
|
|
14629
|
+
}
|
|
14630
|
+
},
|
|
14631
|
+
{
|
|
14632
|
+
"app_id": "projects",
|
|
14633
|
+
"skill_id": "create",
|
|
14634
|
+
"app_namespace_ts": "projects",
|
|
14635
|
+
"skill_method_ts": "create",
|
|
14636
|
+
"app_namespace_py": "projects",
|
|
14637
|
+
"skill_method_py": "create",
|
|
14638
|
+
"description_key": "projects.skills.create.description",
|
|
14639
|
+
"description": "Request creation of a user-visible project. Durable project content is client-side encrypted, so a capable connected client must apply the change.",
|
|
14640
|
+
"schema": {
|
|
14641
|
+
"type": "object",
|
|
14642
|
+
"properties": {
|
|
14643
|
+
"name": {
|
|
14644
|
+
"type": "string",
|
|
14645
|
+
"description": "Short user-facing project name."
|
|
14646
|
+
},
|
|
14647
|
+
"description": {
|
|
14648
|
+
"type": "string",
|
|
14649
|
+
"description": "Optional project description."
|
|
14650
|
+
}
|
|
14651
|
+
},
|
|
14652
|
+
"required": [
|
|
14653
|
+
"name"
|
|
14654
|
+
]
|
|
14655
|
+
}
|
|
14656
|
+
},
|
|
14657
|
+
{
|
|
14658
|
+
"app_id": "projects",
|
|
14659
|
+
"skill_id": "search",
|
|
14660
|
+
"app_namespace_ts": "projects",
|
|
14661
|
+
"skill_method_ts": "search",
|
|
14662
|
+
"app_namespace_py": "projects",
|
|
14663
|
+
"skill_method_py": "search",
|
|
14664
|
+
"description_key": "projects.skills.search.description",
|
|
14665
|
+
"description": "Search the user's encrypted projects through a connected capable client. Do not use server-visible metadata as a private project-content fallback.",
|
|
14666
|
+
"schema": {
|
|
14667
|
+
"type": "object",
|
|
14668
|
+
"properties": {
|
|
14669
|
+
"query": {
|
|
14670
|
+
"type": "string",
|
|
14671
|
+
"description": "Private project text to search for on a connected client."
|
|
14672
|
+
}
|
|
14673
|
+
},
|
|
14674
|
+
"required": [
|
|
14675
|
+
"query"
|
|
14676
|
+
]
|
|
14677
|
+
}
|
|
14678
|
+
},
|
|
14532
14679
|
{
|
|
14533
14680
|
"app_id": "reminder",
|
|
14534
14681
|
"skill_id": "set-reminder",
|
|
@@ -15212,7 +15359,7 @@ Note: The 'id' field is auto-generated if not provided - you don't need to inclu
|
|
|
15212
15359
|
"app_namespace_py": "travel",
|
|
15213
15360
|
"skill_method_py": "search_connections",
|
|
15214
15361
|
"description_key": "app_skills.travel.search_connections.description",
|
|
15215
|
-
"description": 'Search for flight or train connections for a particular date with details (airlines/operators, times, stops, durations, prices). Use when user asks about flights, train connections, or travel between cities on a specific date. Set transport_methods to ["airplane"] for flights or ["train"] for trains. If the user names a provider, set providers to one or more of: google_flights, deutsche_bahn, flix. If no provider is specified, all providers for the selected transport method are
|
|
15362
|
+
"description": 'Search for flight or train connections for a particular date with details (airlines/operators, times, stops, durations, prices). Use when user asks about flights, train connections, or travel between cities on a specific date. Set transport_methods to ["airplane"] for flights or ["train"] for trains. If the user names a provider, set providers to one or more of: google_flights, deutsche_bahn, flix, transitous. If no provider is specified, all providers for the selected transport method are searc',
|
|
15216
15363
|
"schema": {
|
|
15217
15364
|
"type": "object",
|
|
15218
15365
|
"properties": {
|
|
@@ -15266,13 +15413,14 @@ Note: The 'id' field is auto-generated if not provided - you don't need to inclu
|
|
|
15266
15413
|
},
|
|
15267
15414
|
"providers": {
|
|
15268
15415
|
"type": "array",
|
|
15269
|
-
"description": 'Optional provider IDs to search. Use "google_flights" for flights, "deutsche_bahn" for Deutsche Bahn / ICE / Bahn.de / Sparpreis train searches,
|
|
15416
|
+
"description": 'Optional provider IDs to search. Use "google_flights" for flights, "deutsche_bahn" for Deutsche Bahn / ICE / Bahn.de / Sparpreis train searches, "flix" for FlixBus / FlixTrain, and "transitous" only for timetable-only public transport routing. If omitted, priced providers for the selected transport method are used, then filtered by countries if provided.\n',
|
|
15270
15417
|
"items": {
|
|
15271
15418
|
"type": "string",
|
|
15272
15419
|
"enum": [
|
|
15273
15420
|
"google_flights",
|
|
15274
15421
|
"deutsche_bahn",
|
|
15275
|
-
"flix"
|
|
15422
|
+
"flix",
|
|
15423
|
+
"transitous"
|
|
15276
15424
|
]
|
|
15277
15425
|
}
|
|
15278
15426
|
},
|
|
@@ -15331,6 +15479,36 @@ Note: The 'id' field is auto-generated if not provided - you don't need to inclu
|
|
|
15331
15479
|
"type": "string"
|
|
15332
15480
|
}
|
|
15333
15481
|
},
|
|
15482
|
+
"owned_passes": {
|
|
15483
|
+
"type": "array",
|
|
15484
|
+
"description": 'Fare or mobility pass IDs the traveller already owns, such as "deutschland_ticket". Use for pass-aware train pricing when the user says they have a Deutschlandticket.\n',
|
|
15485
|
+
"items": {
|
|
15486
|
+
"type": "string"
|
|
15487
|
+
}
|
|
15488
|
+
},
|
|
15489
|
+
"pass_only": {
|
|
15490
|
+
"type": "boolean",
|
|
15491
|
+
"description": 'If true, prefer routes covered by owned fare passes when supported. Use with owned_passes ["deutschland_ticket"] for Deutschlandticket-only routing.\n',
|
|
15492
|
+
"default": false
|
|
15493
|
+
},
|
|
15494
|
+
"rail_products": {
|
|
15495
|
+
"type": "array",
|
|
15496
|
+
"description": "Stable rail product filters for train searches. Use high_speed for ICE/high-speed, intercity for IC/EC, regional_express for IRE/RE-like services, regional for RB/local, s_bahn for S-Bahn, subway, tram, bus, or ferry.\n",
|
|
15497
|
+
"items": {
|
|
15498
|
+
"type": "string",
|
|
15499
|
+
"enum": [
|
|
15500
|
+
"high_speed",
|
|
15501
|
+
"intercity",
|
|
15502
|
+
"regional_express",
|
|
15503
|
+
"regional",
|
|
15504
|
+
"s_bahn",
|
|
15505
|
+
"subway",
|
|
15506
|
+
"tram",
|
|
15507
|
+
"bus",
|
|
15508
|
+
"ferry"
|
|
15509
|
+
]
|
|
15510
|
+
}
|
|
15511
|
+
},
|
|
15334
15512
|
"min_departure_time": {
|
|
15335
15513
|
"type": "string",
|
|
15336
15514
|
"description": "Earliest acceptable local departure time in HH:MM format."
|
|
@@ -16249,7 +16427,7 @@ var CodeAppSkills = class {
|
|
|
16249
16427
|
return this.runSkill("code", "get_project_overview", input, options);
|
|
16250
16428
|
}
|
|
16251
16429
|
/**
|
|
16252
|
-
* Turn an uploaded screenshot into one standalone index.html code embed with inline CSS and optional inline JavaScript. Use this when the user asks to convert a screenshot, mockup, UI image, or app screen into HTML. Do not use remote image URLs;
|
|
16430
|
+
* Turn an uploaded screenshot into one standalone index.html code embed with inline CSS and optional inline JavaScript. Use this when the user asks to convert a screenshot, mockup, UI image, or app screen into HTML. Do not use remote image URLs; pass source_image with the uploaded image embed_ref in chat, or image_base64 for REST/API calls.
|
|
16253
16431
|
* Description key: app_skills.code.image_to_html.description
|
|
16254
16432
|
* Skill: code/image_to_html
|
|
16255
16433
|
*/
|
|
@@ -16545,6 +16723,50 @@ var PdfAppSkills = class {
|
|
|
16545
16723
|
return this.runSkill("pdf", "view", input, options);
|
|
16546
16724
|
}
|
|
16547
16725
|
};
|
|
16726
|
+
var PlansAppSkills = class {
|
|
16727
|
+
runSkill;
|
|
16728
|
+
constructor(runSkill) {
|
|
16729
|
+
this.runSkill = runSkill;
|
|
16730
|
+
}
|
|
16731
|
+
/**
|
|
16732
|
+
* Request creation of a user-visible plan. Durable plan content is client-side encrypted, so a capable connected client must apply the change.
|
|
16733
|
+
* Description key: plans.skills.create.description
|
|
16734
|
+
* Skill: plans/create
|
|
16735
|
+
*/
|
|
16736
|
+
async create(input, options) {
|
|
16737
|
+
return this.runSkill("plans", "create", input, options);
|
|
16738
|
+
}
|
|
16739
|
+
/**
|
|
16740
|
+
* Search the user's encrypted plans through a connected capable client. Do not use server-visible metadata as a private plan-content fallback.
|
|
16741
|
+
* Description key: plans.skills.search.description
|
|
16742
|
+
* Skill: plans/search
|
|
16743
|
+
*/
|
|
16744
|
+
async search(input, options) {
|
|
16745
|
+
return this.runSkill("plans", "search", input, options);
|
|
16746
|
+
}
|
|
16747
|
+
};
|
|
16748
|
+
var ProjectsAppSkills = class {
|
|
16749
|
+
runSkill;
|
|
16750
|
+
constructor(runSkill) {
|
|
16751
|
+
this.runSkill = runSkill;
|
|
16752
|
+
}
|
|
16753
|
+
/**
|
|
16754
|
+
* Request creation of a user-visible project. Durable project content is client-side encrypted, so a capable connected client must apply the change.
|
|
16755
|
+
* Description key: projects.skills.create.description
|
|
16756
|
+
* Skill: projects/create
|
|
16757
|
+
*/
|
|
16758
|
+
async create(input, options) {
|
|
16759
|
+
return this.runSkill("projects", "create", input, options);
|
|
16760
|
+
}
|
|
16761
|
+
/**
|
|
16762
|
+
* Search the user's encrypted projects through a connected capable client. Do not use server-visible metadata as a private project-content fallback.
|
|
16763
|
+
* Description key: projects.skills.search.description
|
|
16764
|
+
* Skill: projects/search
|
|
16765
|
+
*/
|
|
16766
|
+
async search(input, options) {
|
|
16767
|
+
return this.runSkill("projects", "search", input, options);
|
|
16768
|
+
}
|
|
16769
|
+
};
|
|
16548
16770
|
var ReminderAppSkills = class {
|
|
16549
16771
|
runSkill;
|
|
16550
16772
|
constructor(runSkill) {
|
|
@@ -16647,7 +16869,7 @@ var TravelAppSkills = class {
|
|
|
16647
16869
|
return this.runSkill("travel", "get_flight", input, options);
|
|
16648
16870
|
}
|
|
16649
16871
|
/**
|
|
16650
|
-
* Search for flight or train connections for a particular date with details (airlines/operators, times, stops, durations, prices). Use when user asks about flights, train connections, or travel between cities on a specific date. Set transport_methods to ["airplane"] for flights or ["train"] for trains. If the user names a provider, set providers to one or more of: google_flights, deutsche_bahn, flix. If no provider is specified, all providers for the selected transport method are
|
|
16872
|
+
* Search for flight or train connections for a particular date with details (airlines/operators, times, stops, durations, prices). Use when user asks about flights, train connections, or travel between cities on a specific date. Set transport_methods to ["airplane"] for flights or ["train"] for trains. If the user names a provider, set providers to one or more of: google_flights, deutsche_bahn, flix, transitous. If no provider is specified, all providers for the selected transport method are searc
|
|
16651
16873
|
* Description key: app_skills.travel.search_connections.description
|
|
16652
16874
|
* Skill: travel/search_connections
|
|
16653
16875
|
*/
|
|
@@ -16789,6 +17011,8 @@ var GeneratedAppSkills = class {
|
|
|
16789
17011
|
this.nutrition = new NutritionAppSkills(runSkill);
|
|
16790
17012
|
this.openmates = new OpenmatesAppSkills(runSkill);
|
|
16791
17013
|
this.pdf = new PdfAppSkills(runSkill);
|
|
17014
|
+
this.plans = new PlansAppSkills(runSkill);
|
|
17015
|
+
this.projects = new ProjectsAppSkills(runSkill);
|
|
16792
17016
|
this.reminder = new ReminderAppSkills(runSkill);
|
|
16793
17017
|
this.shopping = new ShoppingAppSkills(runSkill);
|
|
16794
17018
|
this.socialMedia = new SocialMediaAppSkills(runSkill);
|
|
@@ -16819,6 +17043,8 @@ var GeneratedAppSkills = class {
|
|
|
16819
17043
|
nutrition;
|
|
16820
17044
|
openmates;
|
|
16821
17045
|
pdf;
|
|
17046
|
+
plans;
|
|
17047
|
+
projects;
|
|
16822
17048
|
reminder;
|
|
16823
17049
|
shopping;
|
|
16824
17050
|
socialMedia;
|
|
@@ -21185,7 +21411,7 @@ import { stdin as stdin3, stdout as stdout2 } from "process";
|
|
|
21185
21411
|
import { readFileSync as readFileSync11, realpathSync as realpathSync2, writeFileSync as writeFileSync7 } from "fs";
|
|
21186
21412
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
21187
21413
|
import { basename as basename3, dirname as dirname5 } from "path";
|
|
21188
|
-
import { createHash as
|
|
21414
|
+
import { createHash as createHash14, randomBytes as randomBytes8, randomUUID as randomUUID10 } from "crypto";
|
|
21189
21415
|
import { arch as arch2, platform as platform3 } from "os";
|
|
21190
21416
|
import WebSocket2 from "ws";
|
|
21191
21417
|
|
|
@@ -22086,6 +22312,27 @@ function formatPrice(amount, currency) {
|
|
|
22086
22312
|
const cur = str(currency)?.toUpperCase() ?? "";
|
|
22087
22313
|
return cur ? `${cur} ${amount}` : String(amount);
|
|
22088
22314
|
}
|
|
22315
|
+
function fareNumberValue(value) {
|
|
22316
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
22317
|
+
if (typeof value !== "string" || !value.trim()) return null;
|
|
22318
|
+
const parsed = Number(value);
|
|
22319
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
22320
|
+
}
|
|
22321
|
+
function fareRecord(value) {
|
|
22322
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
22323
|
+
}
|
|
22324
|
+
function formatFare(value) {
|
|
22325
|
+
const fare = fareRecord(value.fare);
|
|
22326
|
+
const confidence = str(fare?.confidence);
|
|
22327
|
+
if (confidence === "timetable_only") return "Timetable only";
|
|
22328
|
+
if (fare?.is_pass_only === true) return str(fare.summary) ?? "Covered by pass";
|
|
22329
|
+
const amount = fare?.amount ?? value.total_price ?? value.price;
|
|
22330
|
+
const currency = fare?.currency ?? value.currency;
|
|
22331
|
+
const price = formatPrice(amount, currency);
|
|
22332
|
+
if (!price) return confidence === "unknown" ? "Fare unknown" : "";
|
|
22333
|
+
if (fare?.is_partial === true || value.fare_is_partial === true) return `${price} (partial fare)`;
|
|
22334
|
+
return price;
|
|
22335
|
+
}
|
|
22089
22336
|
function trunc(s, max) {
|
|
22090
22337
|
return s.length > max ? s.slice(0, max) + "\u2026" : s;
|
|
22091
22338
|
}
|
|
@@ -22583,7 +22830,12 @@ async function renderTravelConnectionsPreview(c, ln, _client) {
|
|
|
22583
22830
|
}
|
|
22584
22831
|
if (count !== null) ln(`\x1B[2m${count} connections\x1B[0m`);
|
|
22585
22832
|
if (Array.isArray(results) && results.length > 0) {
|
|
22586
|
-
const prices = results.map((r) =>
|
|
22833
|
+
const prices = results.map((r) => {
|
|
22834
|
+
const fare = fareRecord(r.fare);
|
|
22835
|
+
const confidence = str(fare?.confidence);
|
|
22836
|
+
if (confidence && !["confirmed", "partial"].includes(confidence)) return null;
|
|
22837
|
+
return fareNumberValue(fare?.amount ?? r.total_price);
|
|
22838
|
+
}).filter((p) => p !== null);
|
|
22587
22839
|
if (prices.length > 0) {
|
|
22588
22840
|
const min = Math.min(...prices);
|
|
22589
22841
|
const currency = str(results[0].currency) ?? "EUR";
|
|
@@ -22605,7 +22857,7 @@ async function renderTravelConnectionsFullscreen(c, client) {
|
|
|
22605
22857
|
const dep = str(r.departure)?.slice(11, 16) ?? "";
|
|
22606
22858
|
const arr = str(r.arrival)?.slice(11, 16) ?? "";
|
|
22607
22859
|
const duration = str(r.duration) ?? "";
|
|
22608
|
-
const price =
|
|
22860
|
+
const price = formatFare(r);
|
|
22609
22861
|
const stops = typeof r.stops === "number" ? r.stops === 0 ? "Direct" : `${r.stops} stops` : "";
|
|
22610
22862
|
const carriers = Array.isArray(r.carriers) ? r.carriers.join(", ") : str(r.carrier) ?? "";
|
|
22611
22863
|
if (origin && dest)
|
|
@@ -22990,7 +23242,7 @@ function renderByDirectType(embed, c, ln) {
|
|
|
22990
23242
|
case "travel-connection": {
|
|
22991
23243
|
const origin = str(c.origin) ?? "";
|
|
22992
23244
|
const dest = str(c.destination) ?? "";
|
|
22993
|
-
const price =
|
|
23245
|
+
const price = formatFare(c);
|
|
22994
23246
|
const dep = str(c.departure)?.slice(11, 16) ?? "";
|
|
22995
23247
|
const arr = str(c.arrival)?.slice(11, 16) ?? "";
|
|
22996
23248
|
if (origin && dest) ln(`${origin} \u2192 ${dest}`);
|
|
@@ -62831,14 +63083,126 @@ function buildAssistantFeedbackDecision(rating) {
|
|
|
62831
63083
|
}
|
|
62832
63084
|
|
|
62833
63085
|
// src/benchmark.ts
|
|
62834
|
-
import { randomUUID as randomUUID8 } from "crypto";
|
|
63086
|
+
import { createHash as createHash13, randomUUID as randomUUID8 } from "crypto";
|
|
62835
63087
|
import { existsSync as existsSync8, mkdtempSync as mkdtempSync2, readFileSync as readFileSync7, readdirSync as readdirSync3, writeFileSync as writeFileSync5 } from "fs";
|
|
62836
63088
|
import { tmpdir } from "os";
|
|
62837
63089
|
import { dirname as dirname4, join as join5, resolve as resolve5 } from "path";
|
|
62838
63090
|
import { fileURLToPath } from "url";
|
|
62839
63091
|
var DEFAULT_JUDGE_MODEL = "google/gemini-3-flash-preview";
|
|
63092
|
+
var DEFAULT_PROMPT_BUDGET_MODEL = "google/gemini-3.5-flash";
|
|
62840
63093
|
var DEFAULT_EXTENSIVE_SIZE = 10;
|
|
62841
63094
|
var DEFAULT_PARALLEL = 4;
|
|
63095
|
+
var PROMPT_BUDGET_TARGET_INPUT_TOKENS_PER_TURN = 16e3;
|
|
63096
|
+
var PROMPT_BUDGET_TARGET_OUTPUT_TOKENS_PER_TURN = 800;
|
|
63097
|
+
var PROMPT_BUDGET_JUDGE_INPUT_TOKENS_PER_RESPONSE = 3e3;
|
|
63098
|
+
var PROMPT_BUDGET_JUDGE_OUTPUT_TOKENS_PER_RESPONSE = 300;
|
|
63099
|
+
var PROMPT_BUDGET_CASES = [
|
|
63100
|
+
{
|
|
63101
|
+
id: "single-writing-concise-email",
|
|
63102
|
+
title: "Concise email rewrite",
|
|
63103
|
+
category: "writing",
|
|
63104
|
+
qualityRubric: "Warm, concise, under 120 words, directly asks to move the Tuesday sync to Thursday afternoon.",
|
|
63105
|
+
turns: [
|
|
63106
|
+
"Write a concise but warm email to a colleague asking to move our Tuesday 10:00 project sync to Thursday afternoon. Keep it under 120 words."
|
|
63107
|
+
]
|
|
63108
|
+
},
|
|
63109
|
+
{
|
|
63110
|
+
id: "single-code-palindrome",
|
|
63111
|
+
title: "TypeScript palindrome helper",
|
|
63112
|
+
category: "coding",
|
|
63113
|
+
qualityRubric: "Produces valid TypeScript, ignores spaces/punctuation/case, includes one short usage example, and avoids unrelated prose.",
|
|
63114
|
+
turns: [
|
|
63115
|
+
"Write a TypeScript function isPalindrome(input: string): boolean that ignores spaces, punctuation, and case. Include only the function and one short usage example."
|
|
63116
|
+
]
|
|
63117
|
+
},
|
|
63118
|
+
{
|
|
63119
|
+
id: "single-current-web-ai-news",
|
|
63120
|
+
title: "Current AI release search",
|
|
63121
|
+
category: "web",
|
|
63122
|
+
qualityRubric: "Uses current web/search behavior, cites sources, and summarizes a major AI model release from the last 30 days without fabricating certainty.",
|
|
63123
|
+
turns: [
|
|
63124
|
+
"Search the web for the latest major AI model release from the last 30 days and summarize what changed, with sources."
|
|
63125
|
+
]
|
|
63126
|
+
},
|
|
63127
|
+
{
|
|
63128
|
+
id: "single-travel-filtered-flight",
|
|
63129
|
+
title: "Filtered flight search",
|
|
63130
|
+
category: "travel",
|
|
63131
|
+
qualityRubric: "Attempts appropriate travel/tool routing, honors direct flight/date/budget constraints where possible, and explains tradeoffs or uncertainty.",
|
|
63132
|
+
turns: [
|
|
63133
|
+
"Find direct flights from Berlin to Barcelona on 2026-09-10 under 220 EUR if possible, and explain the best tradeoff."
|
|
63134
|
+
]
|
|
63135
|
+
},
|
|
63136
|
+
{
|
|
63137
|
+
id: "single-events-accessibility",
|
|
63138
|
+
title: "Accessible family events",
|
|
63139
|
+
category: "events",
|
|
63140
|
+
qualityRubric: "Attempts event/tool routing, separates confirmed from unknown accessibility, and avoids claiming accessibility without evidence.",
|
|
63141
|
+
turns: [
|
|
63142
|
+
"Find wheelchair-accessible family events in Berlin this weekend. Separate confirmed accessibility from unknown accessibility."
|
|
63143
|
+
]
|
|
63144
|
+
},
|
|
63145
|
+
{
|
|
63146
|
+
id: "single-openmates-help",
|
|
63147
|
+
title: "OpenMates skill selection",
|
|
63148
|
+
category: "openmates",
|
|
63149
|
+
qualityRubric: "Correctly explains a suitable OpenMates app/skill path for product comparison and suggests a useful next step.",
|
|
63150
|
+
turns: [
|
|
63151
|
+
"Explain which OpenMates app or skill I should use to compare products before buying a new laptop, and what you would do next."
|
|
63152
|
+
]
|
|
63153
|
+
},
|
|
63154
|
+
{
|
|
63155
|
+
id: "single-safety-privacy-review",
|
|
63156
|
+
title: "Privacy review",
|
|
63157
|
+
category: "safety",
|
|
63158
|
+
qualityRubric: "Identifies privacy risks in benchmark logging and gives practical safer defaults without overclaiming compliance.",
|
|
63159
|
+
turns: [
|
|
63160
|
+
"Review this benchmark design for privacy risks: it logs prompts, outputs, model ids, usage costs, and debug metadata to a shared file. List risks and safer defaults."
|
|
63161
|
+
]
|
|
63162
|
+
},
|
|
63163
|
+
{
|
|
63164
|
+
id: "multiturn-model-evaluation-plan",
|
|
63165
|
+
title: "Model evaluation follow-up plan",
|
|
63166
|
+
category: "planning",
|
|
63167
|
+
qualityRubric: "Maintains continuity across all six turns, refines prior outputs instead of restarting, and produces concrete evaluation criteria.",
|
|
63168
|
+
turns: [
|
|
63169
|
+
"Create a three-step plan for evaluating whether a new AI model is ready for production use.",
|
|
63170
|
+
"Make step 2 more concrete with two measurable checks.",
|
|
63171
|
+
"Add a risk register with severity and mitigation for the top three risks.",
|
|
63172
|
+
"Turn the plan into a one-week execution schedule.",
|
|
63173
|
+
"Identify which tasks could be automated with OpenMates skills.",
|
|
63174
|
+
"Summarize the final plan in one sentence."
|
|
63175
|
+
]
|
|
63176
|
+
},
|
|
63177
|
+
{
|
|
63178
|
+
id: "multiturn-startup-launch",
|
|
63179
|
+
title: "Freelancer AI assistant launch",
|
|
63180
|
+
category: "business",
|
|
63181
|
+
qualityRubric: "Builds on previous launch context, keeps the privacy-first freelancer positioning, and produces specific GTM artifacts.",
|
|
63182
|
+
turns: [
|
|
63183
|
+
"Help me plan the launch of a privacy-first personal AI assistant for freelancers.",
|
|
63184
|
+
"Define the ideal first customer profile and their top pains.",
|
|
63185
|
+
"Draft a landing page hero section for that customer.",
|
|
63186
|
+
"Create a simple pricing experiment with three tiers.",
|
|
63187
|
+
"Write five outbound messages for early customer interviews.",
|
|
63188
|
+
"Summarize the riskiest assumption and how to test it this week."
|
|
63189
|
+
]
|
|
63190
|
+
},
|
|
63191
|
+
{
|
|
63192
|
+
id: "multiturn-debugging-workflow",
|
|
63193
|
+
title: "SvelteKit stale chat debugging",
|
|
63194
|
+
category: "debugging",
|
|
63195
|
+
qualityRubric: "Maintains the stale-chat bug context, distinguishes frontend cache from backend sync, and uses data-testid selector guidance.",
|
|
63196
|
+
turns: [
|
|
63197
|
+
"A SvelteKit page sometimes shows stale chat messages after refresh. Give me a debugging plan.",
|
|
63198
|
+
"Add likely root causes involving IndexedDB and WebSocket sync.",
|
|
63199
|
+
"Suggest instrumentation I should add before fixing code.",
|
|
63200
|
+
"Write a minimal regression test outline using data-testid selectors.",
|
|
63201
|
+
"Explain how to tell if the bug is backend sync or frontend cache.",
|
|
63202
|
+
"Condense this into a checklist I can hand to another engineer."
|
|
63203
|
+
]
|
|
63204
|
+
}
|
|
63205
|
+
];
|
|
62842
63206
|
var FIXTURE_IMAGE_SVG = `<?xml version="1.0" encoding="UTF-8"?>
|
|
62843
63207
|
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="800" viewBox="0 0 1200 800">
|
|
62844
63208
|
<rect width="1200" height="800" fill="#d8ecff"/>
|
|
@@ -63109,6 +63473,10 @@ async function handleBenchmark(client, subcommand, rest, flags) {
|
|
|
63109
63473
|
printBenchmarkHelp();
|
|
63110
63474
|
return;
|
|
63111
63475
|
}
|
|
63476
|
+
if (subcommand === "prompt-budget") {
|
|
63477
|
+
await handlePromptBudgetBenchmark(client, rest, flags);
|
|
63478
|
+
return;
|
|
63479
|
+
}
|
|
63112
63480
|
if (subcommand !== "model") {
|
|
63113
63481
|
throw new Error(`Unknown benchmark command '${subcommand}'. Run 'openmates benchmark --help'.`);
|
|
63114
63482
|
}
|
|
@@ -63183,6 +63551,8 @@ async function handleBenchmark(client, subcommand, rest, flags) {
|
|
|
63183
63551
|
function printBenchmarkHelp() {
|
|
63184
63552
|
console.log(`Benchmark commands:
|
|
63185
63553
|
openmates benchmark model <provider/model> [provider/model...] --confirm-spend-credits [--compare] [--suite quick|extensive|all] [--json]
|
|
63554
|
+
openmates benchmark prompt-budget --phase baseline|postchange --confirm-spend-credits [--model provider/model] [--resume-from <path>] [--json]
|
|
63555
|
+
openmates benchmark prompt-budget compare --baseline <path> --postchange <path> [--json]
|
|
63186
63556
|
|
|
63187
63557
|
Runs real incognito chat requests through the OpenMates product path. Live runs
|
|
63188
63558
|
spend the logged-in user's credits and usage entries are grouped as benchmark spend.
|
|
@@ -63202,6 +63572,74 @@ Options:
|
|
|
63202
63572
|
--output <path> Save JSON result to a file
|
|
63203
63573
|
--json Print JSON result`);
|
|
63204
63574
|
}
|
|
63575
|
+
async function handlePromptBudgetBenchmark(client, rest, flags) {
|
|
63576
|
+
if (rest[0] === "compare") {
|
|
63577
|
+
writePromptBudgetComparison(comparePromptBudgetArtifacts(flags), flags, stringFlag(flags.output));
|
|
63578
|
+
return;
|
|
63579
|
+
}
|
|
63580
|
+
if (rest.length > 0) {
|
|
63581
|
+
throw new Error(`Unknown prompt-budget benchmark argument '${rest[0]}'. Run 'openmates benchmark --help'.`);
|
|
63582
|
+
}
|
|
63583
|
+
const phase = parsePromptBudgetPhase(flags.phase);
|
|
63584
|
+
const targetModel = stringFlag(flags.model) ?? DEFAULT_PROMPT_BUDGET_MODEL;
|
|
63585
|
+
const judgeModel = stringFlag(flags["judge-model"]) ?? DEFAULT_JUDGE_MODEL;
|
|
63586
|
+
const output = stringFlag(flags.output);
|
|
63587
|
+
const resumeFrom = stringFlag(flags["resume-from"]);
|
|
63588
|
+
const dryRun = flags["dry-run"] === true;
|
|
63589
|
+
const responseTimeoutMs = parseResponseTimeoutMs(flags["response-timeout-seconds"]);
|
|
63590
|
+
if (!dryRun && flags["confirm-spend-credits"] !== true) {
|
|
63591
|
+
throw new Error(
|
|
63592
|
+
"Prompt-budget benchmark runs spend real credits from the logged-in account. Rerun with --confirm-spend-credits, or use --dry-run to preview the plan."
|
|
63593
|
+
);
|
|
63594
|
+
}
|
|
63595
|
+
if (dryRun && resumeFrom) {
|
|
63596
|
+
throw new Error("--resume-from is only supported for live prompt-budget runs");
|
|
63597
|
+
}
|
|
63598
|
+
const pricing = loadPricingForModels([targetModel, judgeModel]);
|
|
63599
|
+
const estimate = estimatePromptBudgetCredits(PROMPT_BUDGET_CASES, targetModel, judgeModel, pricing);
|
|
63600
|
+
const resumeArtifact = resumeFrom ? readPromptBudgetResult(resumeFrom) : null;
|
|
63601
|
+
if (resumeArtifact) {
|
|
63602
|
+
validatePromptBudgetResumeArtifact(resumeArtifact, { phase, targetModel, judgeModel });
|
|
63603
|
+
}
|
|
63604
|
+
const runId = stringFlag(flags["run-id"]) ?? resumeArtifact?.runId ?? randomUUID8();
|
|
63605
|
+
const result = makePromptBudgetBaseResult({
|
|
63606
|
+
phase,
|
|
63607
|
+
runId,
|
|
63608
|
+
targetModel,
|
|
63609
|
+
judgeModel,
|
|
63610
|
+
dryRun,
|
|
63611
|
+
estimate
|
|
63612
|
+
});
|
|
63613
|
+
if (resumeArtifact) {
|
|
63614
|
+
result.cases = resumeArtifact.cases.filter(isCompletePromptBudgetCase);
|
|
63615
|
+
recomputePromptBudgetResult(result);
|
|
63616
|
+
}
|
|
63617
|
+
if (dryRun) {
|
|
63618
|
+
writePromptBudgetResult(result, flags, output);
|
|
63619
|
+
return;
|
|
63620
|
+
}
|
|
63621
|
+
if (!client.hasSession()) {
|
|
63622
|
+
throw new Error("Prompt-budget benchmark runs require login. Run 'openmates login' first.");
|
|
63623
|
+
}
|
|
63624
|
+
const completedCaseIds = new Set(result.cases.map((benchmarkCase) => benchmarkCase.id));
|
|
63625
|
+
for (const benchmarkCase of PROMPT_BUDGET_CASES) {
|
|
63626
|
+
if (completedCaseIds.has(benchmarkCase.id)) continue;
|
|
63627
|
+
const caseResult = await runPromptBudgetCase({
|
|
63628
|
+
client,
|
|
63629
|
+
benchmarkCase,
|
|
63630
|
+
phase,
|
|
63631
|
+
runId,
|
|
63632
|
+
targetModel,
|
|
63633
|
+
judgeModel,
|
|
63634
|
+
responseTimeoutMs
|
|
63635
|
+
});
|
|
63636
|
+
result.cases.push(caseResult);
|
|
63637
|
+
recomputePromptBudgetResult(result);
|
|
63638
|
+
if (caseResult.error) break;
|
|
63639
|
+
}
|
|
63640
|
+
recomputePromptBudgetResult(result);
|
|
63641
|
+
writePromptBudgetResult(result, flags, output);
|
|
63642
|
+
}
|
|
63205
63643
|
function parseSuites(value) {
|
|
63206
63644
|
if (value === void 0 || value === false) return ["quick"];
|
|
63207
63645
|
if (value === true) throw new Error("--suite requires a value");
|
|
@@ -63248,6 +63686,24 @@ function parseCaseIds(value) {
|
|
|
63248
63686
|
if (caseIds.length === 0) throw new Error("--case requires at least one case id");
|
|
63249
63687
|
return [...new Set(caseIds)];
|
|
63250
63688
|
}
|
|
63689
|
+
function parsePromptBudgetPhase(value) {
|
|
63690
|
+
if (value === void 0 || value === false) return "baseline";
|
|
63691
|
+
if (value === true) throw new Error("--phase requires baseline or postchange");
|
|
63692
|
+
if (value === "baseline" || value === "postchange") return value;
|
|
63693
|
+
throw new Error("--phase must be baseline or postchange");
|
|
63694
|
+
}
|
|
63695
|
+
function parseResponseTimeoutMs(value) {
|
|
63696
|
+
if (value === void 0 || value === false) return void 0;
|
|
63697
|
+
if (value === true) throw new Error("--response-timeout-seconds requires a value");
|
|
63698
|
+
const parsed = Number.parseInt(value, 10);
|
|
63699
|
+
if (!Number.isInteger(parsed) || parsed < 30 || parsed > 1200) {
|
|
63700
|
+
throw new Error("--response-timeout-seconds must be an integer from 30 to 1200");
|
|
63701
|
+
}
|
|
63702
|
+
return parsed * 1e3;
|
|
63703
|
+
}
|
|
63704
|
+
function stringFlag(value) {
|
|
63705
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
63706
|
+
}
|
|
63251
63707
|
function filterCases(cases, caseIds) {
|
|
63252
63708
|
if (caseIds.length === 0) return cases;
|
|
63253
63709
|
const availableIds = new Set(cases.map((benchmarkCase) => benchmarkCase.id));
|
|
@@ -63304,6 +63760,95 @@ function dedupeCases(cases) {
|
|
|
63304
63760
|
}
|
|
63305
63761
|
return result;
|
|
63306
63762
|
}
|
|
63763
|
+
async function runPromptBudgetCase(params) {
|
|
63764
|
+
const { client, benchmarkCase, phase, runId, targetModel, judgeModel, responseTimeoutMs } = params;
|
|
63765
|
+
const startedAt = Date.now();
|
|
63766
|
+
const history = [];
|
|
63767
|
+
const turns = [];
|
|
63768
|
+
let chatId;
|
|
63769
|
+
try {
|
|
63770
|
+
for (const [index, prompt] of benchmarkCase.turns.entries()) {
|
|
63771
|
+
const turnNumber = index + 1;
|
|
63772
|
+
const rawPrompt = `${modelMention(targetModel)} ${prompt}`;
|
|
63773
|
+
const turnStartedAt = Date.now();
|
|
63774
|
+
const response = await client.sendMessage({
|
|
63775
|
+
message: rawPrompt,
|
|
63776
|
+
chatId,
|
|
63777
|
+
incognito: true,
|
|
63778
|
+
autoApproveSubChats: true,
|
|
63779
|
+
benchmarkMetadata: benchmarkMetadata({
|
|
63780
|
+
runId,
|
|
63781
|
+
suite: `prompt_budget_${phase}`,
|
|
63782
|
+
caseId: `${benchmarkCase.id}:turn-${turnNumber}`,
|
|
63783
|
+
targetModel,
|
|
63784
|
+
judgeModel
|
|
63785
|
+
}),
|
|
63786
|
+
messageHistory: history,
|
|
63787
|
+
precollectResponse: true,
|
|
63788
|
+
responseTimeoutMs
|
|
63789
|
+
});
|
|
63790
|
+
chatId = response.chatId;
|
|
63791
|
+
const turnResult = {
|
|
63792
|
+
turn: turnNumber,
|
|
63793
|
+
prompt,
|
|
63794
|
+
promptHash: stableHash(prompt),
|
|
63795
|
+
assistant: response.assistant,
|
|
63796
|
+
durationMs: Date.now() - turnStartedAt,
|
|
63797
|
+
chatId: response.chatId,
|
|
63798
|
+
messageId: response.messageId,
|
|
63799
|
+
modelName: response.modelName,
|
|
63800
|
+
mateName: response.mateName,
|
|
63801
|
+
category: response.category,
|
|
63802
|
+
tokenUsage: response.tokenUsage,
|
|
63803
|
+
promptBudget: response.promptBudget
|
|
63804
|
+
};
|
|
63805
|
+
turnResult.judge = await judgePromptBudgetTurn({
|
|
63806
|
+
client,
|
|
63807
|
+
benchmarkCase,
|
|
63808
|
+
turn: turnResult,
|
|
63809
|
+
turnNumber,
|
|
63810
|
+
targetModel,
|
|
63811
|
+
judgeModel,
|
|
63812
|
+
runId,
|
|
63813
|
+
phase
|
|
63814
|
+
});
|
|
63815
|
+
turns.push(turnResult);
|
|
63816
|
+
appendHistory(history, "user", rawPrompt);
|
|
63817
|
+
appendHistory(history, "assistant", response.assistant);
|
|
63818
|
+
}
|
|
63819
|
+
const continuityJudge = benchmarkCase.turns.length > 1 ? await judgePromptBudgetContinuity({
|
|
63820
|
+
client,
|
|
63821
|
+
benchmarkCase,
|
|
63822
|
+
turns,
|
|
63823
|
+
targetModel,
|
|
63824
|
+
judgeModel,
|
|
63825
|
+
runId,
|
|
63826
|
+
phase
|
|
63827
|
+
}) : void 0;
|
|
63828
|
+
const passed = turns.every((turn) => (turn.judge?.score ?? 0) >= 4) && (!continuityJudge || (continuityJudge.score ?? 0) >= 4);
|
|
63829
|
+
return {
|
|
63830
|
+
id: benchmarkCase.id,
|
|
63831
|
+
title: benchmarkCase.title,
|
|
63832
|
+
category: benchmarkCase.category,
|
|
63833
|
+
promptHashes: benchmarkCase.turns.map(stableHash),
|
|
63834
|
+
turns,
|
|
63835
|
+
continuityJudge,
|
|
63836
|
+
passed,
|
|
63837
|
+
durationMs: Date.now() - startedAt
|
|
63838
|
+
};
|
|
63839
|
+
} catch (error) {
|
|
63840
|
+
return {
|
|
63841
|
+
id: benchmarkCase.id,
|
|
63842
|
+
title: benchmarkCase.title,
|
|
63843
|
+
category: benchmarkCase.category,
|
|
63844
|
+
promptHashes: benchmarkCase.turns.map(stableHash),
|
|
63845
|
+
turns,
|
|
63846
|
+
passed: false,
|
|
63847
|
+
durationMs: Date.now() - startedAt,
|
|
63848
|
+
error: error instanceof Error ? error.message : String(error)
|
|
63849
|
+
};
|
|
63850
|
+
}
|
|
63851
|
+
}
|
|
63307
63852
|
async function runCaseJob(params) {
|
|
63308
63853
|
const { client, job, judgeModel, runId, imagePath } = params;
|
|
63309
63854
|
const { model, benchmarkCase } = job;
|
|
@@ -63496,6 +64041,82 @@ async function judgeCase(params) {
|
|
|
63496
64041
|
durationMs: Date.now() - startedAt
|
|
63497
64042
|
};
|
|
63498
64043
|
}
|
|
64044
|
+
async function judgePromptBudgetTurn(params) {
|
|
64045
|
+
const startedAt = Date.now();
|
|
64046
|
+
const judgeResponse = await params.client.sendMessage({
|
|
64047
|
+
message: `${modelMention(params.judgeModel)} ${promptBudgetTurnJudgePrompt(params)}`,
|
|
64048
|
+
incognito: true,
|
|
64049
|
+
autoApproveSubChats: true,
|
|
64050
|
+
benchmarkMetadata: benchmarkMetadata({
|
|
64051
|
+
runId: params.runId,
|
|
64052
|
+
suite: `prompt_budget_${params.phase}`,
|
|
64053
|
+
caseId: `${params.benchmarkCase.id}:turn-${params.turnNumber}:judge`,
|
|
64054
|
+
targetModel: params.targetModel,
|
|
64055
|
+
judgeModel: params.judgeModel
|
|
64056
|
+
}),
|
|
64057
|
+
precollectResponse: true
|
|
64058
|
+
});
|
|
64059
|
+
const judgment = parseJudgment(judgeResponse.assistant);
|
|
64060
|
+
return {
|
|
64061
|
+
model: params.judgeModel,
|
|
64062
|
+
score: judgment.score,
|
|
64063
|
+
reason: judgment.reason,
|
|
64064
|
+
raw: judgeResponse.assistant,
|
|
64065
|
+
durationMs: Date.now() - startedAt
|
|
64066
|
+
};
|
|
64067
|
+
}
|
|
64068
|
+
async function judgePromptBudgetContinuity(params) {
|
|
64069
|
+
const startedAt = Date.now();
|
|
64070
|
+
const judgeResponse = await params.client.sendMessage({
|
|
64071
|
+
message: `${modelMention(params.judgeModel)} ${promptBudgetContinuityJudgePrompt(params)}`,
|
|
64072
|
+
incognito: true,
|
|
64073
|
+
autoApproveSubChats: true,
|
|
64074
|
+
benchmarkMetadata: benchmarkMetadata({
|
|
64075
|
+
runId: params.runId,
|
|
64076
|
+
suite: `prompt_budget_${params.phase}`,
|
|
64077
|
+
caseId: `${params.benchmarkCase.id}:continuity-judge`,
|
|
64078
|
+
targetModel: params.targetModel,
|
|
64079
|
+
judgeModel: params.judgeModel
|
|
64080
|
+
}),
|
|
64081
|
+
precollectResponse: true
|
|
64082
|
+
});
|
|
64083
|
+
const judgment = parseJudgment(judgeResponse.assistant);
|
|
64084
|
+
return {
|
|
64085
|
+
model: params.judgeModel,
|
|
64086
|
+
score: judgment.score,
|
|
64087
|
+
reason: judgment.reason,
|
|
64088
|
+
raw: judgeResponse.assistant,
|
|
64089
|
+
durationMs: Date.now() - startedAt
|
|
64090
|
+
};
|
|
64091
|
+
}
|
|
64092
|
+
function promptBudgetTurnJudgePrompt(params) {
|
|
64093
|
+
return [
|
|
64094
|
+
"You are judging a real OpenMates prompt-budget benchmark response.",
|
|
64095
|
+
"Return exactly two plain-text lines, with no markdown, no code block, and no tool use.",
|
|
64096
|
+
"Line 1 format: BENCHMARK_SCORE=<integer from 1 to 5>",
|
|
64097
|
+
"Line 2 format: BENCHMARK_REASON=<one short sentence>",
|
|
64098
|
+
"Score for helpfulness, instruction following, factuality, safety, routing/tool appropriateness, concision, and whether the answer preserves needed context.",
|
|
64099
|
+
`Target model: ${params.targetModel}`,
|
|
64100
|
+
`Case: ${params.benchmarkCase.id}`,
|
|
64101
|
+
`Turn: ${params.turnNumber} of ${params.benchmarkCase.turns.length}`,
|
|
64102
|
+
`Rubric: ${params.benchmarkCase.qualityRubric}`,
|
|
64103
|
+
`User prompt: ${JSON.stringify(params.turn.prompt)}`,
|
|
64104
|
+
`Assistant response: ${JSON.stringify(params.turn.assistant)}`
|
|
64105
|
+
].join("\n");
|
|
64106
|
+
}
|
|
64107
|
+
function promptBudgetContinuityJudgePrompt(params) {
|
|
64108
|
+
return [
|
|
64109
|
+
"You are judging multi-turn continuity for a real OpenMates prompt-budget benchmark conversation.",
|
|
64110
|
+
"Return exactly two plain-text lines, with no markdown, no code block, and no tool use.",
|
|
64111
|
+
"Line 1 format: BENCHMARK_SCORE=<integer from 1 to 5>",
|
|
64112
|
+
"Line 2 format: BENCHMARK_REASON=<one short sentence>",
|
|
64113
|
+
"Score whether the assistant consistently builds on earlier turns, avoids restarting, and satisfies the final requested artifact.",
|
|
64114
|
+
`Target model: ${params.targetModel}`,
|
|
64115
|
+
`Case: ${params.benchmarkCase.id}`,
|
|
64116
|
+
`Rubric: ${params.benchmarkCase.qualityRubric}`,
|
|
64117
|
+
`Turns: ${JSON.stringify(params.turns.map((turn) => ({ prompt: turn.prompt, assistant: turn.assistant })))}`
|
|
64118
|
+
].join("\n");
|
|
64119
|
+
}
|
|
63499
64120
|
async function runPool(items, parallel, worker) {
|
|
63500
64121
|
let index = 0;
|
|
63501
64122
|
const workers = Array.from({ length: Math.min(parallel, items.length) }, async () => {
|
|
@@ -63733,6 +64354,27 @@ function estimateCredits(cases, targetModels, judgeModel, pricing) {
|
|
|
63733
64354
|
assumptions: { targetInputTokens, targetOutputTokens, judgeInputTokens, judgeOutputTokens }
|
|
63734
64355
|
};
|
|
63735
64356
|
}
|
|
64357
|
+
function estimatePromptBudgetCredits(cases, targetModel, judgeModel, pricing) {
|
|
64358
|
+
const targetPricing = pricing.get(targetModel);
|
|
64359
|
+
const judgePricing = pricing.get(judgeModel);
|
|
64360
|
+
if (!targetPricing || !judgePricing) {
|
|
64361
|
+
throw new Error("Prompt-budget benchmark pricing preflight failed after model pricing lookup.");
|
|
64362
|
+
}
|
|
64363
|
+
const targetTurnCount = cases.reduce((sum, benchmarkCase) => sum + benchmarkCase.turns.length, 0);
|
|
64364
|
+
const judgeTurnCount = targetTurnCount + cases.filter((benchmarkCase) => benchmarkCase.turns.length > 1).length;
|
|
64365
|
+
const targetInputTokens = targetTurnCount * PROMPT_BUDGET_TARGET_INPUT_TOKENS_PER_TURN;
|
|
64366
|
+
const targetOutputTokens = targetTurnCount * PROMPT_BUDGET_TARGET_OUTPUT_TOKENS_PER_TURN;
|
|
64367
|
+
const judgeInputTokens = judgeTurnCount * PROMPT_BUDGET_JUDGE_INPUT_TOKENS_PER_RESPONSE;
|
|
64368
|
+
const judgeOutputTokens = judgeTurnCount * PROMPT_BUDGET_JUDGE_OUTPUT_TOKENS_PER_RESPONSE;
|
|
64369
|
+
const targetCredits = creditsFor(targetPricing, targetInputTokens, targetOutputTokens);
|
|
64370
|
+
const judgeCredits = creditsFor(judgePricing, judgeInputTokens, judgeOutputTokens);
|
|
64371
|
+
return {
|
|
64372
|
+
targetCredits,
|
|
64373
|
+
judgeCredits,
|
|
64374
|
+
totalCredits: targetCredits + judgeCredits,
|
|
64375
|
+
assumptions: { targetInputTokens, targetOutputTokens, judgeInputTokens, judgeOutputTokens }
|
|
64376
|
+
};
|
|
64377
|
+
}
|
|
63736
64378
|
function creditsFor(pricing, inputTokens, outputTokens) {
|
|
63737
64379
|
return Math.ceil(inputTokens / pricing.inputTokensPerCredit) + Math.ceil(outputTokens / pricing.outputTokensPerCredit);
|
|
63738
64380
|
}
|
|
@@ -63770,6 +64412,21 @@ function makeBaseResult(params) {
|
|
|
63770
64412
|
}
|
|
63771
64413
|
};
|
|
63772
64414
|
}
|
|
64415
|
+
function makePromptBudgetBaseResult(params) {
|
|
64416
|
+
return {
|
|
64417
|
+
command: "benchmark prompt-budget",
|
|
64418
|
+
status: params.dryRun ? "planned" : "completed",
|
|
64419
|
+
phase: params.phase,
|
|
64420
|
+
runId: params.runId,
|
|
64421
|
+
corpusHash: hashPromptBudgetCorpus(PROMPT_BUDGET_CASES),
|
|
64422
|
+
targetModel: params.targetModel,
|
|
64423
|
+
judgeModel: params.judgeModel,
|
|
64424
|
+
spendsCredits: !params.dryRun,
|
|
64425
|
+
estimatedCredits: params.estimate,
|
|
64426
|
+
cases: [],
|
|
64427
|
+
summary: promptBudgetSummary([])
|
|
64428
|
+
};
|
|
64429
|
+
}
|
|
63773
64430
|
function recomputeResult(result, totalJobs, interrupted) {
|
|
63774
64431
|
const completed = result.cases.length;
|
|
63775
64432
|
const passed = result.cases.filter((caseResult) => caseResult.passed).length;
|
|
@@ -63812,6 +64469,56 @@ function buildComparison(summaries) {
|
|
|
63812
64469
|
function round2(value) {
|
|
63813
64470
|
return Math.round(value * 100) / 100;
|
|
63814
64471
|
}
|
|
64472
|
+
function recomputePromptBudgetResult(result) {
|
|
64473
|
+
result.summary = promptBudgetSummary(result.cases);
|
|
64474
|
+
result.status = result.summary.completedTurns < result.summary.totalTurns || result.cases.some((benchmarkCase) => Boolean(benchmarkCase.error)) ? "partial" : "completed";
|
|
64475
|
+
}
|
|
64476
|
+
function promptBudgetSummary(cases) {
|
|
64477
|
+
const completedTurns = cases.reduce((sum, benchmarkCase) => sum + benchmarkCase.turns.length, 0);
|
|
64478
|
+
const durations = cases.flatMap((benchmarkCase) => benchmarkCase.turns.map((turn) => turn.durationMs)).filter((value) => value > 0);
|
|
64479
|
+
const judgeScores = cases.flatMap((benchmarkCase) => benchmarkCase.turns.map((turn) => turn.judge?.score)).filter((score) => typeof score === "number" && Number.isFinite(score));
|
|
64480
|
+
const continuityScores = cases.map((benchmarkCase) => benchmarkCase.continuityJudge?.score).filter((score) => typeof score === "number" && Number.isFinite(score));
|
|
64481
|
+
return {
|
|
64482
|
+
totalCases: PROMPT_BUDGET_CASES.length,
|
|
64483
|
+
completedCases: cases.length,
|
|
64484
|
+
totalTurns: PROMPT_BUDGET_CASES.reduce((sum, benchmarkCase) => sum + benchmarkCase.turns.length, 0),
|
|
64485
|
+
completedTurns,
|
|
64486
|
+
passedCases: cases.filter((benchmarkCase) => benchmarkCase.passed).length,
|
|
64487
|
+
failedCases: cases.filter((benchmarkCase) => !benchmarkCase.passed).length,
|
|
64488
|
+
averageTurnDurationMs: durations.length > 0 ? Math.round(durations.reduce((sum, value) => sum + value, 0) / durations.length) : null,
|
|
64489
|
+
averageJudgeScore: judgeScores.length > 0 ? round2(judgeScores.reduce((sum, score) => sum + score, 0) / judgeScores.length) : null,
|
|
64490
|
+
averageContinuityScore: continuityScores.length > 0 ? round2(continuityScores.reduce((sum, score) => sum + score, 0) / continuityScores.length) : null
|
|
64491
|
+
};
|
|
64492
|
+
}
|
|
64493
|
+
function validatePromptBudgetResumeArtifact(artifact, expected) {
|
|
64494
|
+
if (artifact.command !== "benchmark prompt-budget") {
|
|
64495
|
+
throw new Error("--resume-from must point to a prompt-budget benchmark artifact");
|
|
64496
|
+
}
|
|
64497
|
+
if (artifact.corpusHash !== hashPromptBudgetCorpus(PROMPT_BUDGET_CASES)) {
|
|
64498
|
+
throw new Error("--resume-from artifact uses a different prompt-budget corpus");
|
|
64499
|
+
}
|
|
64500
|
+
if (artifact.phase !== expected.phase) {
|
|
64501
|
+
throw new Error(`--resume-from artifact phase is ${artifact.phase}, expected ${expected.phase}`);
|
|
64502
|
+
}
|
|
64503
|
+
if (artifact.targetModel !== expected.targetModel || artifact.judgeModel !== expected.judgeModel) {
|
|
64504
|
+
throw new Error("--resume-from artifact target or judge model does not match this run");
|
|
64505
|
+
}
|
|
64506
|
+
}
|
|
64507
|
+
function isCompletePromptBudgetCase(caseResult) {
|
|
64508
|
+
if (caseResult.error) return false;
|
|
64509
|
+
const definition = PROMPT_BUDGET_CASES.find((benchmarkCase) => benchmarkCase.id === caseResult.id);
|
|
64510
|
+
if (!definition) return false;
|
|
64511
|
+
if (caseResult.promptHashes.join("|") !== definition.turns.map(stableHash).join("|")) return false;
|
|
64512
|
+
if (caseResult.turns.length !== definition.turns.length) return false;
|
|
64513
|
+
if (caseResult.turns.some((turn) => !turn.judge)) return false;
|
|
64514
|
+
return definition.turns.length === 1 || Boolean(caseResult.continuityJudge);
|
|
64515
|
+
}
|
|
64516
|
+
function hashPromptBudgetCorpus(cases) {
|
|
64517
|
+
return stableHash(cases.map((benchmarkCase) => ({ id: benchmarkCase.id, turns: benchmarkCase.turns })));
|
|
64518
|
+
}
|
|
64519
|
+
function stableHash(value) {
|
|
64520
|
+
return createHash13("sha256").update(JSON.stringify(value)).digest("hex");
|
|
64521
|
+
}
|
|
63815
64522
|
function defaultImageFixturePath() {
|
|
63816
64523
|
const fixtureDir = join5(dirname4(fileURLToPath(import.meta.url)), "..", "fixtures");
|
|
63817
64524
|
const fixturePath = join5(fixtureDir, "brandenburger-tor.png");
|
|
@@ -63845,6 +64552,143 @@ function writeBenchmarkResult(result, flags, output) {
|
|
|
63845
64552
|
}
|
|
63846
64553
|
}
|
|
63847
64554
|
}
|
|
64555
|
+
function writePromptBudgetResult(result, flags, output) {
|
|
64556
|
+
const json = `${JSON.stringify(result, null, 2)}
|
|
64557
|
+
`;
|
|
64558
|
+
if (output) writeFileSync5(output, json, "utf-8");
|
|
64559
|
+
if (flags.json === true || output) {
|
|
64560
|
+
process.stdout.write(json);
|
|
64561
|
+
return;
|
|
64562
|
+
}
|
|
64563
|
+
console.log(`Prompt-budget benchmark ${result.status}: ${result.phase}`);
|
|
64564
|
+
console.log(`Run ID: ${result.runId}`);
|
|
64565
|
+
console.log(`Corpus: ${result.corpusHash}`);
|
|
64566
|
+
console.log(`Target model: ${result.targetModel}`);
|
|
64567
|
+
console.log(`Judge: ${result.judgeModel}`);
|
|
64568
|
+
console.log(`Estimated credits: ${result.estimatedCredits.totalCredits}`);
|
|
64569
|
+
console.log(`Spend credits: ${result.spendsCredits ? "yes" : "no"}`);
|
|
64570
|
+
if (result.status !== "planned") {
|
|
64571
|
+
console.log(`Passed cases: ${result.summary.passedCases}/${result.summary.completedCases}`);
|
|
64572
|
+
console.log(`Average turn duration: ${result.summary.averageTurnDurationMs ?? "n/a"}ms`);
|
|
64573
|
+
console.log(`Average judge score: ${result.summary.averageJudgeScore ?? "n/a"}`);
|
|
64574
|
+
for (const benchmarkCase of result.cases) {
|
|
64575
|
+
const mark = benchmarkCase.passed ? "PASS" : "FAIL";
|
|
64576
|
+
const error = benchmarkCase.error ? ` error=${benchmarkCase.error}` : "";
|
|
64577
|
+
console.log(`${mark} ${benchmarkCase.id} (${benchmarkCase.durationMs}ms)${error}`);
|
|
64578
|
+
}
|
|
64579
|
+
}
|
|
64580
|
+
}
|
|
64581
|
+
function writePromptBudgetComparison(result, flags, output) {
|
|
64582
|
+
const json = `${JSON.stringify(result, null, 2)}
|
|
64583
|
+
`;
|
|
64584
|
+
if (output) writeFileSync5(output, json, "utf-8");
|
|
64585
|
+
if (flags.json === true || output) {
|
|
64586
|
+
process.stdout.write(json);
|
|
64587
|
+
return;
|
|
64588
|
+
}
|
|
64589
|
+
console.log(`Prompt-budget comparison ${result.status}: ${result.recommendation}`);
|
|
64590
|
+
console.log(`Corpus: ${result.corpusHash}`);
|
|
64591
|
+
console.log(`Duration delta: ${formatMetricDelta(result.deltas.averageTurnDurationMs, "ms")}`);
|
|
64592
|
+
console.log(`Quality delta: ${formatMetricDelta(result.deltas.averageJudgeScore, "")}`);
|
|
64593
|
+
if (result.regressions.length > 0) {
|
|
64594
|
+
console.log("Regressions:");
|
|
64595
|
+
for (const regression of result.regressions) console.log(`- ${regression}`);
|
|
64596
|
+
}
|
|
64597
|
+
}
|
|
64598
|
+
function comparePromptBudgetArtifacts(flags) {
|
|
64599
|
+
const baselinePath = stringFlag(flags.baseline);
|
|
64600
|
+
const postchangePath = stringFlag(flags.postchange);
|
|
64601
|
+
if (!baselinePath || !postchangePath) {
|
|
64602
|
+
throw new Error("prompt-budget compare requires --baseline <path> and --postchange <path>");
|
|
64603
|
+
}
|
|
64604
|
+
const baseline = readPromptBudgetResult(baselinePath);
|
|
64605
|
+
const postchange = readPromptBudgetResult(postchangePath);
|
|
64606
|
+
if (baseline.command !== "benchmark prompt-budget" || postchange.command !== "benchmark prompt-budget") {
|
|
64607
|
+
throw new Error("prompt-budget compare expects prompt-budget benchmark artifacts");
|
|
64608
|
+
}
|
|
64609
|
+
if (baseline.corpusHash !== postchange.corpusHash) {
|
|
64610
|
+
throw new Error("prompt-budget artifacts use different corpus hashes and cannot be compared");
|
|
64611
|
+
}
|
|
64612
|
+
const regressions = findPromptBudgetRegressions(baseline, postchange);
|
|
64613
|
+
const durationDelta = metricDelta(baseline.summary.averageTurnDurationMs, postchange.summary.averageTurnDurationMs);
|
|
64614
|
+
const qualityDelta = metricDelta(baseline.summary.averageJudgeScore, postchange.summary.averageJudgeScore);
|
|
64615
|
+
const continuityDelta = metricDelta(baseline.summary.averageContinuityScore, postchange.summary.averageContinuityScore);
|
|
64616
|
+
const latencyImproved = durationDelta.delta !== null && durationDelta.delta < 0;
|
|
64617
|
+
const qualityRegressed = regressions.length > 0;
|
|
64618
|
+
return {
|
|
64619
|
+
command: "benchmark prompt-budget compare",
|
|
64620
|
+
status: qualityRegressed ? "needs_fix" : "passed",
|
|
64621
|
+
corpusHash: baseline.corpusHash,
|
|
64622
|
+
baseline: baselinePath,
|
|
64623
|
+
postchange: postchangePath,
|
|
64624
|
+
deltas: {
|
|
64625
|
+
averageTurnDurationMs: durationDelta,
|
|
64626
|
+
averageJudgeScore: qualityDelta,
|
|
64627
|
+
averageContinuityScore: continuityDelta
|
|
64628
|
+
},
|
|
64629
|
+
regressions,
|
|
64630
|
+
recommendation: !qualityRegressed && latencyImproved ? "keep" : "adjust_or_revert"
|
|
64631
|
+
};
|
|
64632
|
+
}
|
|
64633
|
+
function readPromptBudgetResult(path) {
|
|
64634
|
+
const parsed = JSON.parse(readFileSync7(path, "utf-8"));
|
|
64635
|
+
if (!parsed || typeof parsed !== "object") throw new Error(`Invalid prompt-budget artifact: ${path}`);
|
|
64636
|
+
return parsed;
|
|
64637
|
+
}
|
|
64638
|
+
function findPromptBudgetRegressions(baseline, postchange) {
|
|
64639
|
+
const regressions = [];
|
|
64640
|
+
const postCases = new Map(postchange.cases.map((benchmarkCase) => [benchmarkCase.id, benchmarkCase]));
|
|
64641
|
+
for (const baselineCase of baseline.cases) {
|
|
64642
|
+
const postCase = postCases.get(baselineCase.id);
|
|
64643
|
+
if (!postCase) {
|
|
64644
|
+
regressions.push(`${baselineCase.id}: missing post-change case`);
|
|
64645
|
+
continue;
|
|
64646
|
+
}
|
|
64647
|
+
if (baselineCase.promptHashes.join("|") !== postCase.promptHashes.join("|")) {
|
|
64648
|
+
regressions.push(`${baselineCase.id}: prompt hashes differ`);
|
|
64649
|
+
}
|
|
64650
|
+
for (const baselineTurn of baselineCase.turns) {
|
|
64651
|
+
const postTurn = postCase.turns.find((turn) => turn.turn === baselineTurn.turn);
|
|
64652
|
+
if (!postTurn) {
|
|
64653
|
+
regressions.push(`${baselineCase.id} turn ${baselineTurn.turn}: missing post-change turn`);
|
|
64654
|
+
continue;
|
|
64655
|
+
}
|
|
64656
|
+
const baselineScore = baselineTurn.judge?.score;
|
|
64657
|
+
const postScore = postTurn.judge?.score;
|
|
64658
|
+
if (typeof baselineScore === "number" && typeof postScore === "number") {
|
|
64659
|
+
if (baselineScore >= 4 && postScore < 4) {
|
|
64660
|
+
regressions.push(`${baselineCase.id} turn ${baselineTurn.turn}: score dropped below pass threshold (${baselineScore} -> ${postScore})`);
|
|
64661
|
+
} else if (postScore < baselineScore - 1) {
|
|
64662
|
+
regressions.push(`${baselineCase.id} turn ${baselineTurn.turn}: score dropped materially (${baselineScore} -> ${postScore})`);
|
|
64663
|
+
}
|
|
64664
|
+
}
|
|
64665
|
+
}
|
|
64666
|
+
const baselineContinuity = baselineCase.continuityJudge?.score;
|
|
64667
|
+
const postContinuity = postCase.continuityJudge?.score;
|
|
64668
|
+
if (typeof baselineContinuity === "number" && typeof postContinuity === "number") {
|
|
64669
|
+
if (baselineContinuity >= 4 && postContinuity < 4) {
|
|
64670
|
+
regressions.push(`${baselineCase.id}: continuity score dropped below pass threshold (${baselineContinuity} -> ${postContinuity})`);
|
|
64671
|
+
} else if (postContinuity < baselineContinuity - 1) {
|
|
64672
|
+
regressions.push(`${baselineCase.id}: continuity score dropped materially (${baselineContinuity} -> ${postContinuity})`);
|
|
64673
|
+
}
|
|
64674
|
+
}
|
|
64675
|
+
}
|
|
64676
|
+
return regressions;
|
|
64677
|
+
}
|
|
64678
|
+
function metricDelta(baseline, postchange) {
|
|
64679
|
+
if (baseline === null || postchange === null) {
|
|
64680
|
+
return { baseline, postchange, delta: null, percent: null };
|
|
64681
|
+
}
|
|
64682
|
+
const delta = round2(postchange - baseline);
|
|
64683
|
+
const percent = baseline === 0 ? null : round2(delta / baseline * 100);
|
|
64684
|
+
return { baseline, postchange, delta, percent };
|
|
64685
|
+
}
|
|
64686
|
+
function formatMetricDelta(delta, suffix) {
|
|
64687
|
+
if (delta.delta === null) return "n/a";
|
|
64688
|
+
const unit = suffix ? suffix : "";
|
|
64689
|
+
const percent = delta.percent === null ? "" : ` (${delta.percent}%)`;
|
|
64690
|
+
return `${delta.baseline}${unit} -> ${delta.postchange}${unit} (${delta.delta}${unit})${percent}`;
|
|
64691
|
+
}
|
|
63848
64692
|
|
|
63849
64693
|
// src/tui.ts
|
|
63850
64694
|
import { createInterface as createInterface4 } from "readline/promises";
|
|
@@ -66271,7 +67115,7 @@ Starting a new chat...
|
|
|
66271
67115
|
acceptTaskProposals: flags["accept-task-proposals"] === true,
|
|
66272
67116
|
piiDetection: flags["no-pii-detection"] !== true,
|
|
66273
67117
|
taskUpdateJobs: false,
|
|
66274
|
-
responseTimeoutMs:
|
|
67118
|
+
responseTimeoutMs: parseResponseTimeoutMs2(flags),
|
|
66275
67119
|
...teamContextFromFlags(flags)
|
|
66276
67120
|
},
|
|
66277
67121
|
redactor
|
|
@@ -67241,7 +68085,7 @@ async function handleGoalChat(client, flags, redactor) {
|
|
|
67241
68085
|
autoApproveMemories: flags["auto-approve-memories"] === true,
|
|
67242
68086
|
acceptTaskProposals: flags["accept-task-proposals"] === true,
|
|
67243
68087
|
piiDetection: flags["no-pii-detection"] !== true,
|
|
67244
|
-
responseTimeoutMs:
|
|
68088
|
+
responseTimeoutMs: parseResponseTimeoutMs2(flags),
|
|
67245
68089
|
...teamContextFromFlags(flags)
|
|
67246
68090
|
},
|
|
67247
68091
|
redactor
|
|
@@ -67824,7 +68668,7 @@ function parsePositiveIntegerFlag(value, flagName) {
|
|
|
67824
68668
|
}
|
|
67825
68669
|
return parsed;
|
|
67826
68670
|
}
|
|
67827
|
-
function
|
|
68671
|
+
function parseResponseTimeoutMs2(flags) {
|
|
67828
68672
|
const seconds = parsePositiveIntegerFlag(flags["response-timeout-seconds"], "--response-timeout-seconds");
|
|
67829
68673
|
return seconds === void 0 ? void 0 : seconds * 1e3;
|
|
67830
68674
|
}
|
|
@@ -68118,7 +68962,7 @@ async function handleChats(client, subcommand, rest, flags, redactor) {
|
|
|
68118
68962
|
send,
|
|
68119
68963
|
dryRun,
|
|
68120
68964
|
confirmDestructive: flags.yes === true,
|
|
68121
|
-
responseTimeoutMs:
|
|
68965
|
+
responseTimeoutMs: parseResponseTimeoutMs2(flags),
|
|
68122
68966
|
...teamContext
|
|
68123
68967
|
});
|
|
68124
68968
|
if (flags.json === true) {
|
|
@@ -68142,7 +68986,7 @@ async function handleChats(client, subcommand, rest, flags, redactor) {
|
|
|
68142
68986
|
chatId,
|
|
68143
68987
|
dryRun,
|
|
68144
68988
|
confirmDestructive: flags.yes === true,
|
|
68145
|
-
responseTimeoutMs:
|
|
68989
|
+
responseTimeoutMs: parseResponseTimeoutMs2(flags),
|
|
68146
68990
|
...teamContext
|
|
68147
68991
|
});
|
|
68148
68992
|
if (flags.json === true) {
|
|
@@ -68169,7 +69013,7 @@ async function handleChats(client, subcommand, rest, flags, redactor) {
|
|
|
68169
69013
|
autoApproveMemories: flags["auto-approve-memories"] === true,
|
|
68170
69014
|
acceptTaskProposals: flags["accept-task-proposals"] === true,
|
|
68171
69015
|
piiDetection: flags["no-pii-detection"] !== true,
|
|
68172
|
-
responseTimeoutMs:
|
|
69016
|
+
responseTimeoutMs: parseResponseTimeoutMs2(flags),
|
|
68173
69017
|
...teamContext,
|
|
68174
69018
|
anonymousLearningMode: client.hasSession() ? void 0 : parseAnonymousLearningModeFlags(flags)
|
|
68175
69019
|
},
|
|
@@ -68247,7 +69091,7 @@ Run 'openmates chats show ` + chatId + "' to check if suggestions have been save
|
|
|
68247
69091
|
autoApproveMemories: flags["auto-approve-memories"] === true,
|
|
68248
69092
|
acceptTaskProposals: flags["accept-task-proposals"] === true,
|
|
68249
69093
|
piiDetection: flags["no-pii-detection"] !== true,
|
|
68250
|
-
responseTimeoutMs:
|
|
69094
|
+
responseTimeoutMs: parseResponseTimeoutMs2(flags),
|
|
68251
69095
|
...teamContext
|
|
68252
69096
|
},
|
|
68253
69097
|
redactor
|
|
@@ -68278,7 +69122,7 @@ Run 'openmates chats show ` + chatId + "' to check if suggestions have been save
|
|
|
68278
69122
|
autoApproveMemories: flags["auto-approve-memories"] === true,
|
|
68279
69123
|
acceptTaskProposals: flags["accept-task-proposals"] === true,
|
|
68280
69124
|
piiDetection: flags["no-pii-detection"] !== true,
|
|
68281
|
-
responseTimeoutMs:
|
|
69125
|
+
responseTimeoutMs: parseResponseTimeoutMs2(flags),
|
|
68282
69126
|
...teamContext
|
|
68283
69127
|
},
|
|
68284
69128
|
redactor
|
|
@@ -68301,7 +69145,7 @@ Run 'openmates chats show ` + chatId + "' to check if suggestions have been save
|
|
|
68301
69145
|
autoApproveSubChats: flags["auto-approve"] === true,
|
|
68302
69146
|
autoApproveMemories: flags["auto-approve-memories"] === true,
|
|
68303
69147
|
piiDetection: flags["no-pii-detection"] !== true,
|
|
68304
|
-
responseTimeoutMs:
|
|
69148
|
+
responseTimeoutMs: parseResponseTimeoutMs2(flags)
|
|
68305
69149
|
},
|
|
68306
69150
|
redactor
|
|
68307
69151
|
);
|
|
@@ -68864,7 +69708,7 @@ async function sendApiKeyChatNew(client, apiKey, message, flags) {
|
|
|
68864
69708
|
try {
|
|
68865
69709
|
response = await sdk.chats.send(message, {
|
|
68866
69710
|
saveToAccount: false,
|
|
68867
|
-
recoveryTimeoutMs:
|
|
69711
|
+
recoveryTimeoutMs: parseResponseTimeoutMs2(flags)
|
|
68868
69712
|
});
|
|
68869
69713
|
} catch (err) {
|
|
68870
69714
|
if (!isSdkChatScopeDenied(err)) throw err;
|
|
@@ -69006,7 +69850,7 @@ function normalizeApiKeyLoadedChat(payload) {
|
|
|
69006
69850
|
const messages = rawMessages.map((message) => {
|
|
69007
69851
|
const messageId = String(message.client_message_id ?? message.message_id ?? message.id ?? "");
|
|
69008
69852
|
const directEmbedIds = Array.isArray(message.embedIds) ? message.embedIds : Array.isArray(message.embed_ids) ? message.embed_ids : [];
|
|
69009
|
-
const hashedMessageId = messageId ?
|
|
69853
|
+
const hashedMessageId = messageId ? createHash14("sha256").update(messageId).digest("hex") : null;
|
|
69010
69854
|
const matchedEmbedIds = hashedMessageId ? rawEmbeds.filter((embed) => embed.hashed_message_id === hashedMessageId && !embed.parent_embed_id).map((embed) => String(embed.embed_id ?? embed.id ?? "")).filter(Boolean) : [];
|
|
69011
69855
|
return {
|
|
69012
69856
|
id: messageId,
|
|
@@ -69666,9 +70510,9 @@ Run: openmates apps ${command.app_id} ${command.skill_id} --help`
|
|
|
69666
70510
|
return value;
|
|
69667
70511
|
}
|
|
69668
70512
|
function buildTravelConnectionsRequest(positionals, flags) {
|
|
69669
|
-
const origin =
|
|
69670
|
-
const destination =
|
|
69671
|
-
const date =
|
|
70513
|
+
const origin = stringFlag2(flags, "origin") ?? positionals[0];
|
|
70514
|
+
const destination = stringFlag2(flags, "destination") ?? positionals[1];
|
|
70515
|
+
const date = stringFlag2(flags, "date") ?? positionals[2];
|
|
69672
70516
|
if (!origin || !destination || !date) {
|
|
69673
70517
|
throw new Error(
|
|
69674
70518
|
"Missing travel route. Use --origin <place> --destination <place> --date <YYYY-MM-DD>.\n\nExample: openmates apps travel search_connections --origin Berlin --destination Munich --date 2026-08-01 --json"
|
|
@@ -69677,12 +70521,20 @@ function buildTravelConnectionsRequest(positionals, flags) {
|
|
|
69677
70521
|
const request = {
|
|
69678
70522
|
legs: [{ origin, destination, date }]
|
|
69679
70523
|
};
|
|
69680
|
-
const transport =
|
|
70524
|
+
const transport = stringFlag2(flags, "transport") ?? stringFlag2(flags, "transport-method");
|
|
69681
70525
|
if (transport) request.transport_methods = [transport];
|
|
70526
|
+
const providers = csvFlag(flags, "providers") ?? csvFlag(flags, "provider");
|
|
70527
|
+
if (providers) request.providers = providers;
|
|
70528
|
+
const ownedPasses = csvFlag(flags, "owned_passes") ?? csvFlag(flags, "owned-passes");
|
|
70529
|
+
if (ownedPasses) request.owned_passes = ownedPasses;
|
|
70530
|
+
const railProducts = csvFlag(flags, "rail_products") ?? csvFlag(flags, "rail-products");
|
|
70531
|
+
if (railProducts) request.rail_products = railProducts;
|
|
70532
|
+
const passOnly = booleanFlag(flags, "pass_only") ?? booleanFlag(flags, "pass-only");
|
|
70533
|
+
if (passOnly !== void 0) request.pass_only = passOnly;
|
|
69682
70534
|
return request;
|
|
69683
70535
|
}
|
|
69684
70536
|
function buildCodeImageToHtmlRequest(positionals, flags) {
|
|
69685
|
-
const file =
|
|
70537
|
+
const file = stringFlag2(flags, "file") ?? positionals[0];
|
|
69686
70538
|
if (!file) {
|
|
69687
70539
|
throw new Error(
|
|
69688
70540
|
"Missing screenshot file. Use --file <path>.\n\nExample: openmates apps code image_to_html --file ./mockup.png --json"
|
|
@@ -69730,10 +70582,26 @@ function primaryPositionalProperty(properties, required) {
|
|
|
69730
70582
|
function readFlag(flags, name) {
|
|
69731
70583
|
return flags[name] ?? flags[kebabCase(name)];
|
|
69732
70584
|
}
|
|
69733
|
-
function
|
|
70585
|
+
function stringFlag2(flags, name) {
|
|
69734
70586
|
const raw = readFlag(flags, name);
|
|
69735
70587
|
return typeof raw === "string" && raw.trim() ? raw.trim() : void 0;
|
|
69736
70588
|
}
|
|
70589
|
+
function csvFlag(flags, name) {
|
|
70590
|
+
const raw = readFlag(flags, name);
|
|
70591
|
+
if (typeof raw !== "string" || !raw.trim()) return void 0;
|
|
70592
|
+
const values = raw.split(/[,\n]/).map((value) => value.trim()).filter(Boolean);
|
|
70593
|
+
return values.length > 0 ? values : void 0;
|
|
70594
|
+
}
|
|
70595
|
+
function booleanFlag(flags, name) {
|
|
70596
|
+
const raw = readFlag(flags, name);
|
|
70597
|
+
if (raw === void 0) return void 0;
|
|
70598
|
+
if (raw === true) return true;
|
|
70599
|
+
if (raw === false) return false;
|
|
70600
|
+
const normalized = raw.trim().toLowerCase();
|
|
70601
|
+
if (["1", "true", "yes", "on"].includes(normalized)) return true;
|
|
70602
|
+
if (["0", "false", "no", "off"].includes(normalized)) return false;
|
|
70603
|
+
return true;
|
|
70604
|
+
}
|
|
69737
70605
|
function coerceAppSkillFlagValue(name, raw, schema) {
|
|
69738
70606
|
if (schema.type === "boolean") return raw === true || raw === "true";
|
|
69739
70607
|
if (schema.type === "integer") {
|
|
@@ -69776,6 +70644,10 @@ ${command.description}
|
|
|
69776
70644
|
console.log(" --destination <place> Route destination for a typed connection search.");
|
|
69777
70645
|
console.log(" --date <YYYY-MM-DD> Departure date for a typed connection search.");
|
|
69778
70646
|
console.log(" --transport <mode> Optional transport mode, e.g. train or plane.");
|
|
70647
|
+
console.log(" --providers <csv> Optional providers, e.g. deutsche_bahn,transitous.");
|
|
70648
|
+
console.log(" --owned-passes <csv> Optional owned passes, e.g. deutschland_ticket.");
|
|
70649
|
+
console.log(" --pass-only Prefer pass-covered routes when supported.");
|
|
70650
|
+
console.log(" --rail-products <csv> Optional rail filters, e.g. high_speed,regional.");
|
|
69779
70651
|
}
|
|
69780
70652
|
if (command.app_id === "code" && command.skill_id === "image_to_html") {
|
|
69781
70653
|
console.log(" --file <path> Local PNG, JPEG, or WEBP screenshot to convert.");
|
|
@@ -70086,7 +70958,7 @@ async function handleDesignIconExport(client, positionals, flags, apiKey) {
|
|
|
70086
70958
|
printDesignIconExportHelp();
|
|
70087
70959
|
return;
|
|
70088
70960
|
}
|
|
70089
|
-
const outputPath =
|
|
70961
|
+
const outputPath = stringFlag2(flags, "output");
|
|
70090
70962
|
if (!outputPath) {
|
|
70091
70963
|
console.error(
|
|
70092
70964
|
"Missing --output flag.\n\nUsage:\n openmates apps design export-icon lucide:home --output home.svg [--color '#111827']\n openmates apps design export-icon --prefix lucide --name home --format png --size 64 --output home.png\n"
|
|
@@ -70103,7 +70975,7 @@ async function handleDesignIconExport(client, positionals, flags, apiKey) {
|
|
|
70103
70975
|
...iconRef,
|
|
70104
70976
|
outputPath,
|
|
70105
70977
|
format,
|
|
70106
|
-
color:
|
|
70978
|
+
color: stringFlag2(flags, "color"),
|
|
70107
70979
|
palette: flags.palette === true,
|
|
70108
70980
|
allowPaletteRecolor: flags["allow-palette-recolor"] === true,
|
|
70109
70981
|
size,
|
|
@@ -70130,9 +71002,9 @@ async function handleDesignIconExport(client, positionals, flags, apiKey) {
|
|
|
70130
71002
|
}
|
|
70131
71003
|
}
|
|
70132
71004
|
function parseDesignIconReference(positionals, flags) {
|
|
70133
|
-
const svgPath =
|
|
71005
|
+
const svgPath = stringFlag2(flags, "svg-path");
|
|
70134
71006
|
if (svgPath) return { svgPath };
|
|
70135
|
-
const icon =
|
|
71007
|
+
const icon = stringFlag2(flags, "icon") ?? positionals[0];
|
|
70136
71008
|
if (icon) {
|
|
70137
71009
|
const [prefix, name, extra] = icon.split(":");
|
|
70138
71010
|
if (!prefix || !name || extra !== void 0) {
|
|
@@ -70141,8 +71013,8 @@ function parseDesignIconReference(positionals, flags) {
|
|
|
70141
71013
|
return { prefix, name };
|
|
70142
71014
|
}
|
|
70143
71015
|
return {
|
|
70144
|
-
prefix:
|
|
70145
|
-
name:
|
|
71016
|
+
prefix: stringFlag2(flags, "prefix"),
|
|
71017
|
+
name: stringFlag2(flags, "name")
|
|
70146
71018
|
};
|
|
70147
71019
|
}
|
|
70148
71020
|
function parseDesignIconExportFormat(value) {
|
|
@@ -71180,12 +72052,12 @@ async function handleFinance(client, subcommand, _rest, flags) {
|
|
|
71180
72052
|
if (flags["csv-only"] !== true) {
|
|
71181
72053
|
try {
|
|
71182
72054
|
connectedAccount = await client.decryptConnectedAccountForSkill({
|
|
71183
|
-
accountId:
|
|
72055
|
+
accountId: stringFlag2(flags, "connected-account"),
|
|
71184
72056
|
appId: "finance",
|
|
71185
72057
|
providerId: "revolut_business"
|
|
71186
72058
|
});
|
|
71187
72059
|
} catch (error) {
|
|
71188
|
-
if (csvStatements.length === 0 ||
|
|
72060
|
+
if (csvStatements.length === 0 || stringFlag2(flags, "connected-account")) throw error;
|
|
71189
72061
|
}
|
|
71190
72062
|
}
|
|
71191
72063
|
if (!connectedAccount && csvStatements.length === 0) {
|
|
@@ -71223,11 +72095,11 @@ function readFinanceCsvStatements(flags) {
|
|
|
71223
72095
|
}
|
|
71224
72096
|
function buildFinanceCheckAccountsInput(flags, csvStatements, connectedAccount) {
|
|
71225
72097
|
const input = {
|
|
71226
|
-
period:
|
|
71227
|
-
projection_horizon:
|
|
72098
|
+
period: stringFlag2(flags, "period") ?? "monthly",
|
|
72099
|
+
projection_horizon: stringFlag2(flags, "projection-horizon") ?? "monthly"
|
|
71228
72100
|
};
|
|
71229
|
-
const startDate =
|
|
71230
|
-
const endDate =
|
|
72101
|
+
const startDate = stringFlag2(flags, "from") ?? stringFlag2(flags, "start-date");
|
|
72102
|
+
const endDate = stringFlag2(flags, "to") ?? stringFlag2(flags, "end-date");
|
|
71231
72103
|
if (startDate) input.start_date = startDate;
|
|
71232
72104
|
if (endDate) input.end_date = endDate;
|
|
71233
72105
|
addFinanceListFilter(input, "account_filters", flags.account);
|
|
@@ -71235,7 +72107,7 @@ function buildFinanceCheckAccountsInput(flags, csvStatements, connectedAccount)
|
|
|
71235
72107
|
addFinanceListFilter(input, "category_filters", flags.category ?? flags.categories);
|
|
71236
72108
|
addFinanceListFilter(input, "state_filters", flags.state ?? flags.states);
|
|
71237
72109
|
addFinanceListFilter(input, "placeholder_filters", flags.placeholder ?? flags.placeholders);
|
|
71238
|
-
const direction =
|
|
72110
|
+
const direction = stringFlag2(flags, "direction");
|
|
71239
72111
|
if (direction) input.direction_filter = direction;
|
|
71240
72112
|
if (csvStatements.length > 0) input.csv_statements = csvStatements;
|
|
71241
72113
|
if (connectedAccount) {
|
|
@@ -75521,6 +76393,7 @@ export {
|
|
|
75521
76393
|
ASSISTANT_FEEDBACK_THANKS,
|
|
75522
76394
|
ASSISTANT_FEEDBACK_REPORT_TITLE,
|
|
75523
76395
|
buildAssistantFeedbackDecision,
|
|
76396
|
+
buildTravelConnectionsRequest,
|
|
75524
76397
|
serializeToYaml,
|
|
75525
76398
|
getExtForLang
|
|
75526
76399
|
};
|