@tangle-network/agent-runtime 0.105.0 → 0.106.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/activation-CRjVfOiy.js +203 -0
- package/dist/activation-CRjVfOiy.js.map +1 -0
- package/dist/activation-DyWB0K6E.d.ts +98 -0
- package/dist/agent.d.ts +216 -234
- package/dist/agent.js +615 -547
- package/dist/agent.js.map +1 -1
- package/dist/analyst-loop-DGdtVENg.js +517 -0
- package/dist/analyst-loop-DGdtVENg.js.map +1 -0
- package/dist/analyst-loop.d.ts +10 -43
- package/dist/analyst-loop.js +2 -11
- package/dist/backends-CiOCyRHb.js +743 -0
- package/dist/backends-CiOCyRHb.js.map +1 -0
- package/dist/candidate-execution/index.d.ts +4 -192
- package/dist/candidate-execution/index.js +5 -66
- package/dist/candidate-execution-DqMbv1rU.js +488 -0
- package/dist/candidate-execution-DqMbv1rU.js.map +1 -0
- package/dist/coder-yhVWbdWc.js +40 -0
- package/dist/coder-yhVWbdWc.js.map +1 -0
- package/dist/conversation-BpLQZGPH.js +1265 -0
- package/dist/conversation-BpLQZGPH.js.map +1 -0
- package/dist/conversation.d.ts +2 -804
- package/dist/conversation.js +2 -55
- package/dist/environment-provider-CfLUmKmf.js +1168 -0
- package/dist/environment-provider-CfLUmKmf.js.map +1 -0
- package/dist/environment-provider-IUGU3epE.d.ts +964 -0
- package/dist/environment-provider.d.ts +2 -78
- package/dist/environment-provider.js +2 -18
- package/dist/errors-DEAvWQPy.js +111 -0
- package/dist/errors-DEAvWQPy.js.map +1 -0
- package/dist/improvement-adapter-Cr5TBSji.d.ts +166 -0
- package/dist/improvement-cycle-C095kS_g.js +2624 -0
- package/dist/improvement-cycle-C095kS_g.js.map +1 -0
- package/dist/index-BGdxAwfO.d.ts +838 -0
- package/dist/index-C-FYUuFG.d.ts +193 -0
- package/dist/index-CbNM40xG.d.ts +140 -0
- package/dist/index-DruSKs3Z.d.ts +7224 -0
- package/dist/index-Efjb3nrQ.d.ts +671 -0
- package/dist/index.d.ts +14 -1058
- package/dist/index.js +2106 -2082
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +768 -889
- package/dist/intelligence.js +1466 -1367
- package/dist/intelligence.js.map +1 -1
- package/dist/{kb-gate-CwHO0vz6.d.ts → kb-gate-C8z2juK8.d.ts} +33 -31
- package/dist/kb-gate-DpaSwXVx.js +80 -0
- package/dist/kb-gate-DpaSwXVx.js.map +1 -0
- package/dist/knowledge-Ku-mPP4f.js +433 -0
- package/dist/knowledge-Ku-mPP4f.js.map +1 -0
- package/dist/knowledge.d.ts +2 -150
- package/dist/knowledge.js +2 -40
- package/dist/local-harness-Dh8PJ0ot.d.ts +455 -0
- package/dist/loop-runner-bin-BXsFEUcf.d.ts +162 -0
- package/dist/loop-runner-bin-CRagyuNb.js +268 -0
- package/dist/loop-runner-bin-CRagyuNb.js.map +1 -0
- package/dist/loop-runner-bin.d.ts +2 -15
- package/dist/loop-runner-bin.js +2 -24
- package/dist/loops.d.ts +5 -4164
- package/dist/loops.js +8 -419
- package/dist/mcp/bin.d.ts +1 -1
- package/dist/mcp/bin.js +116 -146
- package/dist/mcp/bin.js.map +1 -1
- package/dist/mcp/index.d.ts +295 -436
- package/dist/mcp/index.js +1242 -1177
- package/dist/mcp/index.js.map +1 -1
- package/dist/mcp/memory-bin.d.ts +1 -1
- package/dist/mcp/memory-bin.js +37 -16
- package/dist/mcp/memory-bin.js.map +1 -1
- package/dist/memory-server-DL6cE2Ag.js +234 -0
- package/dist/memory-server-DL6cE2Ag.js.map +1 -0
- package/dist/model-resolution-Btd9iIKV.js +98 -0
- package/dist/model-resolution-Btd9iIKV.js.map +1 -0
- package/dist/openai-tools-B68JaOCx.d.ts +24 -0
- package/dist/openai-tools-CKLy1C7M.js +44 -0
- package/dist/openai-tools-CKLy1C7M.js.map +1 -0
- package/dist/otel-export-D34IQW1C.js +843 -0
- package/dist/otel-export-D34IQW1C.js.map +1 -0
- package/dist/platform.d.ts +188 -185
- package/dist/platform.js +185 -218
- package/dist/platform.js.map +1 -1
- package/dist/prepare-B53sPV7M.js +4292 -0
- package/dist/prepare-B53sPV7M.js.map +1 -0
- package/dist/primeintellect/index.d.ts +149 -145
- package/dist/primeintellect/index.js +626 -1003
- package/dist/primeintellect/index.js.map +1 -1
- package/dist/profiles.d.ts +296 -490
- package/dist/profiles.js +722 -749
- package/dist/profiles.js.map +1 -1
- package/dist/protected-model-port-BLPH5VVv.js +722 -0
- package/dist/protected-model-port-BLPH5VVv.js.map +1 -0
- package/dist/protected-model-port-BP6Z4eau.d.ts +283 -0
- package/dist/protected-redaction--F3v1oo8.js +200 -0
- package/dist/protected-redaction--F3v1oo8.js.map +1 -0
- package/dist/redact-D-u-rrcn.js +82 -0
- package/dist/redact-D-u-rrcn.js.map +1 -0
- package/dist/redact-kbIOjfjM.d.ts +452 -0
- package/dist/researcher-CoVqNhfI.js +433 -0
- package/dist/researcher-CoVqNhfI.js.map +1 -0
- package/dist/runtime-5zM5XxLr.js +5408 -0
- package/dist/runtime-5zM5XxLr.js.map +1 -0
- package/dist/sandbox-events-DeI5xX8P.js +316 -0
- package/dist/sandbox-events-DeI5xX8P.js.map +1 -0
- package/dist/spawn-journal-CwPvKUTa.js +456 -0
- package/dist/spawn-journal-CwPvKUTa.js.map +1 -0
- package/dist/structural-rollout-IXUEplky.js +1445 -0
- package/dist/structural-rollout-IXUEplky.js.map +1 -0
- package/dist/substrate-B0TYNrXn.js +27 -0
- package/dist/substrate-B0TYNrXn.js.map +1 -0
- package/dist/{substrate-Dyiqmd3o.d.ts → substrate-BcnuSHXm.d.ts} +57 -54
- package/dist/supervise-BiRutHS9.js +7321 -0
- package/dist/supervise-BiRutHS9.js.map +1 -0
- package/dist/supervisor-DTKhF-RV.js +3826 -0
- package/dist/supervisor-DTKhF-RV.js.map +1 -0
- package/dist/testing.d.ts +4 -3
- package/dist/testing.js +2478 -2637
- package/dist/testing.js.map +1 -1
- package/dist/tool-server-RcWgLIsL.js +141 -0
- package/dist/tool-server-RcWgLIsL.js.map +1 -0
- package/dist/trace-propagation-CJJC7SVB.js +57 -0
- package/dist/trace-propagation-CJJC7SVB.js.map +1 -0
- package/dist/types-BaIpb0XT.d.ts +492 -0
- package/dist/types-BevOjfTY.d.ts +765 -0
- package/dist/types-C9j4qg6l.d.ts +500 -0
- package/dist/types-Cja5OTGh.d.ts +190 -0
- package/dist/util-MVgdwuIS.js +145 -0
- package/dist/util-MVgdwuIS.js.map +1 -0
- package/dist/workspace-archive-BosQiotA.js +575 -0
- package/dist/workspace-archive-BosQiotA.js.map +1 -0
- package/package.json +44 -56
- package/skills/build-with-agent-runtime/SKILL.md +1 -0
- package/dist/activation-Cm-9MMX1.d.ts +0 -69
- package/dist/analyst-loop.js.map +0 -1
- package/dist/backends-BO4Jqoe2.d.ts +0 -134
- package/dist/candidate-execution/index.js.map +0 -1
- package/dist/chunk-2KGAN2HM.js +0 -1256
- package/dist/chunk-2KGAN2HM.js.map +0 -1
- package/dist/chunk-2VZYRB55.js +0 -834
- package/dist/chunk-2VZYRB55.js.map +0 -1
- package/dist/chunk-3MDZX7YU.js +0 -125
- package/dist/chunk-3MDZX7YU.js.map +0 -1
- package/dist/chunk-5IBQAPVB.js +0 -29
- package/dist/chunk-5IBQAPVB.js.map +0 -1
- package/dist/chunk-5QOB7H74.js +0 -387
- package/dist/chunk-5QOB7H74.js.map +0 -1
- package/dist/chunk-C5RTIBNZ.js +0 -48
- package/dist/chunk-C5RTIBNZ.js.map +0 -1
- package/dist/chunk-CTRA64LY.js +0 -680
- package/dist/chunk-CTRA64LY.js.map +0 -1
- package/dist/chunk-DPEUKJRO.js +0 -27
- package/dist/chunk-DPEUKJRO.js.map +0 -1
- package/dist/chunk-FD2MBMOH.js +0 -494
- package/dist/chunk-FD2MBMOH.js.map +0 -1
- package/dist/chunk-H5QPIZNX.js +0 -907
- package/dist/chunk-H5QPIZNX.js.map +0 -1
- package/dist/chunk-HLKC4UYB.js +0 -3514
- package/dist/chunk-HLKC4UYB.js.map +0 -1
- package/dist/chunk-HNP72PNU.js +0 -192
- package/dist/chunk-HNP72PNU.js.map +0 -1
- package/dist/chunk-ISPWRSEC.js +0 -408
- package/dist/chunk-ISPWRSEC.js.map +0 -1
- package/dist/chunk-KRBFHMV6.js +0 -1228
- package/dist/chunk-KRBFHMV6.js.map +0 -1
- package/dist/chunk-M22PL4GH.js +0 -171
- package/dist/chunk-M22PL4GH.js.map +0 -1
- package/dist/chunk-NURIQBJQ.js +0 -72
- package/dist/chunk-NURIQBJQ.js.map +0 -1
- package/dist/chunk-OPVWXJ2H.js +0 -75
- package/dist/chunk-OPVWXJ2H.js.map +0 -1
- package/dist/chunk-PZZKQVQV.js +0 -52
- package/dist/chunk-PZZKQVQV.js.map +0 -1
- package/dist/chunk-Q2JSAVQ3.js +0 -671
- package/dist/chunk-Q2JSAVQ3.js.map +0 -1
- package/dist/chunk-RDOAVVHY.js +0 -699
- package/dist/chunk-RDOAVVHY.js.map +0 -1
- package/dist/chunk-SGKPNBXE.js +0 -60
- package/dist/chunk-SGKPNBXE.js.map +0 -1
- package/dist/chunk-SMQXZGLZ.js +0 -351
- package/dist/chunk-SMQXZGLZ.js.map +0 -1
- package/dist/chunk-UHEZW5BU.js +0 -2918
- package/dist/chunk-UHEZW5BU.js.map +0 -1
- package/dist/chunk-UPWGXKXB.js +0 -254
- package/dist/chunk-UPWGXKXB.js.map +0 -1
- package/dist/chunk-VISA6CI3.js +0 -200
- package/dist/chunk-VISA6CI3.js.map +0 -1
- package/dist/chunk-WMTCUOQL.js +0 -474
- package/dist/chunk-WMTCUOQL.js.map +0 -1
- package/dist/chunk-WRTOVNN4.js +0 -4728
- package/dist/chunk-WRTOVNN4.js.map +0 -1
- package/dist/chunk-WSTRQZYQ.js +0 -5576
- package/dist/chunk-WSTRQZYQ.js.map +0 -1
- package/dist/chunk-XBG2W2VW.js +0 -6434
- package/dist/chunk-XBG2W2VW.js.map +0 -1
- package/dist/chunk-YEJR7IXO.js +0 -68
- package/dist/chunk-YEJR7IXO.js.map +0 -1
- package/dist/chunk-YJZA2BIK.js +0 -1338
- package/dist/chunk-YJZA2BIK.js.map +0 -1
- package/dist/completion-gate-BMy5LGoP.d.ts +0 -256
- package/dist/conversation.js.map +0 -1
- package/dist/coordination-BZZSVYpZ.d.ts +0 -2570
- package/dist/environment-provider.js.map +0 -1
- package/dist/improvement-adapter-HAZz-7vK.d.ts +0 -203
- package/dist/knowledge.js.map +0 -1
- package/dist/local-harness-BDNcl6jI.d.ts +0 -175
- package/dist/loop-runner-bin-68aoM9-7.d.ts +0 -199
- package/dist/loop-runner-bin.js.map +0 -1
- package/dist/loops.js.map +0 -1
- package/dist/openai-tools-fnj6SRVg.d.ts +0 -46
- package/dist/protected-model-port-B4t-OQjL.d.ts +0 -271
- package/dist/redact-BuFjxBUJ.d.ts +0 -481
- package/dist/sanitize-DEbPNtyI.d.ts +0 -328
- package/dist/structural-rollout-DEh6CEsa.d.ts +0 -446
- package/dist/supervise-4_48wwvy.d.ts +0 -379
- package/dist/types-BCoemcXU.d.ts +0 -802
- package/dist/types-BwoZWq-i.d.ts +0 -510
- package/dist/types-Cbp72qyu.d.ts +0 -491
- package/dist/types-CmYCMbFT.d.ts +0 -203
- package/dist/types-CvNwMzQt.d.ts +0 -957
- package/dist/worktree-fanout-DxhAWr5Z.d.ts +0 -999
|
@@ -1,1118 +1,741 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
} from "
|
|
4
|
-
import "
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
import { mkdir, readFile, rename, rm, stat, writeFile } from "fs/promises";
|
|
9
|
-
import { basename, dirname, isAbsolute, join, normalize, relative, resolve, sep } from "path";
|
|
10
|
-
import { canonicalJson } from "@tangle-network/agent-eval";
|
|
11
|
-
|
|
12
|
-
// src/primeintellect/validation.ts
|
|
1
|
+
import { n as createOpenAICompatibleBackend } from "../backends-CiOCyRHb.js";
|
|
2
|
+
import { canonicalJson, validateRunRecord } from "@tangle-network/agent-eval";
|
|
3
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
4
|
+
import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
5
|
+
import { basename, dirname, isAbsolute, join, normalize, relative, resolve, sep } from "node:path";
|
|
6
|
+
//#region src/primeintellect/validation.ts
|
|
7
|
+
/** Validate the PrimeIntellect prompt shape shared by package creation and runner input. */
|
|
13
8
|
function validatePrimeIntellectPrompt(value, path) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
for (const [index, message] of value.entries()) {
|
|
19
|
-
validateMessage(message, `${path}[${index}]`);
|
|
20
|
-
}
|
|
21
|
-
return value;
|
|
9
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
10
|
+
if (!Array.isArray(value) || value.length === 0) throw new Error(`${path} must be a non-empty string or message array`);
|
|
11
|
+
for (const [index, message] of value.entries()) validateMessage(message, `${path}[${index}]`);
|
|
12
|
+
return value;
|
|
22
13
|
}
|
|
23
14
|
function validatePrimeIntellectJson(value, path) {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
}
|
|
41
|
-
throw new Error(`${path} is not JSON serializable`);
|
|
15
|
+
if (value === null || ["string", "boolean"].includes(typeof value)) return;
|
|
16
|
+
if (typeof value === "number") {
|
|
17
|
+
if (!Number.isFinite(value)) throw new Error(`${path} contains a non-finite number`);
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
if (Array.isArray(value)) {
|
|
21
|
+
value.forEach((entry, index) => {
|
|
22
|
+
validatePrimeIntellectJson(entry, `${path}[${index}]`);
|
|
23
|
+
});
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
if (typeof value === "object") {
|
|
27
|
+
for (const [key, entry] of Object.entries(value)) validatePrimeIntellectJson(entry, `${path}.${key}`);
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
throw new Error(`${path} is not JSON serializable`);
|
|
42
31
|
}
|
|
43
32
|
function validatePrimeIntellectJsonObject(value, path) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
33
|
+
const output = record$1(value, path);
|
|
34
|
+
validatePrimeIntellectJson(output, path);
|
|
35
|
+
return output;
|
|
47
36
|
}
|
|
48
37
|
function validateMessage(value, path) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
38
|
+
const message = record$1(value, path);
|
|
39
|
+
const role = message.role;
|
|
40
|
+
if (![
|
|
41
|
+
"system",
|
|
42
|
+
"user",
|
|
43
|
+
"assistant",
|
|
44
|
+
"tool"
|
|
45
|
+
].includes(String(role))) throw new Error(`${path}.role is invalid`);
|
|
46
|
+
if (role === "system" || role === "user") {
|
|
47
|
+
assertOnlyKeys(message, ["role", "content"], path);
|
|
48
|
+
validateContent(message.content, `${path}.content`);
|
|
49
|
+
} else if (role === "assistant") {
|
|
50
|
+
assertOnlyKeys(message, [
|
|
51
|
+
"role",
|
|
52
|
+
"content",
|
|
53
|
+
"reasoning_content",
|
|
54
|
+
"tool_calls",
|
|
55
|
+
"provider_state"
|
|
56
|
+
], path);
|
|
57
|
+
optionalNullableString(message.content, `${path}.content`);
|
|
58
|
+
optionalNullableString(message.reasoning_content, `${path}.reasoning_content`);
|
|
59
|
+
if (message.tool_calls !== void 0) validateToolCalls(message.tool_calls, `${path}.tool_calls`);
|
|
60
|
+
if (message.provider_state !== void 0) validateProviderState(message.provider_state, `${path}.provider_state`);
|
|
61
|
+
} else {
|
|
62
|
+
assertOnlyKeys(message, [
|
|
63
|
+
"role",
|
|
64
|
+
"tool_call_id",
|
|
65
|
+
"content",
|
|
66
|
+
"name"
|
|
67
|
+
], path);
|
|
68
|
+
nonEmptyString$2(message.tool_call_id, `${path}.tool_call_id`);
|
|
69
|
+
if (message.name !== void 0 && typeof message.name !== "string") throw new Error(`${path}.name must be a string`);
|
|
70
|
+
validateContent(message.content, `${path}.content`);
|
|
71
|
+
}
|
|
72
|
+
validatePrimeIntellectJson(message, path);
|
|
80
73
|
}
|
|
81
74
|
function validateToolCalls(value, path) {
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
75
|
+
if (!Array.isArray(value)) throw new Error(`${path} must be an array`);
|
|
76
|
+
for (const [index, rawCall] of value.entries()) {
|
|
77
|
+
const call = record$1(rawCall, `${path}[${index}]`);
|
|
78
|
+
assertOnlyKeys(call, [
|
|
79
|
+
"id",
|
|
80
|
+
"name",
|
|
81
|
+
"arguments"
|
|
82
|
+
], `${path}[${index}]`);
|
|
83
|
+
for (const field of [
|
|
84
|
+
"id",
|
|
85
|
+
"name",
|
|
86
|
+
"arguments"
|
|
87
|
+
]) nonEmptyString$2(call[field], `${path}[${index}].${field}`);
|
|
88
|
+
}
|
|
90
89
|
}
|
|
91
90
|
function validateProviderState(value, path) {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
const state = record(rawState, `${path}[${index}]`);
|
|
95
|
-
validatePrimeIntellectJson(state, `${path}[${index}]`);
|
|
96
|
-
}
|
|
91
|
+
if (!Array.isArray(value)) throw new Error(`${path} must be an array`);
|
|
92
|
+
for (const [index, rawState] of value.entries()) validatePrimeIntellectJson(record$1(rawState, `${path}[${index}]`), `${path}[${index}]`);
|
|
97
93
|
}
|
|
98
94
|
function validateContent(value, path) {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
throw new Error(`${path}[${index}] is not a supported text or image_url content part`);
|
|
116
|
-
}
|
|
95
|
+
if (typeof value === "string") return;
|
|
96
|
+
if (!Array.isArray(value) || value.length === 0) throw new Error(`${path} must be a string or non-empty content array`);
|
|
97
|
+
for (const [index, rawPart] of value.entries()) {
|
|
98
|
+
const part = record$1(rawPart, `${path}[${index}]`);
|
|
99
|
+
if (part.type === "text" && typeof part.text === "string") {
|
|
100
|
+
assertOnlyKeys(part, ["type", "text"], `${path}[${index}]`);
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
const image = part.image_url;
|
|
104
|
+
if (part.type === "image_url" && image !== null && typeof image === "object" && !Array.isArray(image) && typeof image.url === "string") {
|
|
105
|
+
assertOnlyKeys(part, ["type", "image_url"], `${path}[${index}]`);
|
|
106
|
+
assertOnlyKeys(image, ["url"], `${path}[${index}].image_url`);
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
throw new Error(`${path}[${index}] is not a supported text or image_url content part`);
|
|
110
|
+
}
|
|
117
111
|
}
|
|
118
112
|
function optionalNullableString(value, path) {
|
|
119
|
-
|
|
120
|
-
throw new Error(`${path} must be a string or null`);
|
|
121
|
-
}
|
|
113
|
+
if (value !== void 0 && value !== null && typeof value !== "string") throw new Error(`${path} must be a string or null`);
|
|
122
114
|
}
|
|
123
|
-
function nonEmptyString(value, path) {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
127
|
-
return value;
|
|
115
|
+
function nonEmptyString$2(value, path) {
|
|
116
|
+
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
117
|
+
return value;
|
|
128
118
|
}
|
|
129
|
-
function record(value, path) {
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
}
|
|
133
|
-
return value;
|
|
119
|
+
function record$1(value, path) {
|
|
120
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(`${path} must be an object`);
|
|
121
|
+
return value;
|
|
134
122
|
}
|
|
135
123
|
function assertOnlyKeys(value, allowed, path) {
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
var DEFAULT_SCORING_TIMEOUT = 300;
|
|
124
|
+
for (const key of Object.keys(value)) if (!allowed.includes(key)) throw new Error(`${path}.${key} is not supported`);
|
|
125
|
+
}
|
|
126
|
+
//#endregion
|
|
127
|
+
//#region src/primeintellect/package.ts
|
|
128
|
+
const VERIFIERS_RANGE = ">=0.2.0,<0.3.0";
|
|
129
|
+
const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/;
|
|
130
|
+
const PACKAGE_NAME = /^[a-z][a-z0-9-]{0,62}$/;
|
|
131
|
+
const VERSION = /^\d+\.\d+\.\d+(?:[-+][a-zA-Z0-9.-]+)?$/;
|
|
132
|
+
const DEFAULT_MAX_TURNS = 16;
|
|
133
|
+
const DEFAULT_ROLLOUT_TIMEOUT = 3600;
|
|
134
|
+
const DEFAULT_SCORING_TIMEOUT = 300;
|
|
135
|
+
/** Build a complete PrimeIntellect Verifiers package without writing to disk. */
|
|
149
136
|
function createPrimeIntellectPackage(options) {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
taskIdsSha256: sha256(
|
|
195
|
-
validated.tasks.map((task) => `${task.split}:${task.id}`).sort().join("\n")
|
|
196
|
-
),
|
|
197
|
-
filesSha256
|
|
198
|
-
};
|
|
199
|
-
files["manifest.json"] = `${JSON.stringify(manifest, null, 2)}
|
|
200
|
-
`;
|
|
201
|
-
return { manifest, files: Object.freeze(files) };
|
|
202
|
-
}
|
|
137
|
+
const validated = validateOptions(options);
|
|
138
|
+
const moduleName = validated.name.replaceAll("-", "_");
|
|
139
|
+
const rows = validated.tasks.map((task, idx) => taskRow(task, idx));
|
|
140
|
+
const runnerFiles = validated.runner.files ?? {};
|
|
141
|
+
const scoringFiles = validated.scoring.kind === "command" ? validated.scoring.files ?? {} : {};
|
|
142
|
+
const files = {
|
|
143
|
+
"pyproject.toml": renderPyproject(validated, moduleName),
|
|
144
|
+
"prime.eval.toml": renderPrimeConfig(validated, "eval"),
|
|
145
|
+
"prime.train.toml": renderPrimeConfig(validated, "train"),
|
|
146
|
+
"README.md": renderReadme(validated),
|
|
147
|
+
[`${moduleName}/__init__.py`]: renderInit(moduleName),
|
|
148
|
+
[`${moduleName}/taskset.py`]: renderTaskset(moduleName, validated.scoring),
|
|
149
|
+
[`${moduleName}/harness.py`]: renderHarness(moduleName),
|
|
150
|
+
[`${moduleName}/tasks.jsonl`]: `${rows.map((row) => JSON.stringify(row)).join("\n")}\n`,
|
|
151
|
+
[`${moduleName}/runner.json`]: `${JSON.stringify({
|
|
152
|
+
command: validated.runner.command,
|
|
153
|
+
files: runnerFiles,
|
|
154
|
+
setup: validated.runner.setup ?? [],
|
|
155
|
+
forwardEnv: validated.runner.forwardEnv ?? []
|
|
156
|
+
}, null, 2)}\n`
|
|
157
|
+
};
|
|
158
|
+
for (const [path, contents] of Object.entries(scoringFiles)) files[`${moduleName}/scoring/${path}`] = contents;
|
|
159
|
+
const filesSha256 = Object.fromEntries(Object.entries(files).sort(([left], [right]) => left.localeCompare(right)).map(([path, contents]) => [path, sha256(contents)]));
|
|
160
|
+
const manifest = {
|
|
161
|
+
kind: "tangle.primeintellect.package",
|
|
162
|
+
name: validated.name,
|
|
163
|
+
moduleName,
|
|
164
|
+
version: validated.version,
|
|
165
|
+
verifiers: VERIFIERS_RANGE,
|
|
166
|
+
taskCount: validated.tasks.length,
|
|
167
|
+
splits: {
|
|
168
|
+
train: validated.tasks.filter((task) => task.split === "train").length,
|
|
169
|
+
eval: validated.tasks.filter((task) => task.split === "eval").length
|
|
170
|
+
},
|
|
171
|
+
taskIdsSha256: sha256(validated.tasks.map((task) => `${task.split}:${task.id}`).sort().join("\n")),
|
|
172
|
+
filesSha256
|
|
173
|
+
};
|
|
174
|
+
files["manifest.json"] = `${JSON.stringify(manifest, null, 2)}\n`;
|
|
175
|
+
return {
|
|
176
|
+
manifest,
|
|
177
|
+
files: Object.freeze(files)
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
/** Write a bundle through a sibling temporary directory, then rename it into place. */
|
|
203
181
|
async function writePrimeIntellectPackage(bundle, outputDirectory, options = {}) {
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
await rm(temporary, { recursive: true, force: true });
|
|
244
|
-
throw error;
|
|
245
|
-
}
|
|
246
|
-
return output;
|
|
182
|
+
const output = resolve(outputDirectory);
|
|
183
|
+
const parent = dirname(output);
|
|
184
|
+
await mkdir(parent, { recursive: true });
|
|
185
|
+
const replacing = await pathExists(output);
|
|
186
|
+
if (replacing) {
|
|
187
|
+
if (!options.replace) throw new Error(`PrimeIntellect output already exists: ${output}`);
|
|
188
|
+
await assertGeneratedPackage(output);
|
|
189
|
+
}
|
|
190
|
+
const temporary = join(parent, `.${basename(output)}.${randomUUID()}.tmp`);
|
|
191
|
+
const backup = replacing ? join(parent, `.${basename(output)}.${randomUUID()}.backup`) : void 0;
|
|
192
|
+
try {
|
|
193
|
+
await mkdir(temporary);
|
|
194
|
+
for (const [path, contents] of Object.entries(bundle.files)) {
|
|
195
|
+
assertRelativePath(path, "bundle file");
|
|
196
|
+
const target = resolve(temporary, path);
|
|
197
|
+
if (target !== temporary && !target.startsWith(`${temporary}${sep}`)) throw new Error(`bundle file escapes output directory: ${path}`);
|
|
198
|
+
await mkdir(dirname(target), { recursive: true });
|
|
199
|
+
await writeFile(target, contents, "utf8");
|
|
200
|
+
}
|
|
201
|
+
if (backup) await rename(output, backup);
|
|
202
|
+
try {
|
|
203
|
+
await rename(temporary, output);
|
|
204
|
+
} catch (error) {
|
|
205
|
+
if (backup) try {
|
|
206
|
+
await rename(backup, output);
|
|
207
|
+
} catch (restoreError) {
|
|
208
|
+
throw new AggregateError([error, restoreError], `failed to install PrimeIntellect package and restore ${output}`);
|
|
209
|
+
}
|
|
210
|
+
throw error;
|
|
211
|
+
}
|
|
212
|
+
if (backup) await rm(backup, { recursive: true });
|
|
213
|
+
} catch (error) {
|
|
214
|
+
await rm(temporary, {
|
|
215
|
+
recursive: true,
|
|
216
|
+
force: true
|
|
217
|
+
});
|
|
218
|
+
throw error;
|
|
219
|
+
}
|
|
220
|
+
return output;
|
|
247
221
|
}
|
|
248
222
|
function validateOptions(options) {
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
if (/(^|:)latest$/i.test(options.runner.image)) {
|
|
294
|
-
throw new Error("runner.image must not use the mutable latest tag");
|
|
295
|
-
}
|
|
296
|
-
positiveInteger(options.maxTurns ?? DEFAULT_MAX_TURNS, "maxTurns");
|
|
297
|
-
for (const [name, value] of [
|
|
298
|
-
["maxInputTokens", options.maxInputTokens],
|
|
299
|
-
["maxOutputTokens", options.maxOutputTokens],
|
|
300
|
-
["maxTotalTokens", options.maxTotalTokens]
|
|
301
|
-
]) {
|
|
302
|
-
if (value !== void 0) positiveInteger(value, name);
|
|
303
|
-
}
|
|
304
|
-
positiveNumber(options.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT, "rolloutTimeoutSeconds");
|
|
305
|
-
positiveNumber(options.scoringTimeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, "scoringTimeoutSeconds");
|
|
306
|
-
return options;
|
|
223
|
+
if (!PACKAGE_NAME.test(options.name)) throw new Error("PrimeIntellect package name must match /^[a-z][a-z0-9-]{0,62}$/");
|
|
224
|
+
if (!VERSION.test(options.version)) throw new Error("PrimeIntellect package version must be a numeric semantic version");
|
|
225
|
+
if (!Array.isArray(options.tasks) || options.tasks.length === 0) throw new Error("PrimeIntellect package requires tasks");
|
|
226
|
+
const tasks = options.tasks;
|
|
227
|
+
const seen = /* @__PURE__ */ new Set();
|
|
228
|
+
const seenInputs = /* @__PURE__ */ new Map();
|
|
229
|
+
const splitCounts = {
|
|
230
|
+
train: 0,
|
|
231
|
+
eval: 0
|
|
232
|
+
};
|
|
233
|
+
for (const [index, task] of tasks.entries()) {
|
|
234
|
+
validateTask(task, index, options.scoring);
|
|
235
|
+
if (seen.has(task.id)) throw new Error(`duplicate PrimeIntellect task id: ${task.id}`);
|
|
236
|
+
seen.add(task.id);
|
|
237
|
+
const input = canonicalJson({
|
|
238
|
+
prompt: task.prompt,
|
|
239
|
+
systemPrompt: task.systemPrompt ?? null,
|
|
240
|
+
metadata: task.metadata ?? {}
|
|
241
|
+
});
|
|
242
|
+
const duplicate = seenInputs.get(input);
|
|
243
|
+
if (duplicate) throw new Error(`PrimeIntellect tasks ${duplicate.id} (${duplicate.split}) and ${task.id} (${task.split}) expose the same public input`);
|
|
244
|
+
seenInputs.set(input, {
|
|
245
|
+
id: task.id,
|
|
246
|
+
split: task.split
|
|
247
|
+
});
|
|
248
|
+
splitCounts[task.split] += 1;
|
|
249
|
+
}
|
|
250
|
+
if (splitCounts.train === 0 || splitCounts.eval === 0) throw new Error("PrimeIntellect package requires non-empty, disjoint train and eval splits");
|
|
251
|
+
validateScoring(options.scoring);
|
|
252
|
+
validateCommand(options.runner.command, "runner.command");
|
|
253
|
+
validateFiles(options.runner.files ?? {}, "runner.files");
|
|
254
|
+
for (const [index, command] of (options.runner.setup ?? []).entries()) validateCommand(command, `runner.setup[${index}]`);
|
|
255
|
+
validateEnvNames(options.runner.forwardEnv ?? [], "runner.forwardEnv");
|
|
256
|
+
if (typeof options.runner.image !== "string" || options.runner.image.trim().length === 0) throw new Error("runner.image must be a non-empty container image");
|
|
257
|
+
if (/(^|:)latest$/i.test(options.runner.image)) throw new Error("runner.image must not use the mutable latest tag");
|
|
258
|
+
positiveInteger(options.maxTurns ?? DEFAULT_MAX_TURNS, "maxTurns");
|
|
259
|
+
for (const [name, value] of [
|
|
260
|
+
["maxInputTokens", options.maxInputTokens],
|
|
261
|
+
["maxOutputTokens", options.maxOutputTokens],
|
|
262
|
+
["maxTotalTokens", options.maxTotalTokens]
|
|
263
|
+
]) if (value !== void 0) positiveInteger(value, name);
|
|
264
|
+
positiveNumber(options.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT, "rolloutTimeoutSeconds");
|
|
265
|
+
positiveNumber(options.scoringTimeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, "scoringTimeoutSeconds");
|
|
266
|
+
return options;
|
|
307
267
|
}
|
|
308
268
|
function validateTask(task, index, scoring) {
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
if (task.systemPrompt !== void 0 && typeof task.systemPrompt !== "string") {
|
|
323
|
-
throw new Error(`${path}.systemPrompt must be a string`);
|
|
324
|
-
}
|
|
325
|
-
if (task.metadata !== void 0) {
|
|
326
|
-
validatePrimeIntellectJson(task.metadata, `${path}.metadata`);
|
|
327
|
-
}
|
|
328
|
-
if (scoring.kind !== "command") {
|
|
329
|
-
const answers = Array.isArray(task.answer) ? task.answer : [task.answer];
|
|
330
|
-
if (answers.length === 0 || answers.some((answer) => typeof answer !== "string" || answer.length === 0)) {
|
|
331
|
-
throw new Error(`${path}.answer is required for ${scoring.kind} scoring`);
|
|
332
|
-
}
|
|
333
|
-
}
|
|
269
|
+
const path = `tasks[${index}]`;
|
|
270
|
+
if (typeof task.id !== "string" || task.id.trim().length === 0) throw new Error(`${path}.id must be a non-empty string`);
|
|
271
|
+
if (task.split !== "train" && task.split !== "eval") throw new Error(`${path}.split must be train or eval`);
|
|
272
|
+
const prompt = validatePrimeIntellectPrompt(task.prompt, `${path}.prompt`);
|
|
273
|
+
if (Array.isArray(prompt)) {
|
|
274
|
+
if (task.systemPrompt !== void 0 && prompt.some((message) => message.role === "system")) throw new Error(`${path} must not set systemPrompt and include a system message`);
|
|
275
|
+
}
|
|
276
|
+
if (task.systemPrompt !== void 0 && typeof task.systemPrompt !== "string") throw new Error(`${path}.systemPrompt must be a string`);
|
|
277
|
+
if (task.metadata !== void 0) validatePrimeIntellectJson(task.metadata, `${path}.metadata`);
|
|
278
|
+
if (scoring.kind !== "command") {
|
|
279
|
+
const answers = Array.isArray(task.answer) ? task.answer : [task.answer];
|
|
280
|
+
if (answers.length === 0 || answers.some((answer) => typeof answer !== "string" || answer.length === 0)) throw new Error(`${path}.answer is required for ${scoring.kind} scoring`);
|
|
281
|
+
}
|
|
334
282
|
}
|
|
335
283
|
function validateScoring(scoring) {
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
validateEnvNames(scoring.forwardEnv ?? [], "scoring.forwardEnv");
|
|
346
|
-
positiveNumber(scoring.timeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, "scoring.timeoutSeconds");
|
|
284
|
+
if (scoring.kind === "exact") return;
|
|
285
|
+
if (scoring.kind === "reference-judge") {
|
|
286
|
+
if (typeof scoring.model !== "string" || scoring.model.length === 0) throw new Error("reference-judge scoring requires a model");
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
validateCommand(scoring.command, "scoring.command");
|
|
290
|
+
validateFiles(scoring.files ?? {}, "scoring.files");
|
|
291
|
+
validateEnvNames(scoring.forwardEnv ?? [], "scoring.forwardEnv");
|
|
292
|
+
positiveNumber(scoring.timeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, "scoring.timeoutSeconds");
|
|
347
293
|
}
|
|
348
294
|
function validateCommand(command, path) {
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
}
|
|
352
|
-
for (const [index, argument] of command.entries()) {
|
|
353
|
-
if (typeof argument !== "string" || argument.length === 0 || argument.includes("\0")) {
|
|
354
|
-
throw new Error(`${path}[${index}] must be a non-empty string without NUL bytes`);
|
|
355
|
-
}
|
|
356
|
-
}
|
|
295
|
+
if (!Array.isArray(command) || command.length === 0) throw new Error(`${path} must be a non-empty argv array`);
|
|
296
|
+
for (const [index, argument] of command.entries()) if (typeof argument !== "string" || argument.length === 0 || argument.includes("\0")) throw new Error(`${path}[${index}] must be a non-empty string without NUL bytes`);
|
|
357
297
|
}
|
|
358
298
|
function validateFiles(files, path) {
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
299
|
+
for (const [file, contents] of Object.entries(files)) {
|
|
300
|
+
assertRelativePath(file, path);
|
|
301
|
+
if (typeof contents !== "string") throw new Error(`${path}.${file} must be a string`);
|
|
302
|
+
}
|
|
363
303
|
}
|
|
364
304
|
function validateEnvNames(names, path) {
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
305
|
+
const seen = /* @__PURE__ */ new Set();
|
|
306
|
+
for (const [index, name] of names.entries()) {
|
|
307
|
+
if (!ENV_NAME.test(name)) throw new Error(`${path}[${index}] is not a valid environment name`);
|
|
308
|
+
if (seen.has(name)) throw new Error(`${path} contains duplicate name ${name}`);
|
|
309
|
+
seen.add(name);
|
|
310
|
+
}
|
|
371
311
|
}
|
|
372
312
|
function assertRelativePath(path, label) {
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
throw new Error(`${label} contains unsafe path: ${path}`);
|
|
376
|
-
}
|
|
313
|
+
const normalized = normalize(path);
|
|
314
|
+
if (path.length === 0 || path.includes("\0") || isAbsolute(path) || normalized === ".." || normalized.startsWith(`..${sep}`) || relative(".", normalized).startsWith("..")) throw new Error(`${label} contains unsafe path: ${path}`);
|
|
377
315
|
}
|
|
378
316
|
function taskRow(task, idx) {
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
317
|
+
return {
|
|
318
|
+
idx,
|
|
319
|
+
name: task.id,
|
|
320
|
+
prompt: task.prompt,
|
|
321
|
+
system_prompt: task.systemPrompt ?? null,
|
|
322
|
+
split: task.split,
|
|
323
|
+
answer: task.answer ?? null,
|
|
324
|
+
metadata: task.metadata ?? {}
|
|
325
|
+
};
|
|
388
326
|
}
|
|
389
327
|
function renderPyproject(options, moduleName) {
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
name = ${toml(options.name)}
|
|
393
|
-
version = ${toml(options.version)}
|
|
394
|
-
description = ${toml(description)}
|
|
395
|
-
requires-python = ">=3.11,<3.14"
|
|
396
|
-
dependencies = ["verifiers${VERIFIERS_RANGE}"]
|
|
397
|
-
|
|
398
|
-
[build-system]
|
|
399
|
-
requires = ["hatchling"]
|
|
400
|
-
build-backend = "hatchling.build"
|
|
401
|
-
|
|
402
|
-
[tool.hatch.build.targets.wheel]
|
|
403
|
-
packages = [${toml(moduleName)}]
|
|
404
|
-
|
|
405
|
-
[tool.uv]
|
|
406
|
-
prerelease = "allow"
|
|
407
|
-
`;
|
|
328
|
+
const description = options.description ?? `PrimeIntellect tasks for ${options.name}`;
|
|
329
|
+
return `[project]\nname = ${toml(options.name)}\nversion = ${toml(options.version)}\ndescription = ${toml(description)}\nrequires-python = ">=3.11,<3.14"\ndependencies = ["verifiers${VERIFIERS_RANGE}"]\n\n[build-system]\nrequires = ["hatchling"]\nbuild-backend = "hatchling.build"\n\n[tool.hatch.build.targets.wheel]\npackages = [${toml(moduleName)}]\n\n[tool.uv]\nprerelease = "allow"\n`;
|
|
408
330
|
}
|
|
409
331
|
function renderPrimeConfig(options, split) {
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
return `${limits.join("\n")}
|
|
417
|
-
push = false
|
|
418
|
-
|
|
419
|
-
[timeout]
|
|
420
|
-
rollout = ${options.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT}
|
|
421
|
-
scoring = ${options.scoringTimeoutSeconds ?? DEFAULT_SCORING_TIMEOUT}
|
|
422
|
-
|
|
423
|
-
[taskset]
|
|
424
|
-
id = ${toml(options.name)}
|
|
425
|
-
split = ${toml(split)}
|
|
426
|
-
|
|
427
|
-
[harness]
|
|
428
|
-
id = ${toml(options.name)}
|
|
429
|
-
program = ${tomlArray(options.runner.command)}
|
|
430
|
-
forward_env = ${tomlArray(options.runner.forwardEnv ?? [])}
|
|
431
|
-
|
|
432
|
-
[harness.runtime]
|
|
433
|
-
type = "docker"
|
|
434
|
-
image = ${toml(options.runner.image)}
|
|
435
|
-
`;
|
|
332
|
+
return `${[
|
|
333
|
+
`max_turns = ${options.maxTurns ?? DEFAULT_MAX_TURNS}`,
|
|
334
|
+
options.maxInputTokens === void 0 ? void 0 : `max_input_tokens = ${options.maxInputTokens}`,
|
|
335
|
+
options.maxOutputTokens === void 0 ? void 0 : `max_output_tokens = ${options.maxOutputTokens}`,
|
|
336
|
+
options.maxTotalTokens === void 0 ? void 0 : `max_total_tokens = ${options.maxTotalTokens}`
|
|
337
|
+
].filter((line) => line !== void 0).join("\n")}\npush = false\n\n[timeout]\nrollout = ${options.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT}\nscoring = ${options.scoringTimeoutSeconds ?? DEFAULT_SCORING_TIMEOUT}\n\n[taskset]\nid = ${toml(options.name)}\nsplit = ${toml(split)}\n\n[harness]\nid = ${toml(options.name)}\nprogram = ${tomlArray(options.runner.command)}\nforward_env = ${tomlArray(options.runner.forwardEnv ?? [])}\n\n[harness.runtime]\ntype = "docker"\nimage = ${toml(options.runner.image)}\n`;
|
|
436
338
|
}
|
|
437
339
|
function renderInit(moduleName) {
|
|
438
|
-
|
|
439
|
-
from ${moduleName}.taskset import TangleTaskset
|
|
440
|
-
|
|
441
|
-
__all__ = ["TangleRuntimeHarness", "TangleTaskset"]
|
|
442
|
-
`;
|
|
340
|
+
return `from ${moduleName}.harness import TangleRuntimeHarness\nfrom ${moduleName}.taskset import TangleTaskset\n\n__all__ = ["TangleRuntimeHarness", "TangleTaskset"]\n`;
|
|
443
341
|
}
|
|
444
342
|
function renderTaskset(moduleName, scoring) {
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
import json
|
|
448
|
-
import math
|
|
449
|
-
import os
|
|
450
|
-
from importlib.resources import files
|
|
451
|
-
from pathlib import Path
|
|
452
|
-
from typing import Any, Literal
|
|
453
|
-
|
|
454
|
-
import verifiers.v1 as vf
|
|
455
|
-
|
|
456
|
-
SCORING = json.loads(${pythonString(JSON.stringify(config))})
|
|
457
|
-
PACKAGE_ROOT = Path(__file__).resolve().parent
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
class TangleTaskData(vf.TaskData):
|
|
461
|
-
split: Literal["train", "eval"]
|
|
462
|
-
answer: str | list[str] | None = None
|
|
463
|
-
metadata: dict[str, Any] = {}
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
class TangleTaskConfig(vf.TaskConfig):
|
|
467
|
-
scoring: Literal["exact", "reference-judge", "command"] = SCORING["kind"]
|
|
468
|
-
normalization: Literal["none", "trim", "trim-casefold"] = SCORING.get("normalization", "trim")
|
|
469
|
-
judge_model: str = SCORING.get("model", "openai/gpt-5.4-nano")
|
|
470
|
-
judge_prompt: str | None = SCORING.get("prompt")
|
|
471
|
-
judge_view: Literal["last_reply", "full_trace"] = SCORING.get("view", "last_reply")
|
|
472
|
-
score_program: list[str] = SCORING.get("command", [])
|
|
473
|
-
score_forward_env: list[str] = SCORING.get("forwardEnv", [])
|
|
474
|
-
score_timeout_seconds: float = SCORING.get("timeoutSeconds", 300)
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
def _normalize(value: str, mode: str) -> str:
|
|
478
|
-
if mode == "none":
|
|
479
|
-
return value
|
|
480
|
-
value = value.strip()
|
|
481
|
-
return value.casefold() if mode == "trim-casefold" else value
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
async def _run_score_command(config: TangleTaskConfig, data: TangleTaskData, trace: vf.Trace) -> float:
|
|
485
|
-
if not config.score_program:
|
|
486
|
-
raise ValueError("command scoring requires score_program")
|
|
487
|
-
safe_env = {
|
|
488
|
-
key: os.environ[key]
|
|
489
|
-
for key in ("PATH", "HOME", "TMPDIR", "LANG")
|
|
490
|
-
if key in os.environ
|
|
491
|
-
}
|
|
492
|
-
safe_env.update(
|
|
493
|
-
{key: os.environ[key] for key in config.score_forward_env if key in os.environ}
|
|
494
|
-
)
|
|
495
|
-
request = {
|
|
496
|
-
"kind": "tangle.primeintellect.score",
|
|
497
|
-
"task": data.model_dump(mode="json", exclude_none=True),
|
|
498
|
-
"trace": trace.model_dump(mode="json", exclude_none=True),
|
|
499
|
-
}
|
|
500
|
-
process = await asyncio.create_subprocess_exec(
|
|
501
|
-
*config.score_program,
|
|
502
|
-
cwd=PACKAGE_ROOT,
|
|
503
|
-
env=safe_env,
|
|
504
|
-
stdin=asyncio.subprocess.PIPE,
|
|
505
|
-
stdout=asyncio.subprocess.PIPE,
|
|
506
|
-
stderr=asyncio.subprocess.PIPE,
|
|
507
|
-
)
|
|
508
|
-
payload = json.dumps(request, separators=(",", ":")).encode()
|
|
509
|
-
try:
|
|
510
|
-
stdout, stderr = await asyncio.wait_for(
|
|
511
|
-
process.communicate(payload), timeout=config.score_timeout_seconds
|
|
512
|
-
)
|
|
513
|
-
except TimeoutError:
|
|
514
|
-
process.kill()
|
|
515
|
-
await process.communicate()
|
|
516
|
-
raise RuntimeError(
|
|
517
|
-
f"score command timed out after {config.score_timeout_seconds}s"
|
|
518
|
-
)
|
|
519
|
-
if process.returncode != 0:
|
|
520
|
-
detail = (stderr or stdout).decode(errors="replace").strip()[-2000:]
|
|
521
|
-
raise RuntimeError(f"score command exited {process.returncode}: {detail}")
|
|
522
|
-
try:
|
|
523
|
-
result = json.loads(stdout)
|
|
524
|
-
except json.JSONDecodeError as error:
|
|
525
|
-
raise ValueError(f"score command returned invalid JSON: {error}") from error
|
|
526
|
-
if not isinstance(result, dict):
|
|
527
|
-
raise ValueError("score command must return an object")
|
|
528
|
-
reward = result.get("reward")
|
|
529
|
-
if isinstance(reward, bool) or not isinstance(reward, (int, float)) or not math.isfinite(reward):
|
|
530
|
-
raise ValueError("score command reward must be a finite number")
|
|
531
|
-
metrics = result.get("metrics", {})
|
|
532
|
-
if not isinstance(metrics, dict):
|
|
533
|
-
raise ValueError("score command metrics must be an object")
|
|
534
|
-
for name, value in metrics.items():
|
|
535
|
-
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
|
536
|
-
raise ValueError(f"score command metric {name!r} must be a finite number")
|
|
537
|
-
trace.record_metrics(metrics)
|
|
538
|
-
return float(reward)
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
class TangleTask(vf.Task[TangleTaskData, vf.State, TangleTaskConfig]):
|
|
542
|
-
@vf.reward(weight=1.0)
|
|
543
|
-
async def task_reward(self, trace: vf.Trace) -> float:
|
|
544
|
-
if self.config.scoring == "command":
|
|
545
|
-
return await _run_score_command(self.config, self.data, trace)
|
|
546
|
-
if self.data.answer is None:
|
|
547
|
-
raise ValueError(f"task {self.data.name!r} has no reference answer")
|
|
548
|
-
if self.config.scoring == "reference-judge":
|
|
549
|
-
judge = vf.ReferenceJudge(
|
|
550
|
-
vf.ReferenceJudgeConfig(
|
|
551
|
-
model=self.config.judge_model,
|
|
552
|
-
prompt=self.config.judge_prompt,
|
|
553
|
-
view=self.config.judge_view,
|
|
554
|
-
)
|
|
555
|
-
)
|
|
556
|
-
return await judge.score(self.data, trace)
|
|
557
|
-
expected = self.data.answer if isinstance(self.data.answer, list) else [self.data.answer]
|
|
558
|
-
actual = _normalize(trace.last_reply, self.config.normalization)
|
|
559
|
-
return float(any(actual == _normalize(answer, self.config.normalization) for answer in expected))
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
class TangleTasksetConfig(vf.TasksetConfig):
|
|
563
|
-
split: Literal["train", "eval"] = "eval"
|
|
564
|
-
task: TangleTaskConfig = TangleTaskConfig()
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
class TangleTaskset(vf.Taskset[TangleTask, TangleTasksetConfig]):
|
|
568
|
-
def load(self) -> list[TangleTask]:
|
|
569
|
-
resource = files(${pythonString(moduleName)}).joinpath("tasks.jsonl")
|
|
570
|
-
tasks: list[TangleTask] = []
|
|
571
|
-
with resource.open("r", encoding="utf-8") as handle:
|
|
572
|
-
for line in handle:
|
|
573
|
-
if not line.strip():
|
|
574
|
-
continue
|
|
575
|
-
row = json.loads(line)
|
|
576
|
-
if row["split"] != self.config.split:
|
|
577
|
-
continue
|
|
578
|
-
tasks.append(TangleTask(TangleTaskData.model_validate(row), self.config.task))
|
|
579
|
-
if not tasks:
|
|
580
|
-
raise ValueError(f"task split {self.config.split!r} is empty")
|
|
581
|
-
return tasks
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
__all__ = ["TangleTaskset"]
|
|
585
|
-
`;
|
|
343
|
+
const config = scoringConfig(scoring);
|
|
344
|
+
return `import asyncio\nimport json\nimport math\nimport os\nfrom importlib.resources import files\nfrom pathlib import Path\nfrom typing import Any, Literal\n\nimport verifiers.v1 as vf\n\nSCORING = json.loads(${pythonString(JSON.stringify(config))})\nPACKAGE_ROOT = Path(__file__).resolve().parent\n\n\nclass TangleTaskData(vf.TaskData):\n split: Literal["train", "eval"]\n answer: str | list[str] | None = None\n metadata: dict[str, Any] = {}\n\n\nclass TangleTaskConfig(vf.TaskConfig):\n scoring: Literal["exact", "reference-judge", "command"] = SCORING["kind"]\n normalization: Literal["none", "trim", "trim-casefold"] = SCORING.get("normalization", "trim")\n judge_model: str = SCORING.get("model", "openai/gpt-5.4-nano")\n judge_prompt: str | None = SCORING.get("prompt")\n judge_view: Literal["last_reply", "full_trace"] = SCORING.get("view", "last_reply")\n score_program: list[str] = SCORING.get("command", [])\n score_forward_env: list[str] = SCORING.get("forwardEnv", [])\n score_timeout_seconds: float = SCORING.get("timeoutSeconds", 300)\n\n\ndef _normalize(value: str, mode: str) -> str:\n if mode == "none":\n return value\n value = value.strip()\n return value.casefold() if mode == "trim-casefold" else value\n\n\nasync def _run_score_command(config: TangleTaskConfig, data: TangleTaskData, trace: vf.Trace) -> float:\n if not config.score_program:\n raise ValueError("command scoring requires score_program")\n safe_env = {\n key: os.environ[key]\n for key in ("PATH", "HOME", "TMPDIR", "LANG")\n if key in os.environ\n }\n safe_env.update(\n {key: os.environ[key] for key in config.score_forward_env if key in os.environ}\n )\n request = {\n "kind": "tangle.primeintellect.score",\n "task": data.model_dump(mode="json", exclude_none=True),\n "trace": trace.model_dump(mode="json", exclude_none=True),\n }\n process = await asyncio.create_subprocess_exec(\n *config.score_program,\n cwd=PACKAGE_ROOT,\n env=safe_env,\n stdin=asyncio.subprocess.PIPE,\n stdout=asyncio.subprocess.PIPE,\n stderr=asyncio.subprocess.PIPE,\n )\n payload = json.dumps(request, separators=(",", ":")).encode()\n try:\n stdout, stderr = await asyncio.wait_for(\n process.communicate(payload), timeout=config.score_timeout_seconds\n )\n except TimeoutError:\n process.kill()\n await process.communicate()\n raise RuntimeError(\n f"score command timed out after {config.score_timeout_seconds}s"\n )\n if process.returncode != 0:\n detail = (stderr or stdout).decode(errors="replace").strip()[-2000:]\n raise RuntimeError(f"score command exited {process.returncode}: {detail}")\n try:\n result = json.loads(stdout)\n except json.JSONDecodeError as error:\n raise ValueError(f"score command returned invalid JSON: {error}") from error\n if not isinstance(result, dict):\n raise ValueError("score command must return an object")\n reward = result.get("reward")\n if isinstance(reward, bool) or not isinstance(reward, (int, float)) or not math.isfinite(reward):\n raise ValueError("score command reward must be a finite number")\n metrics = result.get("metrics", {})\n if not isinstance(metrics, dict):\n raise ValueError("score command metrics must be an object")\n for name, value in metrics.items():\n if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):\n raise ValueError(f"score command metric {name!r} must be a finite number")\n trace.record_metrics(metrics)\n return float(reward)\n\n\nclass TangleTask(vf.Task[TangleTaskData, vf.State, TangleTaskConfig]):\n @vf.reward(weight=1.0)\n async def task_reward(self, trace: vf.Trace) -> float:\n if self.config.scoring == "command":\n return await _run_score_command(self.config, self.data, trace)\n if self.data.answer is None:\n raise ValueError(f"task {self.data.name!r} has no reference answer")\n if self.config.scoring == "reference-judge":\n judge = vf.ReferenceJudge(\n vf.ReferenceJudgeConfig(\n model=self.config.judge_model,\n prompt=self.config.judge_prompt,\n view=self.config.judge_view,\n )\n )\n return await judge.score(self.data, trace)\n expected = self.data.answer if isinstance(self.data.answer, list) else [self.data.answer]\n actual = _normalize(trace.last_reply, self.config.normalization)\n return float(any(actual == _normalize(answer, self.config.normalization) for answer in expected))\n\n\nclass TangleTasksetConfig(vf.TasksetConfig):\n split: Literal["train", "eval"] = "eval"\n task: TangleTaskConfig = TangleTaskConfig()\n\n\nclass TangleTaskset(vf.Taskset[TangleTask, TangleTasksetConfig]):\n def load(self) -> list[TangleTask]:\n resource = files(${pythonString(moduleName)}).joinpath("tasks.jsonl")\n tasks: list[TangleTask] = []\n with resource.open("r", encoding="utf-8") as handle:\n for line in handle:\n if not line.strip():\n continue\n row = json.loads(line)\n if row["split"] != self.config.split:\n continue\n tasks.append(TangleTask(TangleTaskData.model_validate(row), self.config.task))\n if not tasks:\n raise ValueError(f"task split {self.config.split!r} is empty")\n return tasks\n\n\n__all__ = ["TangleTaskset"]\n`;
|
|
586
345
|
}
|
|
587
346
|
function renderHarness(moduleName) {
|
|
588
|
-
|
|
589
|
-
from importlib.resources import files
|
|
590
|
-
|
|
591
|
-
import verifiers.v1 as vf
|
|
592
|
-
|
|
593
|
-
RUNNER = json.loads(files(${pythonString(moduleName)}).joinpath("runner.json").read_text(encoding="utf-8"))
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
class TangleRuntimeHarnessConfig(vf.HarnessConfig):
|
|
597
|
-
program: list[str] = RUNNER["command"]
|
|
598
|
-
setup_commands: list[list[str]] = RUNNER["setup"]
|
|
599
|
-
forward_env: list[str] = RUNNER["forwardEnv"]
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
class TangleRuntimeHarness(vf.Harness[TangleRuntimeHarnessConfig]):
|
|
603
|
-
APPENDS_SYSTEM_PROMPT = True
|
|
604
|
-
SUPPORTS_MCP = True
|
|
605
|
-
SUPPORTS_MESSAGE_PROMPT = True
|
|
606
|
-
|
|
607
|
-
async def setup(self, runtime: vf.Runtime) -> None:
|
|
608
|
-
for path, contents in RUNNER["files"].items():
|
|
609
|
-
await runtime.write(path, contents.encode())
|
|
610
|
-
for command in self.config.setup_commands:
|
|
611
|
-
result = await runtime.run(command, self.config.resolved_env)
|
|
612
|
-
if result.exit_code != 0:
|
|
613
|
-
detail = (result.stderr or result.stdout).strip()[-2000:]
|
|
614
|
-
raise RuntimeError(
|
|
615
|
-
f"runner setup command {command[0]!r} exited {result.exit_code}: {detail}"
|
|
616
|
-
)
|
|
617
|
-
|
|
618
|
-
async def launch(
|
|
619
|
-
self,
|
|
620
|
-
ctx: vf.ModelContext,
|
|
621
|
-
trace: vf.Trace,
|
|
622
|
-
runtime: vf.Runtime,
|
|
623
|
-
endpoint: str,
|
|
624
|
-
secret: str,
|
|
625
|
-
mcp_urls: dict[str, str],
|
|
626
|
-
) -> vf.ProgramResult:
|
|
627
|
-
data = trace.task.data
|
|
628
|
-
public_task = {
|
|
629
|
-
"id": data.name or str(data.idx),
|
|
630
|
-
"split": data.split,
|
|
631
|
-
"prompt": data.prompt,
|
|
632
|
-
"metadata": data.metadata,
|
|
633
|
-
}
|
|
634
|
-
if data.system_prompt is not None:
|
|
635
|
-
public_task["systemPrompt"] = data.system_prompt
|
|
636
|
-
env = {
|
|
637
|
-
**self.config.resolved_env,
|
|
638
|
-
"OPENAI_BASE_URL": endpoint,
|
|
639
|
-
"OPENAI_API_KEY": secret,
|
|
640
|
-
"OPENAI_MODEL": ctx.model,
|
|
641
|
-
"TANGLE_PRIME_TASK_JSON": json.dumps(public_task, separators=(",", ":")),
|
|
642
|
-
"TANGLE_PRIME_MCP_SERVERS_JSON": json.dumps(mcp_urls, separators=(",", ":")),
|
|
643
|
-
}
|
|
644
|
-
if not self.config.program:
|
|
645
|
-
raise ValueError("Tangle runtime harness requires a program argv")
|
|
646
|
-
return await runtime.run_program(self.config.program, env)
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
__all__ = ["TangleRuntimeHarness"]
|
|
650
|
-
`;
|
|
347
|
+
return `import json\nfrom importlib.resources import files\n\nimport verifiers.v1 as vf\n\nRUNNER = json.loads(files(${pythonString(moduleName)}).joinpath("runner.json").read_text(encoding="utf-8"))\n\n\nclass TangleRuntimeHarnessConfig(vf.HarnessConfig):\n program: list[str] = RUNNER["command"]\n setup_commands: list[list[str]] = RUNNER["setup"]\n forward_env: list[str] = RUNNER["forwardEnv"]\n\n\nclass TangleRuntimeHarness(vf.Harness[TangleRuntimeHarnessConfig]):\n APPENDS_SYSTEM_PROMPT = True\n SUPPORTS_MCP = True\n SUPPORTS_MESSAGE_PROMPT = True\n\n async def setup(self, runtime: vf.Runtime) -> None:\n for path, contents in RUNNER["files"].items():\n await runtime.write(path, contents.encode())\n for command in self.config.setup_commands:\n result = await runtime.run(command, self.config.resolved_env)\n if result.exit_code != 0:\n detail = (result.stderr or result.stdout).strip()[-2000:]\n raise RuntimeError(\n f"runner setup command {command[0]!r} exited {result.exit_code}: {detail}"\n )\n\n async def launch(\n self,\n ctx: vf.ModelContext,\n trace: vf.Trace,\n runtime: vf.Runtime,\n endpoint: str,\n secret: str,\n mcp_urls: dict[str, str],\n ) -> vf.ProgramResult:\n data = trace.task.data\n public_task = {\n "id": data.name or str(data.idx),\n "split": data.split,\n "prompt": data.prompt,\n "metadata": data.metadata,\n }\n if data.system_prompt is not None:\n public_task["systemPrompt"] = data.system_prompt\n env = {\n **self.config.resolved_env,\n "OPENAI_BASE_URL": endpoint,\n "OPENAI_API_KEY": secret,\n "OPENAI_MODEL": ctx.model,\n "TANGLE_PRIME_TASK_JSON": json.dumps(public_task, separators=(",", ":")),\n "TANGLE_PRIME_MCP_SERVERS_JSON": json.dumps(mcp_urls, separators=(",", ":")),\n }\n if not self.config.program:\n raise ValueError("Tangle runtime harness requires a program argv")\n return await runtime.run_program(self.config.program, env)\n\n\n__all__ = ["TangleRuntimeHarness"]\n`;
|
|
651
348
|
}
|
|
652
349
|
function scoringConfig(scoring) {
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
};
|
|
350
|
+
if (scoring.kind === "exact") return {
|
|
351
|
+
kind: scoring.kind,
|
|
352
|
+
normalization: scoring.normalization ?? "trim"
|
|
353
|
+
};
|
|
354
|
+
if (scoring.kind === "reference-judge") return {
|
|
355
|
+
kind: scoring.kind,
|
|
356
|
+
model: scoring.model,
|
|
357
|
+
prompt: scoring.prompt ?? null,
|
|
358
|
+
view: scoring.view ?? "last_reply"
|
|
359
|
+
};
|
|
360
|
+
return {
|
|
361
|
+
kind: scoring.kind,
|
|
362
|
+
command: [...scoring.command],
|
|
363
|
+
forwardEnv: [...scoring.forwardEnv ?? []],
|
|
364
|
+
timeoutSeconds: scoring.timeoutSeconds ?? DEFAULT_SCORING_TIMEOUT
|
|
365
|
+
};
|
|
670
366
|
}
|
|
671
367
|
function renderReadme(options) {
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
PrimeIntellect Verifiers tasks that run the caller's Tangle agent program.
|
|
675
|
-
|
|
676
|
-
## Evaluate
|
|
677
|
-
|
|
678
|
-
\`\`\`bash
|
|
679
|
-
uv run eval @ prime.eval.toml --model <provider/model-snapshot>
|
|
680
|
-
\`\`\`
|
|
681
|
-
|
|
682
|
-
## Train
|
|
683
|
-
|
|
684
|
-
Use \`prime.train.toml\` as the environment config for Prime RL. The train and eval rows are disjoint and selected by \`taskset.split\`.
|
|
685
|
-
|
|
686
|
-
The runner receives only the prompt, metadata, intercepted model endpoint, and MCP URLs. Reference answers remain in the task process and are never written into the runner workspace or environment.
|
|
687
|
-
`;
|
|
368
|
+
return `# ${options.name}\n\nPrimeIntellect Verifiers tasks that run the caller's Tangle agent program.\n\n## Evaluate\n\n\`\`\`bash\nuv run eval @ prime.eval.toml --model <provider/model-snapshot>\n\`\`\`\n\n## Train\n\nUse \`prime.train.toml\` as the environment config for Prime RL. The train and eval rows are disjoint and selected by \`taskset.split\`.\n\nThe runner receives only the prompt, metadata, intercepted model endpoint, and MCP URLs. Reference answers remain in the task process and are never written into the runner workspace or environment.\n`;
|
|
688
369
|
}
|
|
689
370
|
function toml(value) {
|
|
690
|
-
|
|
371
|
+
return JSON.stringify(value);
|
|
691
372
|
}
|
|
692
373
|
function tomlArray(values) {
|
|
693
|
-
|
|
374
|
+
return `[${values.map(toml).join(", ")}]`;
|
|
694
375
|
}
|
|
695
376
|
function pythonString(value) {
|
|
696
|
-
|
|
377
|
+
return JSON.stringify(value);
|
|
697
378
|
}
|
|
698
379
|
function positiveInteger(value, path) {
|
|
699
|
-
|
|
700
|
-
throw new Error(`${path} must be a positive integer`);
|
|
380
|
+
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`);
|
|
701
381
|
}
|
|
702
382
|
function positiveNumber(value, path) {
|
|
703
|
-
|
|
383
|
+
if (!Number.isFinite(value) || value <= 0) throw new Error(`${path} must be positive`);
|
|
704
384
|
}
|
|
705
385
|
async function pathExists(path) {
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
386
|
+
try {
|
|
387
|
+
await stat(path);
|
|
388
|
+
return true;
|
|
389
|
+
} catch (error) {
|
|
390
|
+
if (error.code === "ENOENT") return false;
|
|
391
|
+
throw error;
|
|
392
|
+
}
|
|
713
393
|
}
|
|
714
394
|
async function assertGeneratedPackage(output) {
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
}
|
|
723
|
-
if (manifest === null || typeof manifest !== "object" || manifest.kind !== "tangle.primeintellect.package") {
|
|
724
|
-
throw new Error(`refusing to replace ${output}: manifest kind does not match`);
|
|
725
|
-
}
|
|
395
|
+
let manifest;
|
|
396
|
+
try {
|
|
397
|
+
manifest = JSON.parse(await readFile(join(output, "manifest.json"), "utf8"));
|
|
398
|
+
} catch (error) {
|
|
399
|
+
throw new Error(`refusing to replace ${output}: it is not a generated PrimeIntellect package (${error instanceof Error ? error.message : String(error)})`);
|
|
400
|
+
}
|
|
401
|
+
if (manifest === null || typeof manifest !== "object" || manifest.kind !== "tangle.primeintellect.package") throw new Error(`refusing to replace ${output}: manifest kind does not match`);
|
|
726
402
|
}
|
|
727
403
|
function sha256(value) {
|
|
728
|
-
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
404
|
+
return createHash("sha256").update(value, "utf8").digest("hex");
|
|
405
|
+
}
|
|
406
|
+
//#endregion
|
|
407
|
+
//#region src/primeintellect/runner.ts
|
|
408
|
+
const ENV = {
|
|
409
|
+
task: "TANGLE_PRIME_TASK_JSON",
|
|
410
|
+
model: "OPENAI_MODEL",
|
|
411
|
+
baseUrl: "OPENAI_BASE_URL",
|
|
412
|
+
apiKey: "OPENAI_API_KEY",
|
|
413
|
+
mcpServers: "TANGLE_PRIME_MCP_SERVERS_JSON"
|
|
738
414
|
};
|
|
415
|
+
/** Read and validate the private process contract installed by the generated Prime harness. */
|
|
739
416
|
function readPrimeIntellectEpisodeContext(env = process.env) {
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
}
|
|
417
|
+
const rawTask = requiredEnv(env, ENV.task);
|
|
418
|
+
const rawMcp = env[ENV.mcpServers] ?? "{}";
|
|
419
|
+
const parsedTask = parseJson(rawTask, ENV.task);
|
|
420
|
+
const parsedMcp = parseJson(rawMcp, ENV.mcpServers);
|
|
421
|
+
const task = validatePublicTask(parsedTask);
|
|
422
|
+
const mcpServers = validateStringMap(parsedMcp, ENV.mcpServers);
|
|
423
|
+
return {
|
|
424
|
+
task,
|
|
425
|
+
model: {
|
|
426
|
+
name: requiredEnv(env, ENV.model),
|
|
427
|
+
baseUrl: requiredEnv(env, ENV.baseUrl),
|
|
428
|
+
apiKey: requiredEnv(env, ENV.apiKey)
|
|
429
|
+
},
|
|
430
|
+
mcpServers
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
/** Build the existing runtime backend against Prime's intercepted model endpoint. */
|
|
756
434
|
function createPrimeIntellectBackend(context, options = {}) {
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
}
|
|
435
|
+
return createOpenAICompatibleBackend({
|
|
436
|
+
...options,
|
|
437
|
+
apiKey: context.model.apiKey,
|
|
438
|
+
baseUrl: context.model.baseUrl,
|
|
439
|
+
model: context.model.name,
|
|
440
|
+
kind: options.kind ?? "primeintellect"
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
/**
|
|
444
|
+
* Execute the caller's canonical runtime program inside a Prime rollout.
|
|
445
|
+
* The callback may call runPersonified, runAgentic, runAgentRounds, or any product wrapper.
|
|
446
|
+
*/
|
|
765
447
|
async function runPrimeIntellectProgram(run, options = {}) {
|
|
766
|
-
|
|
448
|
+
return run(readPrimeIntellectEpisodeContext(options.env));
|
|
767
449
|
}
|
|
768
450
|
function requiredEnv(env, name) {
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
}
|
|
773
|
-
return value;
|
|
451
|
+
const value = env[name];
|
|
452
|
+
if (typeof value !== "string" || value.length === 0) throw new Error(`PrimeIntellect runner requires ${name}`);
|
|
453
|
+
return value;
|
|
774
454
|
}
|
|
775
455
|
function parseJson(value, name) {
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
);
|
|
782
|
-
}
|
|
456
|
+
try {
|
|
457
|
+
return JSON.parse(value);
|
|
458
|
+
} catch (error) {
|
|
459
|
+
throw new Error(`${name} must contain valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
460
|
+
}
|
|
783
461
|
}
|
|
784
462
|
function validatePublicTask(value) {
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
463
|
+
const task = record(value, ENV.task);
|
|
464
|
+
for (const privateField of [
|
|
465
|
+
"answer",
|
|
466
|
+
"reference",
|
|
467
|
+
"scoring",
|
|
468
|
+
"score"
|
|
469
|
+
]) if (privateField in task) throw new Error(`${ENV.task} exposed private field ${privateField}`);
|
|
470
|
+
for (const field of Object.keys(task)) if (![
|
|
471
|
+
"id",
|
|
472
|
+
"split",
|
|
473
|
+
"prompt",
|
|
474
|
+
"systemPrompt",
|
|
475
|
+
"metadata"
|
|
476
|
+
].includes(field)) throw new Error(`${ENV.task}.${field} is not supported`);
|
|
477
|
+
const id = nonEmptyString$1(task.id, `${ENV.task}.id`);
|
|
478
|
+
const split = validateSplit(task.split, `${ENV.task}.split`);
|
|
479
|
+
const prompt = validatePrimeIntellectPrompt(task.prompt, `${ENV.task}.prompt`);
|
|
480
|
+
const systemPrompt = optionalString(task.systemPrompt, `${ENV.task}.systemPrompt`);
|
|
481
|
+
if (systemPrompt !== void 0 && Array.isArray(prompt) && prompt.some((message) => message.role === "system")) throw new Error(`${ENV.task} must not set systemPrompt and include a system message`);
|
|
482
|
+
const metadata = task.metadata === void 0 ? void 0 : validatePrimeIntellectJsonObject(task.metadata, `${ENV.task}.metadata`);
|
|
483
|
+
return {
|
|
484
|
+
id,
|
|
485
|
+
split,
|
|
486
|
+
prompt,
|
|
487
|
+
...systemPrompt !== void 0 ? { systemPrompt } : {},
|
|
488
|
+
...metadata !== void 0 ? { metadata } : {}
|
|
489
|
+
};
|
|
811
490
|
}
|
|
812
491
|
function validateSplit(value, path) {
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
}
|
|
816
|
-
return value;
|
|
492
|
+
if (value !== "train" && value !== "eval") throw new Error(`${path} must be train or eval`);
|
|
493
|
+
return value;
|
|
817
494
|
}
|
|
818
495
|
function validateStringMap(value, path) {
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
}
|
|
835
|
-
function nonEmptyString2(value, path) {
|
|
836
|
-
if (typeof value !== "string" || value.length === 0) {
|
|
837
|
-
throw new Error(`${path} must be a non-empty string`);
|
|
838
|
-
}
|
|
839
|
-
return value;
|
|
496
|
+
const input = record(value, path);
|
|
497
|
+
const output = {};
|
|
498
|
+
for (const [key, entry] of Object.entries(input)) {
|
|
499
|
+
if (typeof entry !== "string" || entry.length === 0) throw new Error(`${path}.${key} must be a non-empty string`);
|
|
500
|
+
output[key] = entry;
|
|
501
|
+
}
|
|
502
|
+
return output;
|
|
503
|
+
}
|
|
504
|
+
function record(value, path) {
|
|
505
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(`${path} must be an object`);
|
|
506
|
+
return value;
|
|
507
|
+
}
|
|
508
|
+
function nonEmptyString$1(value, path) {
|
|
509
|
+
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
510
|
+
return value;
|
|
840
511
|
}
|
|
841
512
|
function optionalString(value, path) {
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
513
|
+
if (value === void 0) return void 0;
|
|
514
|
+
if (typeof value !== "string") throw new Error(`${path} must be a string`);
|
|
515
|
+
return value;
|
|
845
516
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
517
|
+
//#endregion
|
|
518
|
+
//#region src/primeintellect/traces.ts
|
|
519
|
+
/** Parse Prime's durable `traces.jsonl` and reject malformed rows with a line number. */
|
|
849
520
|
function parsePrimeIntellectTraces(jsonl) {
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
}
|
|
521
|
+
const traces = [];
|
|
522
|
+
for (const [lineIndex, line] of jsonl.split("\n").entries()) {
|
|
523
|
+
if (!line.trim()) continue;
|
|
524
|
+
let parsed;
|
|
525
|
+
try {
|
|
526
|
+
parsed = JSON.parse(line);
|
|
527
|
+
} catch (error) {
|
|
528
|
+
throw new Error(`PrimeIntellect trace line ${lineIndex + 1} is invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
529
|
+
}
|
|
530
|
+
traces.push(validateTrace(parsed, lineIndex + 1));
|
|
531
|
+
}
|
|
532
|
+
if (traces.length === 0) throw new Error("PrimeIntellect traces.jsonl contains no traces");
|
|
533
|
+
return traces;
|
|
534
|
+
}
|
|
535
|
+
/** Convert all Prime traces to agent-eval RunRecords while retaining one shared run config. */
|
|
866
536
|
function importPrimeIntellectTraces(jsonl, defaults) {
|
|
867
|
-
|
|
868
|
-
(trace) => primeIntellectTraceToRunRecord(trace, defaults)
|
|
869
|
-
);
|
|
537
|
+
return parsePrimeIntellectTraces(jsonl).map((trace) => primeIntellectTraceToRunRecord(trace, defaults));
|
|
870
538
|
}
|
|
539
|
+
/** Project one complete Prime trace into the common agent-eval analysis row. */
|
|
871
540
|
function primeIntellectTraceToRunRecord(trace, options) {
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
541
|
+
const split = trace.task.data.split;
|
|
542
|
+
if (split !== "train" && split !== "eval") throw new Error(`PrimeIntellect trace ${trace.id} has no train/eval split`);
|
|
543
|
+
const reward = sumFinite(Object.values(trace.rewards), `trace ${trace.id} rewards`);
|
|
544
|
+
const usage = aggregateUsage([...trace.nodes.map((node) => node.usage).filter((value) => value != null), ...trace.extra_usage ?? []]);
|
|
545
|
+
const errors = trace.errors ?? [];
|
|
546
|
+
const raw = {
|
|
547
|
+
reward,
|
|
548
|
+
"prime.turns": trace.nodes.filter((node) => node.sampled === true).length,
|
|
549
|
+
"prime.branches": countBranches(trace.nodes),
|
|
550
|
+
"prime.errors": errors.length,
|
|
551
|
+
execution_error_count: errors.length,
|
|
552
|
+
"prime.completed": trace.is_completed ? 1 : 0,
|
|
553
|
+
"prime.cost_complete": usage.costComplete ? 1 : 0,
|
|
554
|
+
"prime.reported_cost_usd": usage.reportedCostUsd
|
|
555
|
+
};
|
|
556
|
+
for (const [name, value] of Object.entries(trace.rewards)) raw[`reward.${name}`] = finite(value, `trace ${trace.id} reward ${name}`);
|
|
557
|
+
for (const [name, value] of Object.entries(trace.metrics)) raw[`metric.${name}`] = finite(value, `trace ${trace.id} metric ${name}`);
|
|
558
|
+
return validateRunRecord({
|
|
559
|
+
runId: trace.id,
|
|
560
|
+
experimentId: options.experimentId,
|
|
561
|
+
candidateId: options.candidateId,
|
|
562
|
+
seed: options.seed,
|
|
563
|
+
model: options.model,
|
|
564
|
+
promptHash: options.promptHash,
|
|
565
|
+
configHash: options.configHash,
|
|
566
|
+
commitSha: options.commitSha,
|
|
567
|
+
wallMs: traceWallMs(trace),
|
|
568
|
+
costUsd: usage.costComplete ? usage.reportedCostUsd : null,
|
|
569
|
+
costProvenance: usage.costComplete ? {
|
|
570
|
+
kind: "observed",
|
|
571
|
+
usd: usage.reportedCostUsd
|
|
572
|
+
} : {
|
|
573
|
+
kind: "uncaptured",
|
|
574
|
+
usd: null
|
|
575
|
+
},
|
|
576
|
+
terminalOutcome: errors.length > 0 ? "failed" : trace.is_completed ? "succeeded" : "incomplete",
|
|
577
|
+
...errors[0] ? { terminalFailureReason: `${errors[0].type}:${errors[0].message}` } : {},
|
|
578
|
+
tokenUsage: {
|
|
579
|
+
input: usage.input,
|
|
580
|
+
output: usage.output,
|
|
581
|
+
...usage.reasoning !== void 0 ? { reasoning: usage.reasoning } : {},
|
|
582
|
+
...usage.cached !== void 0 ? { cached: usage.cached } : {}
|
|
583
|
+
},
|
|
584
|
+
outcome: split === "eval" ? {
|
|
585
|
+
holdoutScore: reward,
|
|
586
|
+
raw
|
|
587
|
+
} : {
|
|
588
|
+
searchScore: reward,
|
|
589
|
+
raw
|
|
590
|
+
},
|
|
591
|
+
splitTag: split === "eval" ? "holdout" : "search",
|
|
592
|
+
scenarioId: trace.task.data.name ?? String(trace.task.data.idx)
|
|
593
|
+
});
|
|
921
594
|
}
|
|
922
595
|
function validateTrace(value, line) {
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
validateNumberMap(trace.metrics, `PrimeIntellect trace line ${line}.metrics`);
|
|
953
|
-
if (trace.extra_usage !== void 0 && !Array.isArray(trace.extra_usage)) {
|
|
954
|
-
throw new Error(`PrimeIntellect trace line ${line}.extra_usage must be an array`);
|
|
955
|
-
}
|
|
956
|
-
for (const [index, usage] of (trace.extra_usage ?? []).entries()) {
|
|
957
|
-
validateUsage(usage, `PrimeIntellect trace line ${line}.extra_usage[${index}]`);
|
|
958
|
-
}
|
|
959
|
-
if (trace.errors !== void 0 && !Array.isArray(trace.errors)) {
|
|
960
|
-
throw new Error(`PrimeIntellect trace line ${line}.errors must be an array`);
|
|
961
|
-
}
|
|
962
|
-
for (const [index, rawError] of (trace.errors ?? []).entries()) {
|
|
963
|
-
const error = object(rawError, `PrimeIntellect trace line ${line}.errors[${index}]`);
|
|
964
|
-
nonEmptyString3(error.type, `PrimeIntellect trace line ${line}.errors[${index}].type`);
|
|
965
|
-
nonEmptyString3(error.message, `PrimeIntellect trace line ${line}.errors[${index}].message`);
|
|
966
|
-
if (error.traceback !== void 0 && error.traceback !== null && typeof error.traceback !== "string") {
|
|
967
|
-
throw new Error(
|
|
968
|
-
`PrimeIntellect trace line ${line}.errors[${index}].traceback must be a string or null`
|
|
969
|
-
);
|
|
970
|
-
}
|
|
971
|
-
}
|
|
972
|
-
if (trace.is_completed !== void 0 && typeof trace.is_completed !== "boolean") {
|
|
973
|
-
throw new Error(`PrimeIntellect trace line ${line}.is_completed must be boolean`);
|
|
974
|
-
}
|
|
975
|
-
if (trace.stop_condition !== void 0 && trace.stop_condition !== null && typeof trace.stop_condition !== "string") {
|
|
976
|
-
throw new Error(`PrimeIntellect trace line ${line}.stop_condition must be a string or null`);
|
|
977
|
-
}
|
|
978
|
-
if (trace.timing !== void 0) validateTiming(trace.timing, line);
|
|
979
|
-
return value;
|
|
596
|
+
const trace = object(value, `PrimeIntellect trace line ${line}`);
|
|
597
|
+
nonEmptyString(trace.id, `PrimeIntellect trace line ${line}.id`);
|
|
598
|
+
const task = object(trace.task, `PrimeIntellect trace line ${line}.task`);
|
|
599
|
+
nonEmptyString(task.type, `PrimeIntellect trace line ${line}.task.type`);
|
|
600
|
+
const data = object(task.data, `PrimeIntellect trace line ${line}.task.data`);
|
|
601
|
+
if (!Number.isSafeInteger(data.idx) || data.idx < 0) throw new Error(`PrimeIntellect trace line ${line}.task.data.idx must be a non-negative integer`);
|
|
602
|
+
if (!Array.isArray(trace.nodes)) throw new Error(`PrimeIntellect trace line ${line}.nodes must be an array`);
|
|
603
|
+
for (const [index, rawNode] of trace.nodes.entries()) {
|
|
604
|
+
const node = object(rawNode, `PrimeIntellect trace line ${line}.nodes[${index}]`);
|
|
605
|
+
const parent = node.parent;
|
|
606
|
+
if (parent !== void 0 && parent !== null && (!Number.isSafeInteger(parent) || parent < 0 || parent >= index)) throw new Error(`PrimeIntellect trace line ${line}.nodes[${index}].parent must reference an earlier node`);
|
|
607
|
+
if (node.sampled !== void 0 && typeof node.sampled !== "boolean") throw new Error(`PrimeIntellect trace line ${line}.nodes[${index}].sampled must be boolean`);
|
|
608
|
+
if (node.usage !== void 0 && node.usage !== null) validateUsage(node.usage, `PrimeIntellect trace line ${line}.nodes[${index}].usage`);
|
|
609
|
+
}
|
|
610
|
+
validateNumberMap(trace.rewards, `PrimeIntellect trace line ${line}.rewards`);
|
|
611
|
+
validateNumberMap(trace.metrics, `PrimeIntellect trace line ${line}.metrics`);
|
|
612
|
+
if (trace.extra_usage !== void 0 && !Array.isArray(trace.extra_usage)) throw new Error(`PrimeIntellect trace line ${line}.extra_usage must be an array`);
|
|
613
|
+
for (const [index, usage] of (trace.extra_usage ?? []).entries()) validateUsage(usage, `PrimeIntellect trace line ${line}.extra_usage[${index}]`);
|
|
614
|
+
if (trace.errors !== void 0 && !Array.isArray(trace.errors)) throw new Error(`PrimeIntellect trace line ${line}.errors must be an array`);
|
|
615
|
+
for (const [index, rawError] of (trace.errors ?? []).entries()) {
|
|
616
|
+
const error = object(rawError, `PrimeIntellect trace line ${line}.errors[${index}]`);
|
|
617
|
+
nonEmptyString(error.type, `PrimeIntellect trace line ${line}.errors[${index}].type`);
|
|
618
|
+
nonEmptyString(error.message, `PrimeIntellect trace line ${line}.errors[${index}].message`);
|
|
619
|
+
if (error.traceback !== void 0 && error.traceback !== null && typeof error.traceback !== "string") throw new Error(`PrimeIntellect trace line ${line}.errors[${index}].traceback must be a string or null`);
|
|
620
|
+
}
|
|
621
|
+
if (trace.is_completed !== void 0 && typeof trace.is_completed !== "boolean") throw new Error(`PrimeIntellect trace line ${line}.is_completed must be boolean`);
|
|
622
|
+
if (trace.stop_condition !== void 0 && trace.stop_condition !== null && typeof trace.stop_condition !== "string") throw new Error(`PrimeIntellect trace line ${line}.stop_condition must be a string or null`);
|
|
623
|
+
if (trace.timing !== void 0) validateTiming(trace.timing, line);
|
|
624
|
+
return value;
|
|
980
625
|
}
|
|
981
626
|
function validateTiming(value, line) {
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
627
|
+
const timing = object(value, `PrimeIntellect trace line ${line}.timing`);
|
|
628
|
+
optionalFinite(timing.start, `PrimeIntellect trace line ${line}.timing.start`);
|
|
629
|
+
for (const phase of [
|
|
630
|
+
"setup",
|
|
631
|
+
"generation",
|
|
632
|
+
"finalize",
|
|
633
|
+
"scoring"
|
|
634
|
+
]) {
|
|
635
|
+
if (timing[phase] === void 0) continue;
|
|
636
|
+
const span = object(timing[phase], `PrimeIntellect trace line ${line}.timing.${phase}`);
|
|
637
|
+
optionalFinite(span.start, `PrimeIntellect trace line ${line}.timing.${phase}.start`);
|
|
638
|
+
optionalFinite(span.end, `PrimeIntellect trace line ${line}.timing.${phase}.end`);
|
|
639
|
+
}
|
|
990
640
|
}
|
|
991
641
|
function aggregateUsage(usages) {
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
);
|
|
1031
|
-
}
|
|
1032
|
-
return {
|
|
1033
|
-
input,
|
|
1034
|
-
output,
|
|
1035
|
-
...sawReasoning ? { reasoning } : {},
|
|
1036
|
-
...sawCached ? { cached } : {},
|
|
1037
|
-
reportedCostUsd: costUsd,
|
|
1038
|
-
costComplete: usages.length > 0 && costsReported === usages.length
|
|
1039
|
-
};
|
|
642
|
+
let input = 0;
|
|
643
|
+
let output = 0;
|
|
644
|
+
let reasoning = 0;
|
|
645
|
+
let cached = 0;
|
|
646
|
+
let costUsd = 0;
|
|
647
|
+
let sawReasoning = false;
|
|
648
|
+
let sawCached = false;
|
|
649
|
+
let costsReported = 0;
|
|
650
|
+
for (const [index, usage] of usages.entries()) {
|
|
651
|
+
const prompt = nonNegative(usage.prompt_tokens, `usage[${index}].prompt_tokens`);
|
|
652
|
+
const completion = nonNegative(usage.completion_tokens, `usage[${index}].completion_tokens`);
|
|
653
|
+
const cachedInput = optionalNonNegative(usage.cached_input_tokens, `usage[${index}].cached_input_tokens`);
|
|
654
|
+
const reasoningTokens = optionalNonNegative(usage.reasoning_tokens, `usage[${index}].reasoning_tokens`);
|
|
655
|
+
const cost = optionalNonNegative(usage.cost, `usage[${index}].cost`);
|
|
656
|
+
input += prompt + (cachedInput ?? 0);
|
|
657
|
+
output += completion;
|
|
658
|
+
if (cachedInput !== void 0) {
|
|
659
|
+
cached += cachedInput;
|
|
660
|
+
sawCached = true;
|
|
661
|
+
}
|
|
662
|
+
if (reasoningTokens !== void 0) {
|
|
663
|
+
reasoning += reasoningTokens;
|
|
664
|
+
sawReasoning = true;
|
|
665
|
+
}
|
|
666
|
+
if (cost !== void 0) {
|
|
667
|
+
costUsd += cost;
|
|
668
|
+
costsReported += 1;
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
if (reasoning > output) throw new Error(`PrimeIntellect reasoning token total ${reasoning} exceeds output total ${output}`);
|
|
672
|
+
return {
|
|
673
|
+
input,
|
|
674
|
+
output,
|
|
675
|
+
...sawReasoning ? { reasoning } : {},
|
|
676
|
+
...sawCached ? { cached } : {},
|
|
677
|
+
reportedCostUsd: costUsd,
|
|
678
|
+
costComplete: usages.length > 0 && costsReported === usages.length
|
|
679
|
+
};
|
|
1040
680
|
}
|
|
1041
681
|
function validateUsage(value, path) {
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
682
|
+
const usage = object(value, path);
|
|
683
|
+
nonNegative(usage.prompt_tokens, `${path}.prompt_tokens`);
|
|
684
|
+
nonNegative(usage.completion_tokens, `${path}.completion_tokens`);
|
|
685
|
+
optionalNonNegative(usage.cached_input_tokens, `${path}.cached_input_tokens`);
|
|
686
|
+
optionalNonNegative(usage.reasoning_tokens, `${path}.reasoning_tokens`);
|
|
687
|
+
optionalNonNegative(usage.cost, `${path}.cost`);
|
|
1048
688
|
}
|
|
1049
689
|
function traceWallMs(trace) {
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
690
|
+
const timing = trace.timing;
|
|
691
|
+
if (!timing || typeof timing.start !== "number" || !Number.isFinite(timing.start)) return 0;
|
|
692
|
+
const ends = [
|
|
693
|
+
timing.setup?.end,
|
|
694
|
+
timing.generation?.end,
|
|
695
|
+
timing.finalize?.end,
|
|
696
|
+
timing.scoring?.end
|
|
697
|
+
].filter((value) => typeof value === "number" && Number.isFinite(value));
|
|
698
|
+
if (ends.length === 0) return 0;
|
|
699
|
+
return Math.max(0, (Math.max(...ends) - timing.start) * 1e3);
|
|
1060
700
|
}
|
|
1061
701
|
function countBranches(nodes) {
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
(parent) => Number.isSafeInteger(parent) && parent >= 0
|
|
1066
|
-
)
|
|
1067
|
-
);
|
|
1068
|
-
return nodes.reduce((count, _node, index) => count + (parents.has(index) ? 0 : 1), 0);
|
|
702
|
+
if (nodes.length === 0) return 0;
|
|
703
|
+
const parents = new Set(nodes.map((node) => node.parent).filter((parent) => Number.isSafeInteger(parent) && parent >= 0));
|
|
704
|
+
return nodes.reduce((count, _node, index) => count + (parents.has(index) ? 0 : 1), 0);
|
|
1069
705
|
}
|
|
1070
706
|
function validateNumberMap(value, path) {
|
|
1071
|
-
|
|
1072
|
-
|
|
707
|
+
const map = object(value, path);
|
|
708
|
+
for (const [key, entry] of Object.entries(map)) finite(entry, `${path}.${key}`);
|
|
1073
709
|
}
|
|
1074
710
|
function sumFinite(values, path) {
|
|
1075
|
-
|
|
711
|
+
return values.reduce((sum, value, index) => sum + finite(value, `${path}[${index}]`), 0);
|
|
1076
712
|
}
|
|
1077
713
|
function finite(value, path) {
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
}
|
|
1081
|
-
return value;
|
|
714
|
+
if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${path} must be a finite number`);
|
|
715
|
+
return value;
|
|
1082
716
|
}
|
|
1083
717
|
function nonNegative(value, path) {
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
718
|
+
const parsed = finite(value, path);
|
|
719
|
+
if (parsed < 0) throw new Error(`${path} must be non-negative`);
|
|
720
|
+
return parsed;
|
|
1087
721
|
}
|
|
1088
722
|
function optionalNonNegative(value, path) {
|
|
1089
|
-
|
|
1090
|
-
|
|
723
|
+
if (value === void 0 || value === null) return void 0;
|
|
724
|
+
return nonNegative(value, path);
|
|
1091
725
|
}
|
|
1092
726
|
function optionalFinite(value, path) {
|
|
1093
|
-
|
|
1094
|
-
|
|
727
|
+
if (value === void 0 || value === null) return void 0;
|
|
728
|
+
return finite(value, path);
|
|
1095
729
|
}
|
|
1096
730
|
function object(value, path) {
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
}
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
}
|
|
1108
|
-
export {
|
|
1109
|
-
createPrimeIntellectBackend,
|
|
1110
|
-
createPrimeIntellectPackage,
|
|
1111
|
-
importPrimeIntellectTraces,
|
|
1112
|
-
parsePrimeIntellectTraces,
|
|
1113
|
-
primeIntellectTraceToRunRecord,
|
|
1114
|
-
readPrimeIntellectEpisodeContext,
|
|
1115
|
-
runPrimeIntellectProgram,
|
|
1116
|
-
writePrimeIntellectPackage
|
|
1117
|
-
};
|
|
731
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(`${path} must be an object`);
|
|
732
|
+
return value;
|
|
733
|
+
}
|
|
734
|
+
function nonEmptyString(value, path) {
|
|
735
|
+
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
736
|
+
return value;
|
|
737
|
+
}
|
|
738
|
+
//#endregion
|
|
739
|
+
export { createPrimeIntellectBackend, createPrimeIntellectPackage, importPrimeIntellectTraces, parsePrimeIntellectTraces, primeIntellectTraceToRunRecord, readPrimeIntellectEpisodeContext, runPrimeIntellectProgram, writePrimeIntellectPackage };
|
|
740
|
+
|
|
1118
741
|
//# sourceMappingURL=index.js.map
|