anti-yapping 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +77 -2
- package/core/00-identity.md +8 -0
- package/core/10-safety.md +6 -0
- package/core/20-decision.md +10 -0
- package/core/30-efficiency.md +16 -0
- package/core/always-on.md +18 -0
- package/culture/id/terse-lexicon.yaml +21 -0
- package/dist/cli.js +3361 -0
- package/package.json +60 -4
- package/personas/terse/card.md +7 -0
- package/personas/terse/examples.jsonl +3 -0
- package/personas/terse/lexicon.yaml +7 -0
- package/personas/terse/persona.yaml +22 -0
- package/personas/terse/tests.yaml +14 -0
- package/personas/terse/voice.md +36 -0
- package/profiles/anti-yapping.yaml +20 -0
- package/profiles/terse-id.yaml +20 -0
- package/schemas/coding-fixture.schema.json +53 -0
- package/schemas/config.schema.json +18 -0
- package/schemas/culture.schema.json +75 -0
- package/schemas/eval-suite.schema.json +66 -0
- package/schemas/install-manifest.schema.json +52 -0
- package/schemas/lexicon.schema.json +27 -0
- package/schemas/manifest.schema.json +57 -0
- package/schemas/memes.schema.json +31 -0
- package/schemas/persona.schema.json +68 -0
- package/schemas/profile.schema.json +51 -0
- package/schemas/register-map.schema.json +105 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,3361 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/cli.ts
|
|
4
|
+
import { readFileSync as readFileSync7, readdirSync as readdirSync4 } from "node:fs";
|
|
5
|
+
import { join as join8, resolve as resolve6 } from "node:path";
|
|
6
|
+
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
7
|
+
import { parseArgs } from "node:util";
|
|
8
|
+
|
|
9
|
+
// src/build.ts
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
import {
|
|
12
|
+
existsSync,
|
|
13
|
+
mkdirSync,
|
|
14
|
+
readdirSync,
|
|
15
|
+
readFileSync as readFileSync2,
|
|
16
|
+
unlinkSync,
|
|
17
|
+
writeFileSync
|
|
18
|
+
} from "node:fs";
|
|
19
|
+
import { dirname, extname, join as join2, relative, resolve as resolve2, sep } from "node:path";
|
|
20
|
+
import { parse, stringify } from "yaml";
|
|
21
|
+
|
|
22
|
+
// src/adapters/fake.ts
|
|
23
|
+
var FakeAdapter = class {
|
|
24
|
+
id = "fake";
|
|
25
|
+
capabilities = {
|
|
26
|
+
alwaysOnInstructions: true,
|
|
27
|
+
onDemandSkills: true,
|
|
28
|
+
progressiveDisclosure: true,
|
|
29
|
+
managedBlock: false,
|
|
30
|
+
toolPermissions: false,
|
|
31
|
+
hooks: false
|
|
32
|
+
};
|
|
33
|
+
render(input) {
|
|
34
|
+
const metadata = JSON.stringify(
|
|
35
|
+
{
|
|
36
|
+
name: `hahahah-${input.persona.id}`,
|
|
37
|
+
description: `${input.persona.display_name} persona for ${input.persona.locale} coding work`
|
|
38
|
+
},
|
|
39
|
+
null,
|
|
40
|
+
2
|
|
41
|
+
);
|
|
42
|
+
return [
|
|
43
|
+
{
|
|
44
|
+
path: "always-on.md",
|
|
45
|
+
content: `${input.core.trim()}
|
|
46
|
+
|
|
47
|
+
${input.card.trim()}
|
|
48
|
+
`,
|
|
49
|
+
kind: "always_on"
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
path: "skill-metadata.json",
|
|
53
|
+
content: `${metadata}
|
|
54
|
+
`,
|
|
55
|
+
kind: "skill_metadata"
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
path: "references/voice.md",
|
|
59
|
+
content: `${input.voice.trim()}
|
|
60
|
+
`,
|
|
61
|
+
kind: "skill_body"
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
path: "references/lexicon.md",
|
|
65
|
+
content: `${input.lexiconMarkdown.trim()}
|
|
66
|
+
`,
|
|
67
|
+
kind: "reference"
|
|
68
|
+
},
|
|
69
|
+
...input.coreRefs.map((reference) => ({
|
|
70
|
+
path: `references/core/${reference.path}`,
|
|
71
|
+
content: `${reference.content.trim()}
|
|
72
|
+
`,
|
|
73
|
+
kind: "reference"
|
|
74
|
+
})),
|
|
75
|
+
...input.cultureRefs.map((reference) => ({
|
|
76
|
+
path: `references/culture/${reference.path}`,
|
|
77
|
+
content: `${reference.content.trim()}
|
|
78
|
+
`,
|
|
79
|
+
kind: "reference"
|
|
80
|
+
}))
|
|
81
|
+
];
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
// src/adapters/generic.ts
|
|
86
|
+
function marker(input) {
|
|
87
|
+
return `<!-- hahahah:begin profile=${input.profile.id} version=${input.persona.version} -->`;
|
|
88
|
+
}
|
|
89
|
+
function block(parts) {
|
|
90
|
+
return `${parts.join("\n")}
|
|
91
|
+
`;
|
|
92
|
+
}
|
|
93
|
+
function alwaysOnBlock(input, extra = []) {
|
|
94
|
+
return block([
|
|
95
|
+
marker(input),
|
|
96
|
+
input.core.trim(),
|
|
97
|
+
"",
|
|
98
|
+
input.card.trim(),
|
|
99
|
+
...extra,
|
|
100
|
+
"<!-- hahahah:end -->"
|
|
101
|
+
]);
|
|
102
|
+
}
|
|
103
|
+
var GenericHarnessAdapter = class {
|
|
104
|
+
id;
|
|
105
|
+
scope;
|
|
106
|
+
spec;
|
|
107
|
+
constructor(spec, scope = "workspace") {
|
|
108
|
+
this.spec = spec;
|
|
109
|
+
this.id = spec.id;
|
|
110
|
+
this.scope = scope;
|
|
111
|
+
}
|
|
112
|
+
render(input) {
|
|
113
|
+
const skillName = `hahahah-${input.persona.id}`;
|
|
114
|
+
const description = `HAHahah ${input.persona.display_name} persona for ${input.persona.locale} coding work. Use when this voice is active; includes humor boundaries, serious-mode rules, and culture references.`;
|
|
115
|
+
const root = this.scope === "global" ? this.spec.globalOnDemandRoot : this.spec.onDemandRoot;
|
|
116
|
+
if (this.spec.layout === "fold") {
|
|
117
|
+
return [
|
|
118
|
+
{
|
|
119
|
+
path: this.spec.blockPath,
|
|
120
|
+
content: alwaysOnBlock(input, [
|
|
121
|
+
"",
|
|
122
|
+
"<!-- folded: this target has no on-demand layer, so the voice ships inline -->",
|
|
123
|
+
"",
|
|
124
|
+
input.voice.trim()
|
|
125
|
+
]),
|
|
126
|
+
kind: "always_on"
|
|
127
|
+
}
|
|
128
|
+
];
|
|
129
|
+
}
|
|
130
|
+
if (this.spec.layout === "rules-dir") {
|
|
131
|
+
const rules = [
|
|
132
|
+
{
|
|
133
|
+
path: `${root}/${skillName}-voice.md`,
|
|
134
|
+
content: block(["---", "trigger: always_on", "---", "", input.voice.trim()]),
|
|
135
|
+
kind: "skill_body"
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
path: `${root}/${skillName}-lexicon.md`,
|
|
139
|
+
content: block([
|
|
140
|
+
"---",
|
|
141
|
+
"trigger: model_decision",
|
|
142
|
+
`description: "HAHahah ${input.persona.display_name} lexicon: Indonesian slang, register, and internet-culture terms."`,
|
|
143
|
+
"---",
|
|
144
|
+
"",
|
|
145
|
+
input.lexiconMarkdown.trim()
|
|
146
|
+
]),
|
|
147
|
+
kind: "reference"
|
|
148
|
+
},
|
|
149
|
+
...input.coreRefs.map((reference) => ({
|
|
150
|
+
path: `${root}/${skillName}-core-${reference.path}`,
|
|
151
|
+
content: block([
|
|
152
|
+
"---",
|
|
153
|
+
"trigger: model_decision",
|
|
154
|
+
`description: "HAHahah core policy ${reference.path}: identity, safety, decision, efficiency rules that outrank persona tone."`,
|
|
155
|
+
"---",
|
|
156
|
+
"",
|
|
157
|
+
reference.content.trim()
|
|
158
|
+
]),
|
|
159
|
+
kind: "reference"
|
|
160
|
+
})),
|
|
161
|
+
...input.cultureRefs.map((reference) => ({
|
|
162
|
+
path: `${root}/${skillName}-culture-${reference.path}`,
|
|
163
|
+
content: block([
|
|
164
|
+
"---",
|
|
165
|
+
"trigger: model_decision",
|
|
166
|
+
`description: "HAHahah culture reference ${reference.path}: Indonesian humor boundaries, memes, registers, religious-expression etiquette."`,
|
|
167
|
+
"---",
|
|
168
|
+
"",
|
|
169
|
+
reference.content.trim()
|
|
170
|
+
]),
|
|
171
|
+
kind: "reference"
|
|
172
|
+
}))
|
|
173
|
+
];
|
|
174
|
+
return [
|
|
175
|
+
{
|
|
176
|
+
path: this.spec.blockPath,
|
|
177
|
+
content: alwaysOnBlock(input, [
|
|
178
|
+
"",
|
|
179
|
+
`Voice detail, lexicon, kebijakan penuh, dan referensi budaya ada di \`${root}/\` sebagai rules; yang selalu termuat cuma voice.`
|
|
180
|
+
]),
|
|
181
|
+
kind: "always_on"
|
|
182
|
+
},
|
|
183
|
+
...rules
|
|
184
|
+
];
|
|
185
|
+
}
|
|
186
|
+
const references = [
|
|
187
|
+
{
|
|
188
|
+
path: `${root}/${skillName}/references/lexicon.md`,
|
|
189
|
+
content: `${input.lexiconMarkdown.trim()}
|
|
190
|
+
`,
|
|
191
|
+
kind: "reference"
|
|
192
|
+
},
|
|
193
|
+
...input.coreRefs.map((reference) => ({
|
|
194
|
+
path: `${root}/${skillName}/references/core/${reference.path}`,
|
|
195
|
+
content: `${reference.content.trim()}
|
|
196
|
+
`,
|
|
197
|
+
kind: "reference"
|
|
198
|
+
})),
|
|
199
|
+
...input.cultureRefs.map((reference) => ({
|
|
200
|
+
path: `${root}/${skillName}/references/culture/${reference.path}`,
|
|
201
|
+
content: `${reference.content.trim()}
|
|
202
|
+
`,
|
|
203
|
+
kind: "reference"
|
|
204
|
+
}))
|
|
205
|
+
];
|
|
206
|
+
return [
|
|
207
|
+
{ path: this.spec.blockPath, content: alwaysOnBlock(input), kind: "always_on" },
|
|
208
|
+
{
|
|
209
|
+
path: `${root}/${skillName}/SKILL.md`,
|
|
210
|
+
content: block([
|
|
211
|
+
"---",
|
|
212
|
+
`name: ${skillName}`,
|
|
213
|
+
`description: ${JSON.stringify(description)}`,
|
|
214
|
+
"---",
|
|
215
|
+
"",
|
|
216
|
+
input.voice.trim(),
|
|
217
|
+
"",
|
|
218
|
+
"Load `references/lexicon.md` for terms, `references/core/*.md` for the full identity, safety, decision, and efficiency policies, and `references/culture/*` for humor boundaries and regional guidance. Read a reference only when the task needs it."
|
|
219
|
+
]),
|
|
220
|
+
kind: "skill_body"
|
|
221
|
+
},
|
|
222
|
+
...references
|
|
223
|
+
];
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
// src/harnesses.ts
|
|
228
|
+
var HARNESS_SPECS = [
|
|
229
|
+
{
|
|
230
|
+
id: "claude-code",
|
|
231
|
+
layout: "skill-tree",
|
|
232
|
+
blockPath: "CLAUDE.md",
|
|
233
|
+
onDemandRoot: ".claude/skills",
|
|
234
|
+
globalOnDemandRoot: null,
|
|
235
|
+
alwaysOnTarget: null,
|
|
236
|
+
alwaysOnHardCap: null,
|
|
237
|
+
status: "verified on this machine",
|
|
238
|
+
source: "https://code.claude.com/docs/en/skills",
|
|
239
|
+
notes: [
|
|
240
|
+
"Reference target: install, doctor, uninstall, headless run and permission posture all measured here.",
|
|
241
|
+
"Docs list AGENTS.md only as a fallback when no CLAUDE.md exists, so the block belongs in CLAUDE.md."
|
|
242
|
+
]
|
|
243
|
+
},
|
|
244
|
+
{
|
|
245
|
+
id: "opencode",
|
|
246
|
+
layout: "skill-tree",
|
|
247
|
+
blockPath: "AGENTS.md",
|
|
248
|
+
onDemandRoot: ".claude/skills",
|
|
249
|
+
globalOnDemandRoot: null,
|
|
250
|
+
alwaysOnTarget: null,
|
|
251
|
+
alwaysOnHardCap: null,
|
|
252
|
+
status: "verified on this machine",
|
|
253
|
+
source: "opencode 1.18.31: `debug skill` listed the installed skill; a live run echoed the AGENTS.md marker",
|
|
254
|
+
notes: [
|
|
255
|
+
"opencode discovers the Claude-compatible skill tree, so reusing .claude/skills avoids a second copy under .opencode/.",
|
|
256
|
+
'Permissions are NOT written here: opencode defaults the build agent to "*": allow, and dropping an opencode.json into a user repo would silently widen what an install does. The study posture lives in the runner instead (OPENCODE_CONFIG).'
|
|
257
|
+
]
|
|
258
|
+
},
|
|
259
|
+
{
|
|
260
|
+
id: "agents-md",
|
|
261
|
+
layout: "skill-tree",
|
|
262
|
+
blockPath: "AGENTS.md",
|
|
263
|
+
onDemandRoot: ".agents/skills",
|
|
264
|
+
globalOnDemandRoot: null,
|
|
265
|
+
alwaysOnTarget: null,
|
|
266
|
+
alwaysOnHardCap: null,
|
|
267
|
+
status: "documented, not verified",
|
|
268
|
+
source: "https://agentskills.io/specification",
|
|
269
|
+
notes: [
|
|
270
|
+
"Vendor-neutral target for Codex and Cursor, both of which document .agents/skills.",
|
|
271
|
+
"No Cursor or Codex run on this machine has read the output; the paths follow the published standard."
|
|
272
|
+
]
|
|
273
|
+
},
|
|
274
|
+
{
|
|
275
|
+
id: "antigravity",
|
|
276
|
+
layout: "rules-dir",
|
|
277
|
+
blockPath: "AGENTS.md",
|
|
278
|
+
onDemandRoot: ".agents/rules",
|
|
279
|
+
globalOnDemandRoot: "config/rules",
|
|
280
|
+
alwaysOnTarget: null,
|
|
281
|
+
alwaysOnHardCap: null,
|
|
282
|
+
status: "documented, not verified",
|
|
283
|
+
source: "https://antigravity.google/docs/rules/",
|
|
284
|
+
notes: [
|
|
285
|
+
"Rules files need a `trigger` in frontmatter: always_on injects the whole file every turn, model_decision injects only its description until the model opens it.",
|
|
286
|
+
"Documented ceilings are 24KB per file and a 20k-token rule budget; nothing here approaches them.",
|
|
287
|
+
"`opencode debug agent` style introspection does not reflect this layout, so loading has never been observed."
|
|
288
|
+
]
|
|
289
|
+
},
|
|
290
|
+
{
|
|
291
|
+
id: "qoder",
|
|
292
|
+
layout: "skill-tree",
|
|
293
|
+
blockPath: "AGENTS.md",
|
|
294
|
+
onDemandRoot: ".qoder/skills",
|
|
295
|
+
globalOnDemandRoot: null,
|
|
296
|
+
alwaysOnTarget: null,
|
|
297
|
+
alwaysOnHardCap: null,
|
|
298
|
+
status: "documented, not verified",
|
|
299
|
+
source: "https://docs.qoder.com/cli/Skills",
|
|
300
|
+
notes: [
|
|
301
|
+
"Docs give memory as <project>/AGENTS.md plus .qoder/rules/, and skills as .qoder/skills/<name>/SKILL.md.",
|
|
302
|
+
"The skills page does not claim adherence to the open Agent Skills standard, and auto-trigger is unverified; user-level skills also override project-level ones on a name clash.",
|
|
303
|
+
"Qoder Desktop was not on PATH for any run in this repository, so no Qoder integration has been executed."
|
|
304
|
+
]
|
|
305
|
+
},
|
|
306
|
+
{
|
|
307
|
+
id: "copilot",
|
|
308
|
+
layout: "fold",
|
|
309
|
+
blockPath: ".github/copilot-instructions.md",
|
|
310
|
+
onDemandRoot: null,
|
|
311
|
+
globalOnDemandRoot: null,
|
|
312
|
+
// core + card + voice measured at 2342 tokens on rusuh-id; the global 600 ceiling describes a
|
|
313
|
+
// card-only block and would be a lie here.
|
|
314
|
+
alwaysOnTarget: 2650,
|
|
315
|
+
alwaysOnHardCap: 3e3,
|
|
316
|
+
status: "documented, not verified",
|
|
317
|
+
source: "https://docs.github.com/copilot/customizing-copilot/adding-repository-custom-instructions-for-github-copilot",
|
|
318
|
+
notes: [
|
|
319
|
+
"AGENTS.md is supported for the coding agent (github.blog changelog 2025-08-28), but skill support is not documented, so the voice has nowhere to live except the instructions file itself.",
|
|
320
|
+
"Precedence is the reason this is not the reference target: personal instructions outrank repository instructions, so a safety-sensitive layer cannot rely on being the last word.",
|
|
321
|
+
"Folding costs the full voice on every turn \u2014 the build warns about it rather than hiding it."
|
|
322
|
+
]
|
|
323
|
+
}
|
|
324
|
+
];
|
|
325
|
+
function harnessSpec(id) {
|
|
326
|
+
return HARNESS_SPECS.find((spec) => spec.id === id);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// src/budget.ts
|
|
330
|
+
function evaluateBudget(spec, tokens) {
|
|
331
|
+
let status = "ok";
|
|
332
|
+
if (tokens > spec.hardCap) {
|
|
333
|
+
status = "error";
|
|
334
|
+
} else if (tokens > spec.target) {
|
|
335
|
+
status = "warning";
|
|
336
|
+
}
|
|
337
|
+
return { ...spec, tokens, status };
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// src/schema.ts
|
|
341
|
+
import { readFileSync } from "node:fs";
|
|
342
|
+
import { join, resolve } from "node:path";
|
|
343
|
+
import { fileURLToPath } from "node:url";
|
|
344
|
+
import Ajv from "ajv";
|
|
345
|
+
var defaultPackageRoot = resolve(fileURLToPath(import.meta.url), "..", "..");
|
|
346
|
+
var schemaFiles = {
|
|
347
|
+
profile: "profile.schema.json",
|
|
348
|
+
persona: "persona.schema.json",
|
|
349
|
+
culture: "culture.schema.json",
|
|
350
|
+
memes: "memes.schema.json",
|
|
351
|
+
registers: "register-map.schema.json",
|
|
352
|
+
lexicon: "lexicon.schema.json",
|
|
353
|
+
config: "config.schema.json",
|
|
354
|
+
manifest: "manifest.schema.json",
|
|
355
|
+
install: "install-manifest.schema.json",
|
|
356
|
+
evalSuite: "eval-suite.schema.json",
|
|
357
|
+
codingFixture: "coding-fixture.schema.json"
|
|
358
|
+
};
|
|
359
|
+
var validators = /* @__PURE__ */ new Map();
|
|
360
|
+
var SchemaValidationError = class extends Error {
|
|
361
|
+
issues;
|
|
362
|
+
source;
|
|
363
|
+
constructor(source, issues) {
|
|
364
|
+
super(`Schema validation failed for ${source}:
|
|
365
|
+
${issues.join("\n")}`);
|
|
366
|
+
this.name = "SchemaValidationError";
|
|
367
|
+
this.source = source;
|
|
368
|
+
this.issues = issues;
|
|
369
|
+
}
|
|
370
|
+
};
|
|
371
|
+
function validate(name, data, source = "inline", root = defaultPackageRoot) {
|
|
372
|
+
const key = `${root}:${name}`;
|
|
373
|
+
let compiled = validators.get(key);
|
|
374
|
+
if (!compiled) {
|
|
375
|
+
const schemaPath = join(root, "schemas", schemaFiles[name]);
|
|
376
|
+
const schema = JSON.parse(readFileSync(schemaPath, "utf8"));
|
|
377
|
+
const AjvCtor = Ajv;
|
|
378
|
+
const ajv = new AjvCtor({ allErrors: true, strict: false });
|
|
379
|
+
compiled = ajv.compile(schema);
|
|
380
|
+
validators.set(key, compiled);
|
|
381
|
+
}
|
|
382
|
+
if (!compiled(data)) {
|
|
383
|
+
const issues = (compiled.errors ?? []).map((error) => {
|
|
384
|
+
const location = error.instancePath || "/";
|
|
385
|
+
const property = error.params && typeof error.params === "object" && "missingProperty" in error.params ? ` (${String(error.params.missingProperty)})` : "";
|
|
386
|
+
return `${location}${property} ${error.message ?? "invalid value"}`.trim();
|
|
387
|
+
});
|
|
388
|
+
throw new SchemaValidationError(source, issues);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// src/tokenizer.ts
|
|
393
|
+
import { countTokens as countWithO200k } from "gpt-tokenizer";
|
|
394
|
+
var TOKENIZER_PROFILE = "o200k_base";
|
|
395
|
+
function countTokens(text) {
|
|
396
|
+
return countWithO200k(text);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// src/build.ts
|
|
400
|
+
var ADAPTERS = {
|
|
401
|
+
fake: () => new FakeAdapter(),
|
|
402
|
+
...Object.fromEntries(
|
|
403
|
+
HARNESS_SPECS.map((spec) => [
|
|
404
|
+
spec.id,
|
|
405
|
+
(scope) => new GenericHarnessAdapter(spec, scope)
|
|
406
|
+
])
|
|
407
|
+
)
|
|
408
|
+
};
|
|
409
|
+
var CORE_POLICY_FILES = ["00-identity.md", "10-safety.md", "20-decision.md", "30-efficiency.md"];
|
|
410
|
+
var HARD_CAPS = {
|
|
411
|
+
always_on: 600,
|
|
412
|
+
skill_metadata: 120,
|
|
413
|
+
skill_body: 2e3,
|
|
414
|
+
references: 5e3
|
|
415
|
+
};
|
|
416
|
+
var BuildError = class extends Error {
|
|
417
|
+
constructor(message) {
|
|
418
|
+
super(message);
|
|
419
|
+
this.name = "BuildError";
|
|
420
|
+
}
|
|
421
|
+
};
|
|
422
|
+
function sha256(text) {
|
|
423
|
+
return createHash("sha256").update(text).digest("hex");
|
|
424
|
+
}
|
|
425
|
+
function readText(path) {
|
|
426
|
+
return readFileSync2(path, "utf8").replace(/\r\n/g, "\n");
|
|
427
|
+
}
|
|
428
|
+
function readYaml(path) {
|
|
429
|
+
return parse(readText(path));
|
|
430
|
+
}
|
|
431
|
+
function resolveInside(base, child) {
|
|
432
|
+
const basePath = resolve2(base);
|
|
433
|
+
const target = resolve2(basePath, child);
|
|
434
|
+
const prefix = basePath.endsWith(sep) ? basePath : `${basePath}${sep}`;
|
|
435
|
+
if (target !== basePath && !target.startsWith(prefix)) {
|
|
436
|
+
throw new BuildError(`Refusing path outside ${relative(process.cwd(), basePath)}: ${child}`);
|
|
437
|
+
}
|
|
438
|
+
return target;
|
|
439
|
+
}
|
|
440
|
+
function readNonEmpty(path, label) {
|
|
441
|
+
const value = readText(path).trim();
|
|
442
|
+
if (!value) {
|
|
443
|
+
throw new BuildError(`${label} is empty: ${path}`);
|
|
444
|
+
}
|
|
445
|
+
return value;
|
|
446
|
+
}
|
|
447
|
+
function loadProfile(root, id) {
|
|
448
|
+
const path = join2(root, "profiles", `${id}.yaml`);
|
|
449
|
+
const data = readYaml(path);
|
|
450
|
+
validate("profile", data, `profiles/${id}.yaml`, root);
|
|
451
|
+
return data;
|
|
452
|
+
}
|
|
453
|
+
function loadPersona(root, id) {
|
|
454
|
+
const path = join2(root, "personas", id, "persona.yaml");
|
|
455
|
+
const data = readYaml(path);
|
|
456
|
+
validate("persona", data, `personas/${id}/persona.yaml`, root);
|
|
457
|
+
return data;
|
|
458
|
+
}
|
|
459
|
+
function loadPersonaLexicon(root, persona) {
|
|
460
|
+
const path = resolveInside(join2(root, "personas", persona.id), persona.lexicon);
|
|
461
|
+
const data = readYaml(path);
|
|
462
|
+
validate("lexicon", data, `personas/${persona.id}/${persona.lexicon}`, root);
|
|
463
|
+
const lexicon = data;
|
|
464
|
+
if (lexicon.persona !== persona.id) {
|
|
465
|
+
throw new BuildError(`Lexicon persona ${lexicon.persona} does not match ${persona.id}`);
|
|
466
|
+
}
|
|
467
|
+
return lexicon;
|
|
468
|
+
}
|
|
469
|
+
function loadCultureRef(root, ref) {
|
|
470
|
+
const path = resolveInside(join2(root, "culture"), ref);
|
|
471
|
+
const content = readNonEmpty(path, `Culture reference ${ref}`);
|
|
472
|
+
const extension = extname(path).toLowerCase();
|
|
473
|
+
if (extension === ".yaml" || extension === ".yml") {
|
|
474
|
+
const data = readYaml(path);
|
|
475
|
+
const schema = data?.schema;
|
|
476
|
+
if (schema === "hahahah.culture.lexicon/v1") {
|
|
477
|
+
validate("culture", data, `culture/${ref}`, root);
|
|
478
|
+
return { path: ref, content, parsed: data };
|
|
479
|
+
}
|
|
480
|
+
if (schema === "hahahah.culture.memes/v1") {
|
|
481
|
+
validate("memes", data, `culture/${ref}`, root);
|
|
482
|
+
return { path: ref, content, parsed: data };
|
|
483
|
+
}
|
|
484
|
+
if (schema === "hahahah.culture.registers/v1") {
|
|
485
|
+
validate("registers", data, `culture/${ref}`, root);
|
|
486
|
+
return { path: ref, content, parsed: data };
|
|
487
|
+
}
|
|
488
|
+
throw new BuildError(`Unknown culture schema in culture/${ref}`);
|
|
489
|
+
}
|
|
490
|
+
return { path: ref, content };
|
|
491
|
+
}
|
|
492
|
+
function reviewIssues(persona, cultureFiles, memeFiles, registerFiles) {
|
|
493
|
+
const issues = [];
|
|
494
|
+
for (const [name, state] of Object.entries(persona.reviews)) {
|
|
495
|
+
if (name !== "reviewed_at" && state !== "approved") {
|
|
496
|
+
issues.push(`persona ${persona.id}: ${name} review is ${String(state)}`);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
for (const culture of cultureFiles) {
|
|
500
|
+
if (culture.reviewed_by !== "approved") {
|
|
501
|
+
issues.push(`culture ${culture.locale}: reviewed_by is ${culture.reviewed_by}`);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
for (const memes of memeFiles) {
|
|
505
|
+
if (memes.reviewed_by !== "approved") {
|
|
506
|
+
issues.push(`meme notes: reviewed_by is ${memes.reviewed_by}`);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
for (const registers of registerFiles) {
|
|
510
|
+
if (registers.reviewed_by !== "approved") {
|
|
511
|
+
issues.push(`register map ${registers.locale}: reviewed_by is ${registers.reviewed_by}`);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
return issues;
|
|
515
|
+
}
|
|
516
|
+
function expiryIssues(persona, reviewables, today) {
|
|
517
|
+
const issues = [];
|
|
518
|
+
if (persona.expires < today) {
|
|
519
|
+
issues.push(`persona ${persona.id} expired on ${persona.expires}`);
|
|
520
|
+
}
|
|
521
|
+
for (const culture of reviewables) {
|
|
522
|
+
if (culture.next_review < today) {
|
|
523
|
+
issues.push(`culture review expired on ${culture.next_review}`);
|
|
524
|
+
}
|
|
525
|
+
for (const entry of culture.entries ?? []) {
|
|
526
|
+
if (entry.expires < today) {
|
|
527
|
+
issues.push(`culture entry ${entry.id} expired on ${entry.expires}`);
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
return issues;
|
|
532
|
+
}
|
|
533
|
+
function renderLexicon(persona, lexicon) {
|
|
534
|
+
const lines = [
|
|
535
|
+
`# ${persona.display_name} lexicon references`,
|
|
536
|
+
"",
|
|
537
|
+
"Use only in the listed context; respect the cap and the global no-repetition rule.",
|
|
538
|
+
"",
|
|
539
|
+
...lexicon.entries.map((entry) => {
|
|
540
|
+
const cap = entry.max_per_response ?? 1;
|
|
541
|
+
return `- ${entry.culture_ref}: ${entry.usage} (max ${cap} per response)`;
|
|
542
|
+
})
|
|
543
|
+
];
|
|
544
|
+
return lines.join("\n");
|
|
545
|
+
}
|
|
546
|
+
var LEXICON_ENTRY_KEYS = [
|
|
547
|
+
"id",
|
|
548
|
+
"term",
|
|
549
|
+
"gloss",
|
|
550
|
+
"register",
|
|
551
|
+
"regions",
|
|
552
|
+
"decay_class",
|
|
553
|
+
"offensiveness",
|
|
554
|
+
"contexts",
|
|
555
|
+
"avoid_when",
|
|
556
|
+
"max_per_response",
|
|
557
|
+
"purpose"
|
|
558
|
+
];
|
|
559
|
+
var MEME_ENTRY_KEYS = ["id", "pattern", "context", "avoid_when", "note"];
|
|
560
|
+
var REGISTER_KEYS = ["locale", "levels", "honorifics", "defaults", "common_errors", "banyumasan"];
|
|
561
|
+
function pick(source, keys) {
|
|
562
|
+
const record = source;
|
|
563
|
+
return Object.fromEntries(keys.filter((key) => key in record).map((key) => [key, record[key]]));
|
|
564
|
+
}
|
|
565
|
+
function runtimeCulture(ref) {
|
|
566
|
+
const parsed = ref.parsed;
|
|
567
|
+
if (parsed === void 0) {
|
|
568
|
+
return ref.content;
|
|
569
|
+
}
|
|
570
|
+
if (parsed.schema === "hahahah.culture.lexicon/v1") {
|
|
571
|
+
return stringify({
|
|
572
|
+
schema: parsed.schema,
|
|
573
|
+
locale: parsed.locale,
|
|
574
|
+
entries: parsed.entries.map((entry) => pick(entry, LEXICON_ENTRY_KEYS))
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
if (parsed.schema === "hahahah.culture.memes/v1") {
|
|
578
|
+
return stringify({
|
|
579
|
+
schema: parsed.schema,
|
|
580
|
+
entries: parsed.entries.map((entry) => pick(entry, MEME_ENTRY_KEYS))
|
|
581
|
+
});
|
|
582
|
+
}
|
|
583
|
+
return stringify({ schema: parsed.schema, ...pick(parsed, REGISTER_KEYS) });
|
|
584
|
+
}
|
|
585
|
+
function measure(artifacts) {
|
|
586
|
+
return artifacts.map((artifact) => {
|
|
587
|
+
const content = artifact.content.replace(/\r\n/g, "\n");
|
|
588
|
+
return {
|
|
589
|
+
...artifact,
|
|
590
|
+
content,
|
|
591
|
+
tokens: countTokens(content),
|
|
592
|
+
sha256: sha256(content)
|
|
593
|
+
};
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
function sumTokens(artifacts, kind) {
|
|
597
|
+
return artifacts.filter((artifact) => artifact.kind === kind).reduce((total, artifact) => total + artifact.tokens, 0);
|
|
598
|
+
}
|
|
599
|
+
function skillFrontmatterTokens(artifacts) {
|
|
600
|
+
const skill = artifacts.find((artifact) => artifact.path.endsWith("/SKILL.md"));
|
|
601
|
+
if (!skill) {
|
|
602
|
+
return 0;
|
|
603
|
+
}
|
|
604
|
+
const frontmatter = skill.content.match(/^---\n[\s\S]*?\n---\n/);
|
|
605
|
+
return frontmatter ? countTokens(frontmatter[0]) : 0;
|
|
606
|
+
}
|
|
607
|
+
function skillMetadataTokens(artifacts) {
|
|
608
|
+
return sumTokens(artifacts, "skill_metadata") + skillFrontmatterTokens(artifacts);
|
|
609
|
+
}
|
|
610
|
+
function contentHash(artifacts) {
|
|
611
|
+
const hash = createHash("sha256");
|
|
612
|
+
for (const artifact of [...artifacts].sort((a, b) => a.path.localeCompare(b.path))) {
|
|
613
|
+
hash.update(artifact.path);
|
|
614
|
+
hash.update("\0");
|
|
615
|
+
hash.update(artifact.content);
|
|
616
|
+
hash.update("\0");
|
|
617
|
+
}
|
|
618
|
+
return hash.digest("hex");
|
|
619
|
+
}
|
|
620
|
+
function cleanupManagedFiles(outDir, nextPaths) {
|
|
621
|
+
const manifestPath = join2(outDir, "manifest.json");
|
|
622
|
+
if (!existsSync(manifestPath)) {
|
|
623
|
+
return;
|
|
624
|
+
}
|
|
625
|
+
let old;
|
|
626
|
+
try {
|
|
627
|
+
old = JSON.parse(readFileSync2(manifestPath, "utf8"));
|
|
628
|
+
} catch {
|
|
629
|
+
throw new BuildError(`Refusing to clean unreadable manifest: ${manifestPath}`);
|
|
630
|
+
}
|
|
631
|
+
if (old.schema !== "hahahah.manifest/v1" || !Array.isArray(old.files)) {
|
|
632
|
+
throw new BuildError(`Refusing to clean unmanaged manifest: ${manifestPath}`);
|
|
633
|
+
}
|
|
634
|
+
for (const file of old.files) {
|
|
635
|
+
if (typeof file.path !== "string" || nextPaths.has(file.path)) {
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
const path = resolveInside(outDir, file.path);
|
|
639
|
+
if (existsSync(path)) {
|
|
640
|
+
unlinkSync(path);
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
function writeArtifacts(outDir, artifacts, manifest) {
|
|
645
|
+
for (const artifact of artifacts) {
|
|
646
|
+
const path = resolveInside(outDir, artifact.path);
|
|
647
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
648
|
+
writeFileSync(path, artifact.content, "utf8");
|
|
649
|
+
}
|
|
650
|
+
mkdirSync(outDir, { recursive: true });
|
|
651
|
+
writeFileSync(join2(outDir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}
|
|
652
|
+
`, "utf8");
|
|
653
|
+
}
|
|
654
|
+
function makeWarnings(persona, cultureFiles, review, expiry) {
|
|
655
|
+
return [
|
|
656
|
+
...review.map((issue) => `pending review: ${issue}`),
|
|
657
|
+
...expiry.map((issue) => `expired: ${issue}`),
|
|
658
|
+
...cultureFiles.flatMap((culture) => culture.entries).filter((entry) => entry.purpose === "awareness").map((entry) => `awareness-only entry: ${entry.id}`)
|
|
659
|
+
];
|
|
660
|
+
}
|
|
661
|
+
function build(options) {
|
|
662
|
+
const root = resolve2(options.root);
|
|
663
|
+
const profile = loadProfile(root, options.profileId);
|
|
664
|
+
if (!profile.adapters.includes(options.adapterId)) {
|
|
665
|
+
throw new BuildError(`Profile ${profile.id} does not declare adapter ${options.adapterId}`);
|
|
666
|
+
}
|
|
667
|
+
const makeAdapter = ADAPTERS[options.adapterId];
|
|
668
|
+
if (!makeAdapter) {
|
|
669
|
+
throw new BuildError(`Adapter ${options.adapterId} is not implemented`);
|
|
670
|
+
}
|
|
671
|
+
const scope = options.scope ?? "workspace";
|
|
672
|
+
const spec = harnessSpec(options.adapterId);
|
|
673
|
+
if (scope === "global") {
|
|
674
|
+
if (!spec?.globalOnDemandRoot) {
|
|
675
|
+
throw new BuildError(
|
|
676
|
+
`--scope global needs a documented user-level path: ${options.adapterId} has none (${spec ? spec.source : "not a harness row"})`
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
const adapter = makeAdapter(scope);
|
|
681
|
+
const persona = loadPersona(root, profile.persona);
|
|
682
|
+
const personaDir = join2(root, "personas", persona.id);
|
|
683
|
+
const core = readNonEmpty(join2(root, "core", "always-on.md"), "Core always-on");
|
|
684
|
+
const coreRefs = CORE_POLICY_FILES.map((name) => ({
|
|
685
|
+
path: name,
|
|
686
|
+
content: readNonEmpty(join2(root, "core", name), `Core policy ${name}`)
|
|
687
|
+
}));
|
|
688
|
+
const card = readNonEmpty(
|
|
689
|
+
resolveInside(personaDir, persona.card),
|
|
690
|
+
`Persona card ${persona.card}`
|
|
691
|
+
);
|
|
692
|
+
const voice = readNonEmpty(
|
|
693
|
+
resolveInside(personaDir, persona.voice),
|
|
694
|
+
`Persona voice ${persona.voice}`
|
|
695
|
+
);
|
|
696
|
+
const lexicon = loadPersonaLexicon(root, persona);
|
|
697
|
+
const culture = persona.culture_refs.map((ref) => loadCultureRef(root, ref));
|
|
698
|
+
const cultureFiles = culture.flatMap(
|
|
699
|
+
(ref) => ref.parsed?.schema === "hahahah.culture.lexicon/v1" ? [ref.parsed] : []
|
|
700
|
+
);
|
|
701
|
+
const memeFiles = culture.flatMap(
|
|
702
|
+
(ref) => ref.parsed?.schema === "hahahah.culture.memes/v1" ? [ref.parsed] : []
|
|
703
|
+
);
|
|
704
|
+
const registerFiles = culture.flatMap(
|
|
705
|
+
(ref) => ref.parsed?.schema === "hahahah.culture.registers/v1" ? [ref.parsed] : []
|
|
706
|
+
);
|
|
707
|
+
const artifacts = adapter.render({
|
|
708
|
+
profile,
|
|
709
|
+
persona,
|
|
710
|
+
core,
|
|
711
|
+
card,
|
|
712
|
+
voice,
|
|
713
|
+
lexiconMarkdown: renderLexicon(persona, lexicon),
|
|
714
|
+
coreRefs,
|
|
715
|
+
cultureRefs: culture.map((ref) => ({ path: ref.path, content: runtimeCulture(ref) }))
|
|
716
|
+
});
|
|
717
|
+
const measured = measure(artifacts);
|
|
718
|
+
const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
719
|
+
const review = reviewIssues(persona, cultureFiles, memeFiles, registerFiles);
|
|
720
|
+
const expiry = expiryIssues(persona, [...cultureFiles, ...memeFiles, ...registerFiles], today);
|
|
721
|
+
const warnings = makeWarnings(persona, cultureFiles, review, expiry);
|
|
722
|
+
if (options.release && (review.length > 0 || expiry.length > 0)) {
|
|
723
|
+
throw new BuildError(`Release build blocked:
|
|
724
|
+
${[...review, ...expiry].join("\n")}`);
|
|
725
|
+
}
|
|
726
|
+
const budgetSpecs = {
|
|
727
|
+
// A folded block carries the whole voice, so the card-sized global numbers no longer
|
|
728
|
+
// describe it and the harness row states its own target and ceiling instead.
|
|
729
|
+
always_on: {
|
|
730
|
+
target: spec?.alwaysOnTarget ?? profile.budgets.always_on_tokens,
|
|
731
|
+
hardCap: spec?.alwaysOnHardCap ?? HARD_CAPS.always_on
|
|
732
|
+
},
|
|
733
|
+
skill_metadata: {
|
|
734
|
+
target: profile.budgets.skill_metadata_tokens,
|
|
735
|
+
hardCap: HARD_CAPS.skill_metadata
|
|
736
|
+
},
|
|
737
|
+
skill_body: { target: profile.budgets.skill_body_tokens, hardCap: HARD_CAPS.skill_body },
|
|
738
|
+
references: { target: profile.budgets.reference_tokens_max, hardCap: HARD_CAPS.references }
|
|
739
|
+
};
|
|
740
|
+
if (spec?.layout === "fold") {
|
|
741
|
+
warnings.push(
|
|
742
|
+
`folded layout: ${spec.id} has no on-demand layer, so the full voice is injected on every turn (always_on ceiling ${spec.alwaysOnHardCap})`
|
|
743
|
+
);
|
|
744
|
+
}
|
|
745
|
+
const budgets = {
|
|
746
|
+
always_on: evaluateBudget(budgetSpecs.always_on, sumTokens(measured, "always_on")),
|
|
747
|
+
skill_metadata: evaluateBudget(budgetSpecs.skill_metadata, skillMetadataTokens(measured)),
|
|
748
|
+
skill_body: evaluateBudget(
|
|
749
|
+
budgetSpecs.skill_body,
|
|
750
|
+
Math.max(0, sumTokens(measured, "skill_body") - skillFrontmatterTokens(measured))
|
|
751
|
+
),
|
|
752
|
+
references: evaluateBudget(budgetSpecs.references, sumTokens(measured, "reference"))
|
|
753
|
+
};
|
|
754
|
+
const failedBudgets = Object.entries(budgets).filter(([, report]) => report.status === "error").map(([name, report]) => `${name}: ${report.tokens} > hard cap ${report.hardCap}`);
|
|
755
|
+
if (failedBudgets.length > 0) {
|
|
756
|
+
throw new BuildError(`Token budget exceeded:
|
|
757
|
+
${failedBudgets.join("\n")}`);
|
|
758
|
+
}
|
|
759
|
+
for (const [name, report] of Object.entries(budgets)) {
|
|
760
|
+
if (report.status === "warning") {
|
|
761
|
+
warnings.push(
|
|
762
|
+
`token target exceeded: ${name} ${report.tokens}/${report.target} (hard cap ${report.hardCap})`
|
|
763
|
+
);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
const version = (() => {
|
|
767
|
+
try {
|
|
768
|
+
return JSON.parse(readFileSync2(join2(root, "package.json"), "utf8")).version ?? "0.0.0";
|
|
769
|
+
} catch {
|
|
770
|
+
return "0.0.0";
|
|
771
|
+
}
|
|
772
|
+
})();
|
|
773
|
+
const manifest = {
|
|
774
|
+
schema: "hahahah.manifest/v1",
|
|
775
|
+
generated_by: `hahahah@${version}`,
|
|
776
|
+
profile: profile.id,
|
|
777
|
+
persona: persona.id,
|
|
778
|
+
adapter: adapter.id,
|
|
779
|
+
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
780
|
+
content_hash: contentHash(measured),
|
|
781
|
+
tokenizer: TOKENIZER_PROFILE,
|
|
782
|
+
release: options.release === true,
|
|
783
|
+
files: [...measured].sort((a, b) => a.path.localeCompare(b.path)).map((artifact) => ({
|
|
784
|
+
path: artifact.path,
|
|
785
|
+
sha256: artifact.sha256,
|
|
786
|
+
tokens: artifact.tokens
|
|
787
|
+
})),
|
|
788
|
+
budgets,
|
|
789
|
+
warnings
|
|
790
|
+
};
|
|
791
|
+
validate("manifest", manifest, "generated manifest", root);
|
|
792
|
+
const outDir = options.outDir ?? join2(root, ".hahahah", "build", adapter.id);
|
|
793
|
+
if (options.write !== false) {
|
|
794
|
+
if (existsSync(outDir) && !existsSync(join2(outDir, "manifest.json"))) {
|
|
795
|
+
let entries = [];
|
|
796
|
+
try {
|
|
797
|
+
entries = readdirSync(outDir);
|
|
798
|
+
} catch {
|
|
799
|
+
entries = [];
|
|
800
|
+
}
|
|
801
|
+
if (entries.length > 0) {
|
|
802
|
+
throw new BuildError(`Refusing to write into non-empty unmanaged directory: ${outDir}`);
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
cleanupManagedFiles(outDir, new Set(measured.map((artifact) => artifact.path)));
|
|
806
|
+
writeArtifacts(outDir, measured, manifest);
|
|
807
|
+
}
|
|
808
|
+
return { outDir, manifest, artifacts: measured, warnings };
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// src/coding.ts
|
|
812
|
+
import { spawnSync as spawnSync3 } from "node:child_process";
|
|
813
|
+
import {
|
|
814
|
+
existsSync as existsSync4,
|
|
815
|
+
mkdirSync as mkdirSync5,
|
|
816
|
+
mkdtempSync as mkdtempSync2,
|
|
817
|
+
readFileSync as readFileSync5,
|
|
818
|
+
readdirSync as readdirSync3,
|
|
819
|
+
statSync,
|
|
820
|
+
writeFileSync as writeFileSync5
|
|
821
|
+
} from "node:fs";
|
|
822
|
+
import { dirname as dirname3, join as join6, relative as relative2, resolve as resolve5 } from "node:path";
|
|
823
|
+
import { parse as parse3 } from "yaml";
|
|
824
|
+
|
|
825
|
+
// src/install.ts
|
|
826
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
827
|
+
import {
|
|
828
|
+
existsSync as existsSync2,
|
|
829
|
+
mkdirSync as mkdirSync2,
|
|
830
|
+
readFileSync as readFileSync3,
|
|
831
|
+
readdirSync as readdirSync2,
|
|
832
|
+
renameSync,
|
|
833
|
+
rmdirSync,
|
|
834
|
+
unlinkSync as unlinkSync2,
|
|
835
|
+
writeFileSync as writeFileSync2
|
|
836
|
+
} from "node:fs";
|
|
837
|
+
import { dirname as dirname2, join as join3, resolve as resolve3, sep as sep2 } from "node:path";
|
|
838
|
+
var MANIFEST_PATH = ".hahahah/manifest.json";
|
|
839
|
+
var BLOCK_BEGIN = /<!-- hahahah:begin [^\n]*-->/;
|
|
840
|
+
var BLOCK_END = /<!-- hahahah:end -->/;
|
|
841
|
+
var InstallError = class extends Error {
|
|
842
|
+
constructor(message) {
|
|
843
|
+
super(message);
|
|
844
|
+
this.name = "InstallError";
|
|
845
|
+
}
|
|
846
|
+
};
|
|
847
|
+
function sha2562(text) {
|
|
848
|
+
return createHash2("sha256").update(text).digest("hex");
|
|
849
|
+
}
|
|
850
|
+
function resolveInside2(base, child) {
|
|
851
|
+
const basePath = resolve3(base);
|
|
852
|
+
const target = resolve3(basePath, child);
|
|
853
|
+
const prefix = basePath.endsWith(sep2) ? basePath : `${basePath}${sep2}`;
|
|
854
|
+
if (target !== basePath && !target.startsWith(prefix)) {
|
|
855
|
+
throw new InstallError(`Refusing path outside ${basePath}: ${child}`);
|
|
856
|
+
}
|
|
857
|
+
return target;
|
|
858
|
+
}
|
|
859
|
+
function readManifest(targetRoot) {
|
|
860
|
+
const path = join3(targetRoot, MANIFEST_PATH);
|
|
861
|
+
if (!existsSync2(path)) {
|
|
862
|
+
return null;
|
|
863
|
+
}
|
|
864
|
+
let data;
|
|
865
|
+
try {
|
|
866
|
+
data = JSON.parse(readFileSync3(path, "utf8"));
|
|
867
|
+
} catch {
|
|
868
|
+
throw new InstallError(`Unreadable install manifest: ${path}`);
|
|
869
|
+
}
|
|
870
|
+
validate("install", data, MANIFEST_PATH);
|
|
871
|
+
return data;
|
|
872
|
+
}
|
|
873
|
+
function writeAtomic(path, content) {
|
|
874
|
+
mkdirSync2(dirname2(path), { recursive: true });
|
|
875
|
+
const temp = `${path}.tmp-${process.pid}`;
|
|
876
|
+
writeFileSync2(temp, content, "utf8");
|
|
877
|
+
renameSync(temp, path);
|
|
878
|
+
}
|
|
879
|
+
function parseBlock(text) {
|
|
880
|
+
const begins = [...text.matchAll(new RegExp(BLOCK_BEGIN.source, "g"))];
|
|
881
|
+
const ends = [...text.matchAll(new RegExp(BLOCK_END.source, "g"))];
|
|
882
|
+
if (begins.length === 0 && ends.length === 0) {
|
|
883
|
+
return null;
|
|
884
|
+
}
|
|
885
|
+
if (begins.length !== 1 || ends.length !== 1) {
|
|
886
|
+
throw new InstallError("Malformed or duplicate HAHahah managed block markers");
|
|
887
|
+
}
|
|
888
|
+
const begin = begins[0];
|
|
889
|
+
const end = ends[0];
|
|
890
|
+
if (!begin || !end) {
|
|
891
|
+
throw new InstallError("Malformed HAHahah managed block markers");
|
|
892
|
+
}
|
|
893
|
+
if (begin.index === void 0 || end.index === void 0 || end.index < begin.index) {
|
|
894
|
+
throw new InstallError("Malformed HAHahah managed block order");
|
|
895
|
+
}
|
|
896
|
+
const endIndex = end.index + end[0].length;
|
|
897
|
+
const content = text.slice(begin.index, endIndex);
|
|
898
|
+
return { start: begin.index, end: endIndex, content, hash: sha2562(content.trimEnd()) };
|
|
899
|
+
}
|
|
900
|
+
function blockContent(artifact) {
|
|
901
|
+
const text = artifact.content.replace(/\r\n/g, "\n").trimEnd();
|
|
902
|
+
if (!BLOCK_BEGIN.test(text) || !BLOCK_END.test(text)) {
|
|
903
|
+
throw new InstallError("Adapter did not render a valid managed block");
|
|
904
|
+
}
|
|
905
|
+
return `${text}
|
|
906
|
+
`;
|
|
907
|
+
}
|
|
908
|
+
function installBlock(existing, block2, old, force) {
|
|
909
|
+
const parsed = parseBlock(existing);
|
|
910
|
+
const createdFile = existing.trim() === "";
|
|
911
|
+
if (parsed) {
|
|
912
|
+
if (!old && !force) {
|
|
913
|
+
throw new InstallError(
|
|
914
|
+
"An existing HAHahah managed block has no install manifest; use --force to replace it"
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
if (old && old.block.sha256 !== parsed.hash && !force) {
|
|
918
|
+
throw new InstallError("Managed block was modified; use --force to replace it");
|
|
919
|
+
}
|
|
920
|
+
const before = existing.slice(0, parsed.start);
|
|
921
|
+
const after = existing.slice(parsed.end).replace(/^\n+/, "");
|
|
922
|
+
const separator = before && !before.endsWith("\n") ? "\n" : "";
|
|
923
|
+
const tail = after ? `
|
|
924
|
+
${after}` : "";
|
|
925
|
+
return { text: `${before}${separator}${block2}${tail}`, createdFile };
|
|
926
|
+
}
|
|
927
|
+
return {
|
|
928
|
+
text: `${existing.trimEnd()}${existing.trim() ? "\n\n" : ""}${block2}`,
|
|
929
|
+
createdFile
|
|
930
|
+
};
|
|
931
|
+
}
|
|
932
|
+
function install(options) {
|
|
933
|
+
const targetRoot = resolve3(options.targetRoot);
|
|
934
|
+
const adapterId = options.adapterId ?? "claude-code";
|
|
935
|
+
const scope = options.scope ?? "workspace";
|
|
936
|
+
const built = build({
|
|
937
|
+
root: options.sourceRoot,
|
|
938
|
+
profileId: options.profileId,
|
|
939
|
+
adapterId,
|
|
940
|
+
scope,
|
|
941
|
+
write: false
|
|
942
|
+
});
|
|
943
|
+
const blockArtifact = built.artifacts.find((artifact) => artifact.kind === "always_on");
|
|
944
|
+
if (!blockArtifact) {
|
|
945
|
+
throw new InstallError(`Adapter ${adapterId} rendered no always-on managed block`);
|
|
946
|
+
}
|
|
947
|
+
const blockPath = blockArtifact.path;
|
|
948
|
+
const artifactFiles = built.artifacts.filter((artifact) => artifact.kind !== "always_on");
|
|
949
|
+
const block2 = blockContent(blockArtifact);
|
|
950
|
+
const blockHash = sha2562(block2.trimEnd());
|
|
951
|
+
const old = readManifest(targetRoot);
|
|
952
|
+
const oldFiles = new Map((old?.files ?? []).map((file) => [file.path, file]));
|
|
953
|
+
for (const artifact of artifactFiles) {
|
|
954
|
+
const target = resolveInside2(targetRoot, artifact.path);
|
|
955
|
+
if (!existsSync2(target)) {
|
|
956
|
+
continue;
|
|
957
|
+
}
|
|
958
|
+
const oldFile = oldFiles.get(artifact.path);
|
|
959
|
+
if (!oldFile) {
|
|
960
|
+
if (!options.force) {
|
|
961
|
+
throw new InstallError(`Unmanaged file already exists: ${artifact.path}; use --force`);
|
|
962
|
+
}
|
|
963
|
+
continue;
|
|
964
|
+
}
|
|
965
|
+
if (oldFile.sha256 !== artifact.sha256 && !options.force) {
|
|
966
|
+
throw new InstallError(`Managed file was modified: ${artifact.path}; use --force`);
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
const claudePath = join3(targetRoot, blockPath);
|
|
970
|
+
const existingClaude = existsSync2(claudePath) ? readFileSync3(claudePath, "utf8") : "";
|
|
971
|
+
const merged = installBlock(existingClaude, block2, old, options.force === true);
|
|
972
|
+
const createdClaudeMd = old ? old.created_claude_md : merged.createdFile;
|
|
973
|
+
const manifest = {
|
|
974
|
+
schema: "hahahah.install/v1",
|
|
975
|
+
generated_by: built.manifest.generated_by,
|
|
976
|
+
profile: built.manifest.profile,
|
|
977
|
+
persona: built.manifest.persona,
|
|
978
|
+
adapter: built.manifest.adapter,
|
|
979
|
+
installed_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
980
|
+
files: artifactFiles.map((artifact) => ({
|
|
981
|
+
path: artifact.path,
|
|
982
|
+
sha256: artifact.sha256,
|
|
983
|
+
tokens: artifact.tokens
|
|
984
|
+
})).sort((a, b) => a.path.localeCompare(b.path)),
|
|
985
|
+
block: { path: blockPath, sha256: blockHash },
|
|
986
|
+
created_claude_md: createdClaudeMd
|
|
987
|
+
};
|
|
988
|
+
validate("install", manifest, "install manifest", options.sourceRoot);
|
|
989
|
+
if (!options.dryRun) {
|
|
990
|
+
for (const artifact of artifactFiles) {
|
|
991
|
+
writeAtomic(resolveInside2(targetRoot, artifact.path), artifact.content);
|
|
992
|
+
}
|
|
993
|
+
writeAtomic(claudePath, merged.text);
|
|
994
|
+
writeAtomic(join3(targetRoot, MANIFEST_PATH), `${JSON.stringify(manifest, null, 2)}
|
|
995
|
+
`);
|
|
996
|
+
}
|
|
997
|
+
return {
|
|
998
|
+
manifest,
|
|
999
|
+
targetRoot,
|
|
1000
|
+
warnings: built.warnings,
|
|
1001
|
+
dryRun: options.dryRun === true
|
|
1002
|
+
};
|
|
1003
|
+
}
|
|
1004
|
+
function pruneEmptyDirs(start, stop) {
|
|
1005
|
+
let current = start;
|
|
1006
|
+
while (current !== stop && current.startsWith(stop)) {
|
|
1007
|
+
try {
|
|
1008
|
+
if (readdirSync2(current).length > 0) {
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
rmdirSync(current);
|
|
1012
|
+
} catch {
|
|
1013
|
+
return;
|
|
1014
|
+
}
|
|
1015
|
+
current = dirname2(current);
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
function uninstall(options) {
|
|
1019
|
+
const targetRoot = resolve3(options.targetRoot);
|
|
1020
|
+
const old = readManifest(targetRoot);
|
|
1021
|
+
if (!old) {
|
|
1022
|
+
throw new InstallError(`No HAHahah install manifest at ${targetRoot}`);
|
|
1023
|
+
}
|
|
1024
|
+
const claudePath = join3(targetRoot, old.block.path);
|
|
1025
|
+
const existingClaude = existsSync2(claudePath) ? readFileSync3(claudePath, "utf8") : "";
|
|
1026
|
+
const parsed = parseBlock(existingClaude);
|
|
1027
|
+
if (parsed && parsed.hash !== old.block.sha256 && !options.force) {
|
|
1028
|
+
throw new InstallError("Managed block was modified; use --force to uninstall");
|
|
1029
|
+
}
|
|
1030
|
+
for (const file of old.files) {
|
|
1031
|
+
const target = resolveInside2(targetRoot, file.path);
|
|
1032
|
+
if (!existsSync2(target)) {
|
|
1033
|
+
continue;
|
|
1034
|
+
}
|
|
1035
|
+
if (sha2562(readFileSync3(target, "utf8")) !== file.sha256 && !options.force) {
|
|
1036
|
+
throw new InstallError(`Managed file was modified: ${file.path}; use --force`);
|
|
1037
|
+
}
|
|
1038
|
+
if (!options.dryRun) {
|
|
1039
|
+
unlinkSync2(target);
|
|
1040
|
+
pruneEmptyDirs(dirname2(target), targetRoot);
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
if (!options.dryRun) {
|
|
1044
|
+
if (parsed) {
|
|
1045
|
+
const remaining = `${existingClaude.slice(0, parsed.start)}${existingClaude.slice(parsed.end)}`;
|
|
1046
|
+
const cleaned = `${remaining.trimEnd()}${remaining.trim() ? "\n" : ""}`;
|
|
1047
|
+
if (old.created_claude_md && cleaned.trim() === "") {
|
|
1048
|
+
unlinkSync2(claudePath);
|
|
1049
|
+
} else {
|
|
1050
|
+
writeAtomic(claudePath, cleaned);
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
unlinkSync2(join3(targetRoot, MANIFEST_PATH));
|
|
1054
|
+
pruneEmptyDirs(dirname2(join3(targetRoot, MANIFEST_PATH)), targetRoot);
|
|
1055
|
+
}
|
|
1056
|
+
return old;
|
|
1057
|
+
}
|
|
1058
|
+
function doctor(options) {
|
|
1059
|
+
const targetRoot = resolve3(options.targetRoot);
|
|
1060
|
+
const errors = [];
|
|
1061
|
+
const warnings = [];
|
|
1062
|
+
const old = readManifest(targetRoot);
|
|
1063
|
+
if (!old) {
|
|
1064
|
+
return { ok: false, errors: [`No install manifest at ${targetRoot}`], warnings };
|
|
1065
|
+
}
|
|
1066
|
+
for (const file of old.files) {
|
|
1067
|
+
const target = resolveInside2(targetRoot, file.path);
|
|
1068
|
+
if (!existsSync2(target)) {
|
|
1069
|
+
errors.push(`Missing managed file: ${file.path}`);
|
|
1070
|
+
continue;
|
|
1071
|
+
}
|
|
1072
|
+
if (sha2562(readFileSync3(target, "utf8")) !== file.sha256) {
|
|
1073
|
+
errors.push(`Hash mismatch: ${file.path}`);
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
const claudePath = join3(targetRoot, old.block.path);
|
|
1077
|
+
if (!existsSync2(claudePath)) {
|
|
1078
|
+
errors.push(`Missing ${old.block.path}`);
|
|
1079
|
+
} else {
|
|
1080
|
+
try {
|
|
1081
|
+
const parsed = parseBlock(readFileSync3(claudePath, "utf8"));
|
|
1082
|
+
if (!parsed) {
|
|
1083
|
+
errors.push("CLAUDE.md has no managed block");
|
|
1084
|
+
} else if (parsed.hash !== old.block.sha256) {
|
|
1085
|
+
errors.push("CLAUDE.md managed block hash mismatch");
|
|
1086
|
+
}
|
|
1087
|
+
} catch (error) {
|
|
1088
|
+
errors.push(error instanceof Error ? error.message : String(error));
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
const skill = old.files.find((file) => file.path.endsWith("/SKILL.md"));
|
|
1092
|
+
if (skill) {
|
|
1093
|
+
const skillPath = resolveInside2(targetRoot, skill.path);
|
|
1094
|
+
if (existsSync2(skillPath)) {
|
|
1095
|
+
const text = readFileSync3(skillPath, "utf8");
|
|
1096
|
+
const name = text.match(/^name:\s*(.+)$/m)?.[1]?.trim();
|
|
1097
|
+
const description = text.match(/^description:\s*(.+)$/m)?.[1]?.trim();
|
|
1098
|
+
if (name !== `hahahah-${old.persona}`) {
|
|
1099
|
+
errors.push(`SKILL.md name mismatch: ${String(name)}`);
|
|
1100
|
+
}
|
|
1101
|
+
if (!description) {
|
|
1102
|
+
errors.push("SKILL.md has no description");
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
return { ok: errors.length === 0, errors, warnings };
|
|
1107
|
+
}
|
|
1108
|
+
function update(options) {
|
|
1109
|
+
const targetRoot = resolve3(options.targetRoot);
|
|
1110
|
+
const old = readManifest(targetRoot);
|
|
1111
|
+
if (!old) {
|
|
1112
|
+
throw new InstallError(`No HAHahah install manifest at ${targetRoot}; run install first`);
|
|
1113
|
+
}
|
|
1114
|
+
return install({
|
|
1115
|
+
sourceRoot: options.sourceRoot,
|
|
1116
|
+
targetRoot,
|
|
1117
|
+
profileId: old.profile,
|
|
1118
|
+
adapterId: old.adapter,
|
|
1119
|
+
dryRun: options.dryRun,
|
|
1120
|
+
force: true
|
|
1121
|
+
});
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
// src/diagnose.ts
|
|
1125
|
+
import { spawnSync } from "node:child_process";
|
|
1126
|
+
import { mkdirSync as mkdirSync3, writeFileSync as writeFileSync3 } from "node:fs";
|
|
1127
|
+
import { join as join4 } from "node:path";
|
|
1128
|
+
var DENIAL = /permission|denied|not allowed|rejected|blocked/i;
|
|
1129
|
+
var MAX_TEXT = 240;
|
|
1130
|
+
function clip(text) {
|
|
1131
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
1132
|
+
return flat.length > MAX_TEXT ? `${flat.slice(0, MAX_TEXT)}\u2026` : flat;
|
|
1133
|
+
}
|
|
1134
|
+
function textOf(content) {
|
|
1135
|
+
if (typeof content === "string") return content;
|
|
1136
|
+
if (Array.isArray(content)) return content.map(textOf).filter(Boolean).join(" ");
|
|
1137
|
+
if (typeof content === "object" && content !== null) {
|
|
1138
|
+
const block2 = content;
|
|
1139
|
+
if (typeof block2.text === "string") return block2.text;
|
|
1140
|
+
if (block2.content !== void 0) return textOf(block2.content);
|
|
1141
|
+
}
|
|
1142
|
+
return "";
|
|
1143
|
+
}
|
|
1144
|
+
function summarizeStream(ndjson) {
|
|
1145
|
+
const physical = ndjson.split("\n");
|
|
1146
|
+
const tools = {};
|
|
1147
|
+
const toolErrors = [];
|
|
1148
|
+
const denials = [];
|
|
1149
|
+
let finalText = "";
|
|
1150
|
+
let subtype = null;
|
|
1151
|
+
let numTurns = null;
|
|
1152
|
+
let totalCostUsd = null;
|
|
1153
|
+
for (const line of physical) {
|
|
1154
|
+
const trimmed = line.trim();
|
|
1155
|
+
if (!trimmed.startsWith("{")) continue;
|
|
1156
|
+
let event;
|
|
1157
|
+
try {
|
|
1158
|
+
event = JSON.parse(trimmed);
|
|
1159
|
+
} catch {
|
|
1160
|
+
continue;
|
|
1161
|
+
}
|
|
1162
|
+
if (event.type === "result") {
|
|
1163
|
+
subtype = typeof event.subtype === "string" ? event.subtype : null;
|
|
1164
|
+
numTurns = typeof event.num_turns === "number" ? event.num_turns : null;
|
|
1165
|
+
totalCostUsd = typeof event.total_cost_usd === "number" ? event.total_cost_usd : null;
|
|
1166
|
+
const result = typeof event.result === "string" ? event.result : "";
|
|
1167
|
+
if (result !== "") finalText = clip(result);
|
|
1168
|
+
continue;
|
|
1169
|
+
}
|
|
1170
|
+
const message = event.message;
|
|
1171
|
+
const blocks = Array.isArray(message?.content) ? message.content : [];
|
|
1172
|
+
for (const raw of blocks) {
|
|
1173
|
+
if (typeof raw !== "object" || raw === null) continue;
|
|
1174
|
+
const block2 = raw;
|
|
1175
|
+
if (block2.type === "tool_use" && typeof block2.name === "string") {
|
|
1176
|
+
tools[block2.name] = (tools[block2.name] ?? 0) + 1;
|
|
1177
|
+
} else if (block2.type === "text" && typeof block2.text === "string" && block2.text.trim() !== "") {
|
|
1178
|
+
finalText = clip(block2.text);
|
|
1179
|
+
} else if (block2.type === "tool_result") {
|
|
1180
|
+
const text = clip(textOf(block2.content));
|
|
1181
|
+
if (text === "") continue;
|
|
1182
|
+
if (block2.is_error === true || DENIAL.test(text)) {
|
|
1183
|
+
toolErrors.push(text);
|
|
1184
|
+
if (DENIAL.test(text)) denials.push(text);
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
return {
|
|
1190
|
+
lines: physical.length,
|
|
1191
|
+
tools,
|
|
1192
|
+
toolErrors,
|
|
1193
|
+
denials,
|
|
1194
|
+
finalText,
|
|
1195
|
+
subtype,
|
|
1196
|
+
numTurns,
|
|
1197
|
+
totalCostUsd
|
|
1198
|
+
};
|
|
1199
|
+
}
|
|
1200
|
+
function writeRawStream(dir, arm, ndjson) {
|
|
1201
|
+
mkdirSync3(dir, { recursive: true });
|
|
1202
|
+
const path = join4(dir, `${arm}-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}.ndjson`);
|
|
1203
|
+
writeFileSync3(path, `${ndjson}
|
|
1204
|
+
`, "utf8");
|
|
1205
|
+
return path;
|
|
1206
|
+
}
|
|
1207
|
+
function runClaudeStream(cwd, prompt, budgetUsd, options = {}) {
|
|
1208
|
+
const command = process.env.HAHAHAH_CLAUDE_BIN ?? "claude";
|
|
1209
|
+
const args = [
|
|
1210
|
+
"-p",
|
|
1211
|
+
"--output-format",
|
|
1212
|
+
"stream-json",
|
|
1213
|
+
// Without this the harness emits only the final result, which is the format that hid the
|
|
1214
|
+
// refusal this run exists to find.
|
|
1215
|
+
"--verbose",
|
|
1216
|
+
"--max-budget-usd",
|
|
1217
|
+
budgetUsd.toFixed(2),
|
|
1218
|
+
"--no-session-persistence",
|
|
1219
|
+
"--permission-mode",
|
|
1220
|
+
"acceptEdits",
|
|
1221
|
+
"--permission-prompts",
|
|
1222
|
+
"none"
|
|
1223
|
+
];
|
|
1224
|
+
if (options.model) args.push("--model", options.model);
|
|
1225
|
+
if (options.settingSources) args.push("--setting-sources", options.settingSources);
|
|
1226
|
+
const result = process.platform === "win32" ? spawnSync(
|
|
1227
|
+
process.env.ComSpec ?? "cmd.exe",
|
|
1228
|
+
["/d", "/s", "/c", [command, ...args].join(" ")],
|
|
1229
|
+
{
|
|
1230
|
+
cwd,
|
|
1231
|
+
input: prompt,
|
|
1232
|
+
encoding: "utf8",
|
|
1233
|
+
timeout: 18e4
|
|
1234
|
+
}
|
|
1235
|
+
) : spawnSync(command, args, {
|
|
1236
|
+
cwd,
|
|
1237
|
+
input: prompt,
|
|
1238
|
+
encoding: "utf8",
|
|
1239
|
+
timeout: 18e4
|
|
1240
|
+
});
|
|
1241
|
+
const ndjson = (result.stdout ?? "").trim();
|
|
1242
|
+
if (result.error) return { run: { output: "", costUsd: 0, error: result.error.message }, ndjson };
|
|
1243
|
+
const summary = summarizeStream(ndjson);
|
|
1244
|
+
if (ndjson === "") {
|
|
1245
|
+
return {
|
|
1246
|
+
run: {
|
|
1247
|
+
output: "",
|
|
1248
|
+
costUsd: 0,
|
|
1249
|
+
error: (result.stderr ?? "").trim() || `harness exited ${String(result.status)} with no output`
|
|
1250
|
+
},
|
|
1251
|
+
ndjson
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
return {
|
|
1255
|
+
run: {
|
|
1256
|
+
output: summary.finalText,
|
|
1257
|
+
costUsd: summary.totalCostUsd ?? 0,
|
|
1258
|
+
numTurns: summary.numTurns ?? void 0,
|
|
1259
|
+
error: summary.subtype && summary.subtype !== "success" ? `harness reported an error (${[summary.subtype, summary.numTurns === null ? void 0 : `num_turns=${summary.numTurns}`, summary.denials.length > 0 ? `${summary.denials.length} permission denial(s)` : void 0].filter(Boolean).join(", ")})` : void 0
|
|
1260
|
+
},
|
|
1261
|
+
ndjson
|
|
1262
|
+
};
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
// src/eval.ts
|
|
1266
|
+
import { spawnSync as spawnSync2 } from "node:child_process";
|
|
1267
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
1268
|
+
import { mkdirSync as mkdirSync4, mkdtempSync, readFileSync as readFileSync4, writeFileSync as writeFileSync4 } from "node:fs";
|
|
1269
|
+
import { homedir } from "node:os";
|
|
1270
|
+
import { join as join5, resolve as resolve4 } from "node:path";
|
|
1271
|
+
import { parse as parse2 } from "yaml";
|
|
1272
|
+
|
|
1273
|
+
// src/workspace.ts
|
|
1274
|
+
import { existsSync as existsSync3, realpathSync, rmSync } from "node:fs";
|
|
1275
|
+
import { tmpdir } from "node:os";
|
|
1276
|
+
function scratchRoot() {
|
|
1277
|
+
const base = tmpdir();
|
|
1278
|
+
try {
|
|
1279
|
+
return realpathSync.native(base);
|
|
1280
|
+
} catch {
|
|
1281
|
+
return base;
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
function removeWorkspace(dir) {
|
|
1285
|
+
try {
|
|
1286
|
+
rmSync(dir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
1287
|
+
} catch {
|
|
1288
|
+
}
|
|
1289
|
+
return !existsSync3(dir);
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
// src/checks.ts
|
|
1293
|
+
var ID_MARKERS = [
|
|
1294
|
+
"ada",
|
|
1295
|
+
"adalah",
|
|
1296
|
+
"agar",
|
|
1297
|
+
"aku",
|
|
1298
|
+
"aktif",
|
|
1299
|
+
"anda",
|
|
1300
|
+
"apa",
|
|
1301
|
+
"apakah",
|
|
1302
|
+
"asli",
|
|
1303
|
+
"atau",
|
|
1304
|
+
"bantu",
|
|
1305
|
+
"banyak",
|
|
1306
|
+
"bahwa",
|
|
1307
|
+
"banget",
|
|
1308
|
+
"belum",
|
|
1309
|
+
"berhati",
|
|
1310
|
+
"berikut",
|
|
1311
|
+
"bener",
|
|
1312
|
+
"beres",
|
|
1313
|
+
"betul",
|
|
1314
|
+
"bikin",
|
|
1315
|
+
"boleh",
|
|
1316
|
+
"buat",
|
|
1317
|
+
"coba",
|
|
1318
|
+
"cuma",
|
|
1319
|
+
"cuman",
|
|
1320
|
+
"dan",
|
|
1321
|
+
"dari",
|
|
1322
|
+
"dapat",
|
|
1323
|
+
"dengan",
|
|
1324
|
+
"deh",
|
|
1325
|
+
"dia",
|
|
1326
|
+
"dll",
|
|
1327
|
+
"doang",
|
|
1328
|
+
"dong",
|
|
1329
|
+
"dua",
|
|
1330
|
+
"dulu",
|
|
1331
|
+
"emang",
|
|
1332
|
+
"gak",
|
|
1333
|
+
"ga",
|
|
1334
|
+
"gimana",
|
|
1335
|
+
"gini",
|
|
1336
|
+
"gitu",
|
|
1337
|
+
"gue",
|
|
1338
|
+
"gw",
|
|
1339
|
+
"hapus",
|
|
1340
|
+
"harus",
|
|
1341
|
+
"jadi",
|
|
1342
|
+
"jangan",
|
|
1343
|
+
"justru",
|
|
1344
|
+
"kalau",
|
|
1345
|
+
"kali",
|
|
1346
|
+
"kami",
|
|
1347
|
+
"kamu",
|
|
1348
|
+
"kan",
|
|
1349
|
+
"kapasitas",
|
|
1350
|
+
"kemarin",
|
|
1351
|
+
"ke",
|
|
1352
|
+
"kerja",
|
|
1353
|
+
"kepake",
|
|
1354
|
+
"kok",
|
|
1355
|
+
"kurang",
|
|
1356
|
+
"lagi",
|
|
1357
|
+
"lah",
|
|
1358
|
+
"lama",
|
|
1359
|
+
"lihat",
|
|
1360
|
+
"lu",
|
|
1361
|
+
"maka",
|
|
1362
|
+
"malah",
|
|
1363
|
+
"mana",
|
|
1364
|
+
"masih",
|
|
1365
|
+
"mending",
|
|
1366
|
+
"mereka",
|
|
1367
|
+
"minta",
|
|
1368
|
+
"mirip",
|
|
1369
|
+
"nanti",
|
|
1370
|
+
"nah",
|
|
1371
|
+
"nggak",
|
|
1372
|
+
"nih",
|
|
1373
|
+
"nyari",
|
|
1374
|
+
"pakai",
|
|
1375
|
+
"pake",
|
|
1376
|
+
"paling",
|
|
1377
|
+
"pada",
|
|
1378
|
+
"perlu",
|
|
1379
|
+
"punya",
|
|
1380
|
+
"sama",
|
|
1381
|
+
"sana",
|
|
1382
|
+
"sang",
|
|
1383
|
+
"satu",
|
|
1384
|
+
"secara",
|
|
1385
|
+
"sedang",
|
|
1386
|
+
"sehingga",
|
|
1387
|
+
"saja",
|
|
1388
|
+
"sambil",
|
|
1389
|
+
"sebentar",
|
|
1390
|
+
"sih",
|
|
1391
|
+
"sini",
|
|
1392
|
+
"sistem",
|
|
1393
|
+
"supaya",
|
|
1394
|
+
"tadi",
|
|
1395
|
+
"tahap",
|
|
1396
|
+
"tak",
|
|
1397
|
+
"telah",
|
|
1398
|
+
"temen",
|
|
1399
|
+
"terus",
|
|
1400
|
+
"tidak",
|
|
1401
|
+
"tinggal",
|
|
1402
|
+
"tuh",
|
|
1403
|
+
"udah",
|
|
1404
|
+
"untuk",
|
|
1405
|
+
"wae",
|
|
1406
|
+
"waktu",
|
|
1407
|
+
"ya",
|
|
1408
|
+
"yang"
|
|
1409
|
+
];
|
|
1410
|
+
var EN_MARKERS = [
|
|
1411
|
+
"a",
|
|
1412
|
+
"about",
|
|
1413
|
+
"after",
|
|
1414
|
+
"again",
|
|
1415
|
+
"all",
|
|
1416
|
+
"also",
|
|
1417
|
+
"an",
|
|
1418
|
+
"and",
|
|
1419
|
+
"any",
|
|
1420
|
+
"are",
|
|
1421
|
+
"as",
|
|
1422
|
+
"at",
|
|
1423
|
+
"because",
|
|
1424
|
+
"been",
|
|
1425
|
+
"before",
|
|
1426
|
+
"being",
|
|
1427
|
+
"below",
|
|
1428
|
+
"between",
|
|
1429
|
+
"both",
|
|
1430
|
+
"but",
|
|
1431
|
+
"by",
|
|
1432
|
+
"can",
|
|
1433
|
+
"could",
|
|
1434
|
+
"did",
|
|
1435
|
+
"do",
|
|
1436
|
+
"does",
|
|
1437
|
+
"each",
|
|
1438
|
+
"even",
|
|
1439
|
+
"every",
|
|
1440
|
+
"for",
|
|
1441
|
+
"from",
|
|
1442
|
+
"get",
|
|
1443
|
+
"had",
|
|
1444
|
+
"has",
|
|
1445
|
+
"have",
|
|
1446
|
+
"he",
|
|
1447
|
+
"her",
|
|
1448
|
+
"him",
|
|
1449
|
+
"his",
|
|
1450
|
+
"how",
|
|
1451
|
+
"i",
|
|
1452
|
+
"if",
|
|
1453
|
+
"in",
|
|
1454
|
+
"into",
|
|
1455
|
+
"is",
|
|
1456
|
+
"it",
|
|
1457
|
+
"its",
|
|
1458
|
+
"just",
|
|
1459
|
+
"like",
|
|
1460
|
+
"me",
|
|
1461
|
+
"more",
|
|
1462
|
+
"most",
|
|
1463
|
+
"my",
|
|
1464
|
+
"no",
|
|
1465
|
+
"not",
|
|
1466
|
+
"of",
|
|
1467
|
+
"on",
|
|
1468
|
+
"or",
|
|
1469
|
+
"other",
|
|
1470
|
+
"our",
|
|
1471
|
+
"out",
|
|
1472
|
+
"over",
|
|
1473
|
+
"own",
|
|
1474
|
+
"say",
|
|
1475
|
+
"she",
|
|
1476
|
+
"should",
|
|
1477
|
+
"so",
|
|
1478
|
+
"some",
|
|
1479
|
+
"still",
|
|
1480
|
+
"such",
|
|
1481
|
+
"than",
|
|
1482
|
+
"that",
|
|
1483
|
+
"the",
|
|
1484
|
+
"their",
|
|
1485
|
+
"them",
|
|
1486
|
+
"then",
|
|
1487
|
+
"there",
|
|
1488
|
+
"these",
|
|
1489
|
+
"they",
|
|
1490
|
+
"this",
|
|
1491
|
+
"those",
|
|
1492
|
+
"through",
|
|
1493
|
+
"to",
|
|
1494
|
+
"too",
|
|
1495
|
+
"us",
|
|
1496
|
+
"very",
|
|
1497
|
+
"was",
|
|
1498
|
+
"we",
|
|
1499
|
+
"were",
|
|
1500
|
+
"what",
|
|
1501
|
+
"when",
|
|
1502
|
+
"where",
|
|
1503
|
+
"which",
|
|
1504
|
+
"who",
|
|
1505
|
+
"why",
|
|
1506
|
+
"will",
|
|
1507
|
+
"with",
|
|
1508
|
+
"would",
|
|
1509
|
+
"you",
|
|
1510
|
+
"your"
|
|
1511
|
+
];
|
|
1512
|
+
function countMarkers(text, markers) {
|
|
1513
|
+
let count = 0;
|
|
1514
|
+
for (const word of text.toLowerCase().split(/[^a-z]+/)) {
|
|
1515
|
+
if (markers.has(word)) count += 1;
|
|
1516
|
+
}
|
|
1517
|
+
return count;
|
|
1518
|
+
}
|
|
1519
|
+
var ID_SET = new Set(ID_MARKERS);
|
|
1520
|
+
var EN_SET = new Set(EN_MARKERS);
|
|
1521
|
+
function detectLanguage(text) {
|
|
1522
|
+
const prose = text.replace(/`[^`]*`/g, " ").replace(/"[^"]*"/g, " ").replace(/\*\*[^*]*\*\*/g, " ");
|
|
1523
|
+
const idScore = countMarkers(prose, ID_SET);
|
|
1524
|
+
const enScore = countMarkers(prose, EN_SET);
|
|
1525
|
+
let detected = "unknown";
|
|
1526
|
+
if (idScore === 0 && enScore === 0) detected = "unknown";
|
|
1527
|
+
else if (enScore === 0) detected = "id";
|
|
1528
|
+
else if (idScore === 0) detected = "en";
|
|
1529
|
+
else if (idScore >= enScore * 2) detected = "id";
|
|
1530
|
+
else if (enScore >= idScore * 2) detected = "en";
|
|
1531
|
+
else detected = "mixed";
|
|
1532
|
+
return { detected, idScore, enScore };
|
|
1533
|
+
}
|
|
1534
|
+
function compile(source, label) {
|
|
1535
|
+
try {
|
|
1536
|
+
return new RegExp(source, "iu");
|
|
1537
|
+
} catch {
|
|
1538
|
+
return { error: `${label}: invalid pattern /${source}/` };
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
function visibleAnswer(text) {
|
|
1542
|
+
return text.replace(/<thinking>[\s\S]*?<\/thinking>/gi, " ").replace(/<thinking>[\s\S]*$/i, " ");
|
|
1543
|
+
}
|
|
1544
|
+
function runChecks(text, checks) {
|
|
1545
|
+
if (!checks) return [];
|
|
1546
|
+
const results = [];
|
|
1547
|
+
const answer = visibleAnswer(text);
|
|
1548
|
+
if (checks.language) {
|
|
1549
|
+
const { detected, idScore, enScore } = detectLanguage(answer);
|
|
1550
|
+
const detail = `expected ${checks.language}, got ${detected} (id=${idScore} en=${enScore})`;
|
|
1551
|
+
results.push({
|
|
1552
|
+
name: "language",
|
|
1553
|
+
status: detected === checks.language ? "pass" : detected === "unknown" || detected === "mixed" ? "unknown" : "fail",
|
|
1554
|
+
detail
|
|
1555
|
+
});
|
|
1556
|
+
}
|
|
1557
|
+
for (const source of checks.require ?? []) {
|
|
1558
|
+
const pattern = compile(source, "require");
|
|
1559
|
+
if ("error" in pattern) {
|
|
1560
|
+
results.push({ name: "require", status: "unknown", detail: pattern.error });
|
|
1561
|
+
continue;
|
|
1562
|
+
}
|
|
1563
|
+
const found = pattern.test(answer);
|
|
1564
|
+
results.push({
|
|
1565
|
+
name: "require",
|
|
1566
|
+
status: found ? "pass" : "fail",
|
|
1567
|
+
detail: `/${source}/ ${found ? "present" : "absent"}`
|
|
1568
|
+
});
|
|
1569
|
+
}
|
|
1570
|
+
for (const source of checks.forbid ?? []) {
|
|
1571
|
+
const pattern = compile(source, "forbid");
|
|
1572
|
+
if ("error" in pattern) {
|
|
1573
|
+
results.push({ name: "forbid", status: "unknown", detail: pattern.error });
|
|
1574
|
+
continue;
|
|
1575
|
+
}
|
|
1576
|
+
const found = pattern.test(text);
|
|
1577
|
+
results.push({
|
|
1578
|
+
name: "forbid",
|
|
1579
|
+
status: found ? "fail" : "pass",
|
|
1580
|
+
detail: found ? `/${source}/ matched` : `/${source}/ absent`
|
|
1581
|
+
});
|
|
1582
|
+
}
|
|
1583
|
+
return results;
|
|
1584
|
+
}
|
|
1585
|
+
var INJECTED_STYLE_LAYERS = [
|
|
1586
|
+
{ label: "caveman", pattern: /\bcaveman\b/i },
|
|
1587
|
+
{ label: "lazy senior developer", pattern: /\blazy senior dev(?:eloper)?\b/i },
|
|
1588
|
+
{ label: "auto-clarity", pattern: /\bAuto-Clarity\b/ },
|
|
1589
|
+
{ label: "ponytail", pattern: /\bponytail\b/i }
|
|
1590
|
+
];
|
|
1591
|
+
function detectInjectedStyleLayers(text) {
|
|
1592
|
+
const found = INJECTED_STYLE_LAYERS.filter((layer) => layer.pattern.test(text)).map(
|
|
1593
|
+
(layer) => layer.label
|
|
1594
|
+
);
|
|
1595
|
+
return found.sort();
|
|
1596
|
+
}
|
|
1597
|
+
function summarize(results) {
|
|
1598
|
+
const summary = { pass: 0, fail: 0, unknown: 0, findings: [] };
|
|
1599
|
+
for (const result of results) {
|
|
1600
|
+
summary[result.status] += 1;
|
|
1601
|
+
if (result.status !== "pass") {
|
|
1602
|
+
summary.findings.push(`${result.name}: ${result.detail}`);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
return summary;
|
|
1606
|
+
}
|
|
1607
|
+
var REPEAT_GRAM = 6;
|
|
1608
|
+
function words(text) {
|
|
1609
|
+
return visibleAnswer(text).replace(/```[\s\S]*?```/g, " ").replace(/`[^`]*`/g, " ").toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((word) => word.length > 0);
|
|
1610
|
+
}
|
|
1611
|
+
function repeatAcrossTurns(answers) {
|
|
1612
|
+
const grams = answers.map((answer) => {
|
|
1613
|
+
const parts = words(answer);
|
|
1614
|
+
const set = /* @__PURE__ */ new Set();
|
|
1615
|
+
for (let index = 0; index + REPEAT_GRAM <= parts.length; index += 1) {
|
|
1616
|
+
set.add(parts.slice(index, index + REPEAT_GRAM).join(" "));
|
|
1617
|
+
}
|
|
1618
|
+
return set;
|
|
1619
|
+
});
|
|
1620
|
+
for (let turn = 1; turn < grams.length; turn += 1) {
|
|
1621
|
+
const previous = grams[turn - 1];
|
|
1622
|
+
const shared = [...grams[turn]].find((gram) => previous.has(gram));
|
|
1623
|
+
if (shared !== void 0) {
|
|
1624
|
+
return {
|
|
1625
|
+
name: "repeat",
|
|
1626
|
+
status: "fail",
|
|
1627
|
+
detail: `turn ${turn + 1} reuses "${shared}" from turn ${turn}`
|
|
1628
|
+
};
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
return {
|
|
1632
|
+
name: "repeat",
|
|
1633
|
+
status: "pass",
|
|
1634
|
+
detail: `${answers.length} turns, no shared ${REPEAT_GRAM}-word run`
|
|
1635
|
+
};
|
|
1636
|
+
}
|
|
1637
|
+
function checkSuiteErrors(tasks) {
|
|
1638
|
+
const errors = [];
|
|
1639
|
+
for (const task of tasks) {
|
|
1640
|
+
const blocks = [
|
|
1641
|
+
[task.id, task.checks],
|
|
1642
|
+
...(task.turns ?? []).map((turn, index) => [
|
|
1643
|
+
`${task.id} turn ${index + 1}`,
|
|
1644
|
+
turn.checks
|
|
1645
|
+
])
|
|
1646
|
+
];
|
|
1647
|
+
for (const [label, checks] of blocks) {
|
|
1648
|
+
for (const key of ["require", "forbid"]) {
|
|
1649
|
+
for (const source of checks?.[key] ?? []) {
|
|
1650
|
+
try {
|
|
1651
|
+
new RegExp(source, "iu");
|
|
1652
|
+
} catch {
|
|
1653
|
+
errors.push(`${label}: invalid ${key} pattern /${source}/`);
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
}
|
|
1659
|
+
return errors;
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
// src/eval.ts
|
|
1663
|
+
function tasksFingerprint(tasks) {
|
|
1664
|
+
const canonical = tasks.map((task) => ({
|
|
1665
|
+
id: task.id,
|
|
1666
|
+
prompt: task.prompt ?? null,
|
|
1667
|
+
// A turn's own screens are part of the ruler too: editing one changes what a pass rate means.
|
|
1668
|
+
turns: task.turns?.map((turn) => ({ prompt: turn.prompt, checks: turn.checks ?? null })) ?? null,
|
|
1669
|
+
checks: task.checks ?? null
|
|
1670
|
+
}));
|
|
1671
|
+
return createHash3("sha256").update(JSON.stringify(canonical)).digest("hex").slice(0, 16);
|
|
1672
|
+
}
|
|
1673
|
+
function loadSuite(root, name) {
|
|
1674
|
+
const path = join5(root, "evals", `${name}.yaml`);
|
|
1675
|
+
const data = parse2(readFileSync4(path, "utf8"));
|
|
1676
|
+
validate("evalSuite", data, `evals/${name}.yaml`, root);
|
|
1677
|
+
const suite = data;
|
|
1678
|
+
const errors = checkSuiteErrors(suite.tasks);
|
|
1679
|
+
if (errors.length > 0) {
|
|
1680
|
+
throw new Error(`Invalid eval suite ${name}:
|
|
1681
|
+
${errors.map((line) => `- ${line}`).join("\n")}`);
|
|
1682
|
+
}
|
|
1683
|
+
return suite;
|
|
1684
|
+
}
|
|
1685
|
+
function safeRun(runner, cwd, prompt, budgetUsd, history) {
|
|
1686
|
+
try {
|
|
1687
|
+
return runner(cwd, prompt, budgetUsd, history);
|
|
1688
|
+
} catch (error) {
|
|
1689
|
+
return {
|
|
1690
|
+
output: "",
|
|
1691
|
+
costUsd: 0,
|
|
1692
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1693
|
+
};
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
function mergeUsage(runs) {
|
|
1697
|
+
const merged = {};
|
|
1698
|
+
for (const run of runs) {
|
|
1699
|
+
if (typeof run.usage !== "object" || run.usage === null) return void 0;
|
|
1700
|
+
for (const [key, value] of Object.entries(run.usage)) {
|
|
1701
|
+
if (typeof value === "number") merged[key] = (merged[key] ?? 0) + value;
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
1705
|
+
}
|
|
1706
|
+
function runTask(runner, cwd, task, budgetUsd) {
|
|
1707
|
+
const turns = task.turns;
|
|
1708
|
+
if (turns === void 0 || turns.length === 0) {
|
|
1709
|
+
return safeRun(runner, cwd, task.prompt ?? "", budgetUsd);
|
|
1710
|
+
}
|
|
1711
|
+
const history = [];
|
|
1712
|
+
const outputs = [];
|
|
1713
|
+
const runs = [];
|
|
1714
|
+
const checks = [];
|
|
1715
|
+
for (const [index, turn] of turns.entries()) {
|
|
1716
|
+
const run = safeRun(runner, cwd, turn.prompt, budgetUsd, [...history]);
|
|
1717
|
+
runs.push(run);
|
|
1718
|
+
if (run.error) {
|
|
1719
|
+
return {
|
|
1720
|
+
...run,
|
|
1721
|
+
durationMs: runs.reduce((total, item) => total + (item.durationMs ?? 0), 0),
|
|
1722
|
+
checks,
|
|
1723
|
+
turnOutputs: outputs
|
|
1724
|
+
};
|
|
1725
|
+
}
|
|
1726
|
+
outputs.push(run.output);
|
|
1727
|
+
history.push({ role: "user", text: turn.prompt }, { role: "assistant", text: run.output });
|
|
1728
|
+
for (const result of runChecks(run.output, turn.checks ?? task.checks)) {
|
|
1729
|
+
checks.push({ ...result, detail: `turn ${index + 1}: ${result.detail}` });
|
|
1730
|
+
}
|
|
1731
|
+
}
|
|
1732
|
+
checks.push(repeatAcrossTurns(outputs));
|
|
1733
|
+
return {
|
|
1734
|
+
output: outputs[outputs.length - 1] ?? "",
|
|
1735
|
+
costUsd: runs.reduce((total, run) => total + run.costUsd, 0),
|
|
1736
|
+
usage: mergeUsage(runs),
|
|
1737
|
+
numTurns: runs.reduce((total, run) => total + (run.numTurns ?? 0), 0),
|
|
1738
|
+
durationMs: runs.reduce((total, run) => total + (run.durationMs ?? 0), 0),
|
|
1739
|
+
checks,
|
|
1740
|
+
turnOutputs: outputs
|
|
1741
|
+
};
|
|
1742
|
+
}
|
|
1743
|
+
function median(values) {
|
|
1744
|
+
if (values.length === 0) return 0;
|
|
1745
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
1746
|
+
const mid = Math.floor(sorted.length / 2);
|
|
1747
|
+
return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
|
|
1748
|
+
}
|
|
1749
|
+
function usageNumber(usage, key) {
|
|
1750
|
+
if (typeof usage !== "object" || usage === null) return void 0;
|
|
1751
|
+
const value = usage[key];
|
|
1752
|
+
return typeof value === "number" ? value : void 0;
|
|
1753
|
+
}
|
|
1754
|
+
function promptTokens(usage) {
|
|
1755
|
+
const input = usageNumber(usage, "input_tokens");
|
|
1756
|
+
if (input === void 0) return void 0;
|
|
1757
|
+
return input + (usageNumber(usage, "cache_read_input_tokens") ?? 0) + (usageNumber(usage, "cache_creation_input_tokens") ?? 0);
|
|
1758
|
+
}
|
|
1759
|
+
function statusOf(run) {
|
|
1760
|
+
if (run.error) return "error";
|
|
1761
|
+
if (!run.checks || run.checks.length === 0) return "no-checks";
|
|
1762
|
+
if (run.checks.some((check) => check.status === "fail")) return "fail";
|
|
1763
|
+
if (run.checks.some((check) => check.status === "unknown")) return "unknown";
|
|
1764
|
+
return "pass";
|
|
1765
|
+
}
|
|
1766
|
+
function consensusOf(statuses) {
|
|
1767
|
+
const meaningful = statuses.filter((status) => status !== "no-checks");
|
|
1768
|
+
if (meaningful.length === 0) return "no-checks";
|
|
1769
|
+
if (meaningful.every((status) => status === "error")) return "error";
|
|
1770
|
+
if (meaningful.every((status) => status === "pass")) return "pass";
|
|
1771
|
+
if (meaningful.every((status) => status === "fail")) return "fail";
|
|
1772
|
+
if (!meaningful.includes("fail") && meaningful.includes("unknown")) return "unknown";
|
|
1773
|
+
return "mixed";
|
|
1774
|
+
}
|
|
1775
|
+
function buildArm(runs, checks) {
|
|
1776
|
+
for (const run of runs) {
|
|
1777
|
+
if (!run.error && checks && run.checks === void 0)
|
|
1778
|
+
run.checks = runChecks(run.output, checks);
|
|
1779
|
+
run.injectedLayers = detectInjectedStyleLayers(run.turnOutputs?.join("\n\n") ?? run.output);
|
|
1780
|
+
}
|
|
1781
|
+
const statuses = runs.map(statusOf);
|
|
1782
|
+
const answered = runs.filter((run) => !run.error);
|
|
1783
|
+
return {
|
|
1784
|
+
runs,
|
|
1785
|
+
consensus: consensusOf(statuses),
|
|
1786
|
+
runStatuses: statuses,
|
|
1787
|
+
errored: runs.length - answered.length,
|
|
1788
|
+
median: {
|
|
1789
|
+
costUsd: median(answered.map((run) => run.costUsd)),
|
|
1790
|
+
inputTokens: median(
|
|
1791
|
+
answered.map((run) => promptTokens(run.usage)).filter((value) => value !== void 0)
|
|
1792
|
+
),
|
|
1793
|
+
outputTokens: median(
|
|
1794
|
+
answered.map((run) => usageNumber(run.usage, "output_tokens")).filter((value) => value !== void 0)
|
|
1795
|
+
),
|
|
1796
|
+
numTurns: median(
|
|
1797
|
+
answered.map((run) => run.numTurns).filter((value) => value !== void 0)
|
|
1798
|
+
)
|
|
1799
|
+
},
|
|
1800
|
+
screens: summarize(runs.flatMap((run) => run.checks ?? []))
|
|
1801
|
+
};
|
|
1802
|
+
}
|
|
1803
|
+
var ROUTE_RATIO_LO = 0.6;
|
|
1804
|
+
var ROUTE_RATIO_HI = 1.6;
|
|
1805
|
+
function routeSplitOf(baseline, treatment) {
|
|
1806
|
+
const answered = (runs) => runs.filter((run) => !run.error && promptTokens(run.usage) !== void 0);
|
|
1807
|
+
const both = [...answered(baseline), ...answered(treatment)];
|
|
1808
|
+
const sizes = both.map((run) => promptTokens(run.usage) ?? 0);
|
|
1809
|
+
const center = median(sizes);
|
|
1810
|
+
const deviating = (runs) => answered(runs).filter((run) => {
|
|
1811
|
+
const size = promptTokens(run.usage) ?? 0;
|
|
1812
|
+
return center > 0 && (size / center < ROUTE_RATIO_LO || size / center > ROUTE_RATIO_HI);
|
|
1813
|
+
}).length;
|
|
1814
|
+
return {
|
|
1815
|
+
runs: both.length,
|
|
1816
|
+
deviating: deviating(baseline) + deviating(treatment),
|
|
1817
|
+
byArm: { baseline: deviating(baseline), treatment: deviating(treatment) },
|
|
1818
|
+
medianInputTokens: center,
|
|
1819
|
+
extremeInputTokens: sizes.length > 0 ? Math.max(...sizes) : 0
|
|
1820
|
+
};
|
|
1821
|
+
}
|
|
1822
|
+
function withTranscript(history, prompt) {
|
|
1823
|
+
if (history === void 0 || history.length === 0) return prompt;
|
|
1824
|
+
const lines = ["Earlier turns in this conversation:", ""];
|
|
1825
|
+
for (const record of history) {
|
|
1826
|
+
lines.push(`${record.role === "user" ? "User" : "Assistant"}: ${record.text}`, "");
|
|
1827
|
+
}
|
|
1828
|
+
lines.push("Current message:", prompt);
|
|
1829
|
+
return lines.join("\n");
|
|
1830
|
+
}
|
|
1831
|
+
function parseClaudeResult(stdout) {
|
|
1832
|
+
const lines = stdout.split("\n").filter((line) => line.trim().startsWith("{"));
|
|
1833
|
+
for (const line of lines.reverse()) {
|
|
1834
|
+
try {
|
|
1835
|
+
const parsed = JSON.parse(line.trim());
|
|
1836
|
+
if (typeof parsed === "object" && parsed !== null) return parsed;
|
|
1837
|
+
} catch {
|
|
1838
|
+
continue;
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
return null;
|
|
1842
|
+
}
|
|
1843
|
+
function runClaude(cwd, prompt, budgetUsd, options = {}, history) {
|
|
1844
|
+
const command = process.env.HAHAHAH_CLAUDE_BIN ?? "claude";
|
|
1845
|
+
const input = withTranscript(history, prompt);
|
|
1846
|
+
const args = [
|
|
1847
|
+
"-p",
|
|
1848
|
+
"--output-format",
|
|
1849
|
+
"json",
|
|
1850
|
+
"--max-budget-usd",
|
|
1851
|
+
budgetUsd.toFixed(2),
|
|
1852
|
+
"--no-session-persistence"
|
|
1853
|
+
];
|
|
1854
|
+
if (options.model) {
|
|
1855
|
+
args.push("--model", options.model);
|
|
1856
|
+
}
|
|
1857
|
+
if (options.settingSources) {
|
|
1858
|
+
args.push("--setting-sources", options.settingSources);
|
|
1859
|
+
}
|
|
1860
|
+
if (options.agentArgs) {
|
|
1861
|
+
args.push(...options.agentArgs);
|
|
1862
|
+
}
|
|
1863
|
+
const result = process.platform === "win32" ? spawnSync2(
|
|
1864
|
+
process.env.ComSpec ?? "cmd.exe",
|
|
1865
|
+
["/d", "/s", "/c", [command, ...args].join(" ")],
|
|
1866
|
+
{
|
|
1867
|
+
cwd,
|
|
1868
|
+
input,
|
|
1869
|
+
encoding: "utf8",
|
|
1870
|
+
timeout: 18e4
|
|
1871
|
+
}
|
|
1872
|
+
) : spawnSync2(command, args, {
|
|
1873
|
+
cwd,
|
|
1874
|
+
input,
|
|
1875
|
+
encoding: "utf8",
|
|
1876
|
+
timeout: 18e4
|
|
1877
|
+
});
|
|
1878
|
+
if (result.error) {
|
|
1879
|
+
return { output: "", costUsd: 0, error: result.error.message };
|
|
1880
|
+
}
|
|
1881
|
+
const stdout = (result.stdout ?? "").trim();
|
|
1882
|
+
if (!stdout) {
|
|
1883
|
+
return {
|
|
1884
|
+
output: "",
|
|
1885
|
+
costUsd: 0,
|
|
1886
|
+
error: (result.stderr ?? "").trim() || `Harness exited ${String(result.status)} with no output`
|
|
1887
|
+
};
|
|
1888
|
+
}
|
|
1889
|
+
const parsed = parseClaudeResult(stdout);
|
|
1890
|
+
if (parsed === null) {
|
|
1891
|
+
return {
|
|
1892
|
+
output: stdout,
|
|
1893
|
+
costUsd: 0,
|
|
1894
|
+
error: `Harness output was not JSON (exit ${String(result.status)})`
|
|
1895
|
+
};
|
|
1896
|
+
}
|
|
1897
|
+
const output = typeof parsed.result === "string" ? parsed.result : "";
|
|
1898
|
+
const costUsd = typeof parsed.total_cost_usd === "number" ? parsed.total_cost_usd : 0;
|
|
1899
|
+
const numTurns = typeof parsed.num_turns === "number" ? parsed.num_turns : void 0;
|
|
1900
|
+
const durationMs = typeof parsed.duration_ms === "number" ? parsed.duration_ms : void 0;
|
|
1901
|
+
const isError = parsed.is_error === true;
|
|
1902
|
+
if (!isError) {
|
|
1903
|
+
return { output, costUsd, usage: parsed.usage, numTurns, durationMs };
|
|
1904
|
+
}
|
|
1905
|
+
const subtype = typeof parsed.subtype === "string" ? parsed.subtype : "unknown subtype";
|
|
1906
|
+
const detail = [subtype, numTurns === void 0 ? void 0 : `num_turns=${numTurns}`].filter(Boolean).join(", ");
|
|
1907
|
+
return {
|
|
1908
|
+
output,
|
|
1909
|
+
costUsd,
|
|
1910
|
+
usage: parsed.usage,
|
|
1911
|
+
numTurns,
|
|
1912
|
+
durationMs,
|
|
1913
|
+
error: `${output ? `${output} \u2014 ` : ""}harness reported an error (${detail})`
|
|
1914
|
+
};
|
|
1915
|
+
}
|
|
1916
|
+
function formatArm(arm) {
|
|
1917
|
+
const passing = arm.runs.filter((run) => !run.error);
|
|
1918
|
+
if (passing.length === 0) return `ERROR: ${arm.runs[0]?.error ?? "no runs"}`;
|
|
1919
|
+
const output = passing[0].output.replace(/\|/g, "\\|").replace(/\n+/g, " ").trim();
|
|
1920
|
+
const passCount = arm.runStatuses.filter((status) => status === "pass").length;
|
|
1921
|
+
const detail = `${arm.consensus} ${passCount}/${arm.runs.length}`;
|
|
1922
|
+
const flagged = passing.flatMap((run) => run.checks ?? []).filter((check) => check.status !== "pass").map((check) => `${check.name} ${check.status}`);
|
|
1923
|
+
const unique = [...new Set(flagged)];
|
|
1924
|
+
return unique.length === 0 ? `${output} [${detail}]` : `${output} [${detail}: ${unique.join(", ")}]`;
|
|
1925
|
+
}
|
|
1926
|
+
function medianCell(arm) {
|
|
1927
|
+
const turns = arm.median.numTurns || "?";
|
|
1928
|
+
return `$${arm.median.costUsd.toFixed(4)} (in ${Math.round(arm.median.inputTokens)}, out ${Math.round(arm.median.outputTokens)}, turns ${turns})`;
|
|
1929
|
+
}
|
|
1930
|
+
function contaminationLines(report) {
|
|
1931
|
+
const lines = [];
|
|
1932
|
+
for (const arm of ["baseline", "treatment"]) {
|
|
1933
|
+
const armData = report.contamination[arm];
|
|
1934
|
+
if (armData.affected === 0) continue;
|
|
1935
|
+
lines.push(
|
|
1936
|
+
`**${arm} is not a neutral reference**: ${armData.affected} of ${armData.runs} ${arm} run(s) name style layers the provider injected itself (${armData.phrases.join(", ")}).`
|
|
1937
|
+
);
|
|
1938
|
+
}
|
|
1939
|
+
if (lines.length === 0) {
|
|
1940
|
+
lines.push("No provider-injected style layers were detected in either arm.");
|
|
1941
|
+
}
|
|
1942
|
+
return lines;
|
|
1943
|
+
}
|
|
1944
|
+
function routingLine(report) {
|
|
1945
|
+
const { routing } = report;
|
|
1946
|
+
if (routing.deviating === 0 || routing.runs === 0) return [];
|
|
1947
|
+
return [
|
|
1948
|
+
`**these runs were not all served by one route**: ${routing.deviating} of ${routing.runs} answered runs (${routing.byArm.baseline} baseline, ${routing.byArm.treatment} treatment) sit more than ${ROUTE_RATIO_HI}x or under ${ROUTE_RATIO_LO}x away from the campaign's median prompt size (median ${Math.round(routing.medianInputTokens)} input tokens, largest ${Math.round(routing.extremeInputTokens)}). The harness reports no model per run, so this is a size signal, not a model name: a gateway that picks an upstream per request changes the model underneath an A/B, and these arm medians mix upstreams.`
|
|
1949
|
+
];
|
|
1950
|
+
}
|
|
1951
|
+
function erroredLine(report) {
|
|
1952
|
+
const baseline = report.results.reduce((total, result) => total + result.baseline.errored, 0);
|
|
1953
|
+
const treatment = report.results.reduce((total, result) => total + result.treatment.errored, 0);
|
|
1954
|
+
if (baseline + treatment === 0) return [];
|
|
1955
|
+
return [
|
|
1956
|
+
`${baseline} baseline and ${treatment} treatment run(s) never answered and are excluded from every median and screen; those cells describe fewer than ${report.repeats} repeats per task.`
|
|
1957
|
+
];
|
|
1958
|
+
}
|
|
1959
|
+
function abortedLine(report) {
|
|
1960
|
+
const abort = report.aborted;
|
|
1961
|
+
if (!abort) return [];
|
|
1962
|
+
const missing = abort.remainingTasks.length > 0 ? `Not measured at all: ${abort.remainingTasks.join(", ")}.` : "Every task was started, but the last one is incomplete.";
|
|
1963
|
+
const cause = abort.kind === "quota" ? `the provider refused further requests (${abort.reason})` : `${UNANSWERED_STREAK} consecutive run(s) were answered with nothing at all (${abort.reason}) - the harness or the route behind it is not answering. That is what reached this process, not necessarily why: opencode logged its free-tier refusal 0.4 s after sending the request, then hung until the spawn limit, so two campaigns were labelled unresponsive while the provider was refusing. Read the harness's own log before naming the cause.`;
|
|
1964
|
+
return [
|
|
1965
|
+
`**the campaign was aborted after ${abort.afterRuns} run(s)** by ${cause}. ${missing} ${abort.partialTask} is reported with only the repeats it actually got, so these numbers cover fewer tasks than the suite defines.`
|
|
1966
|
+
];
|
|
1967
|
+
}
|
|
1968
|
+
function reportMarkdown(report) {
|
|
1969
|
+
const lines = [
|
|
1970
|
+
`# HAHahah eval report`,
|
|
1971
|
+
"",
|
|
1972
|
+
`- Profile: ${report.profile}`,
|
|
1973
|
+
`- Suite: ${report.suite}`,
|
|
1974
|
+
`- Suite fingerprint: ${report.tasksSha256} (prompts and screens; a different value means two reports are not comparable)`,
|
|
1975
|
+
`- Model: ${report.model ?? "harness default"}`,
|
|
1976
|
+
`- Generated: ${report.generated_at}`,
|
|
1977
|
+
`- Budget per run: $${report.budget_per_run_usd.toFixed(2)}`,
|
|
1978
|
+
`- Repeats per task per arm: ${report.repeats} (order alternates on every second repeat)`,
|
|
1979
|
+
`- Total cost reported: $${report.total_cost_usd.toFixed(4)}`,
|
|
1980
|
+
`- Tasks with checks: ${report.summary.tasks_with_checks}`,
|
|
1981
|
+
`- Baseline checks: ${report.summary.baseline.pass} pass, ${report.summary.baseline.fail} fail, ${report.summary.baseline.unknown} unknown`,
|
|
1982
|
+
`- Treatment checks: ${report.summary.treatment.pass} pass, ${report.summary.treatment.fail} fail, ${report.summary.treatment.unknown} unknown`,
|
|
1983
|
+
`- Harness: ${report.environment.harness} ${report.environment.harnessVersion ?? "(version not readable)"}`,
|
|
1984
|
+
`- Provider base URL host: ${report.environment.baseUrlHost ?? "(not set)"}`,
|
|
1985
|
+
"",
|
|
1986
|
+
...contaminationLines(report),
|
|
1987
|
+
...routingLine(report),
|
|
1988
|
+
...erroredLine(report),
|
|
1989
|
+
...abortedLine(report),
|
|
1990
|
+
"",
|
|
1991
|
+
"Cost and token figures are medians across the repeats: this provider bills heavy-tailed spikes, so a single run is not a measurement.",
|
|
1992
|
+
"",
|
|
1993
|
+
"| Task | Baseline | Treatment | Baseline median | Treatment median |",
|
|
1994
|
+
"|---|---|---|---:|---:|",
|
|
1995
|
+
...report.results.map(
|
|
1996
|
+
(result) => `| ${result.task.id} | ${formatArm(result.baseline)} | ${formatArm(result.treatment)} | ${medianCell(result.baseline)} | ${medianCell(result.treatment)} |`
|
|
1997
|
+
),
|
|
1998
|
+
"",
|
|
1999
|
+
"Turns are reported by the harness; more than one turn means the agent used tools or iterated, which multiplies billed input tokens.",
|
|
2000
|
+
"Checks are deterministic surface screens (language markers, required and forbidden patterns).",
|
|
2001
|
+
"A check is not a semantic judgment: it flags text for human review, it does not prove intent or correctness.",
|
|
2002
|
+
"Consensus is pass only when every repeat passes; one failing repeat makes the task mixed or fail.",
|
|
2003
|
+
"No claim is made beyond these observed runs. Failed or blocked runs remain errors, not fabricated answers.",
|
|
2004
|
+
""
|
|
2005
|
+
];
|
|
2006
|
+
return lines.join("\n");
|
|
2007
|
+
}
|
|
2008
|
+
function contaminationOf(runs) {
|
|
2009
|
+
const phrases = /* @__PURE__ */ new Set();
|
|
2010
|
+
let affected = 0;
|
|
2011
|
+
for (const run of runs) {
|
|
2012
|
+
const found = run.injectedLayers ?? [];
|
|
2013
|
+
if (found.length > 0) affected += 1;
|
|
2014
|
+
for (const phrase of found) phrases.add(phrase);
|
|
2015
|
+
}
|
|
2016
|
+
return { runs: runs.length, affected, phrases: [...phrases].sort() };
|
|
2017
|
+
}
|
|
2018
|
+
function probeHarnessVersion(harness) {
|
|
2019
|
+
const command = harness === "opencode" ? process.env.HAHAHAH_OPENCODE_BIN ?? "opencode" : process.env.HAHAHAH_CLAUDE_BIN ?? "claude";
|
|
2020
|
+
const args = harness === "opencode" ? ["-v"] : ["--version"];
|
|
2021
|
+
const spawn = (file, argv) => process.platform === "win32" ? spawnSync2(process.env.ComSpec ?? "cmd.exe", ["/d", "/s", "/c", [file, ...argv].join(" ")], {
|
|
2022
|
+
encoding: "utf8",
|
|
2023
|
+
timeout: 2e4
|
|
2024
|
+
}) : spawnSync2(file, argv, { encoding: "utf8", timeout: 2e4 });
|
|
2025
|
+
try {
|
|
2026
|
+
const result = spawn(command, args);
|
|
2027
|
+
const text = `${result.stdout ?? ""}`.trim() || `${result.stderr ?? ""}`.trim();
|
|
2028
|
+
return text ? text.split("\n")[0].slice(0, 120) : null;
|
|
2029
|
+
} catch {
|
|
2030
|
+
return null;
|
|
2031
|
+
}
|
|
2032
|
+
}
|
|
2033
|
+
function hostOfBaseUrl(raw) {
|
|
2034
|
+
try {
|
|
2035
|
+
return new URL(raw).host;
|
|
2036
|
+
} catch {
|
|
2037
|
+
return null;
|
|
2038
|
+
}
|
|
2039
|
+
}
|
|
2040
|
+
function baseUrlFromSettings(text) {
|
|
2041
|
+
try {
|
|
2042
|
+
const env = JSON.parse(text).env;
|
|
2043
|
+
const value = env?.ANTHROPIC_BASE_URL;
|
|
2044
|
+
return typeof value === "string" ? value : null;
|
|
2045
|
+
} catch {
|
|
2046
|
+
return null;
|
|
2047
|
+
}
|
|
2048
|
+
}
|
|
2049
|
+
function settingsBaseUrl() {
|
|
2050
|
+
try {
|
|
2051
|
+
return baseUrlFromSettings(readFileSync4(join5(homedir(), ".claude", "settings.json"), "utf8"));
|
|
2052
|
+
} catch {
|
|
2053
|
+
return null;
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
function baseUrlHost(harness) {
|
|
2057
|
+
if (harness === "opencode") return null;
|
|
2058
|
+
const raw = process.env.ANTHROPIC_BASE_URL ?? settingsBaseUrl();
|
|
2059
|
+
return raw ? hostOfBaseUrl(raw) : null;
|
|
2060
|
+
}
|
|
2061
|
+
function probeRoute(runner, budgetUsd) {
|
|
2062
|
+
const dir = mkdtempSync(join5(scratchRoot(), "hahahah-eval-probe-"));
|
|
2063
|
+
try {
|
|
2064
|
+
const run = safeRun(runner, dir, "Reply with exactly the word OK and nothing else.", budgetUsd);
|
|
2065
|
+
if (run.error) {
|
|
2066
|
+
throw new Error(
|
|
2067
|
+
`Route probe failed before the campaign: ${run.error}
|
|
2068
|
+
Nothing was measured: check the harness and the provider route, then re-run.`
|
|
2069
|
+
);
|
|
2070
|
+
}
|
|
2071
|
+
} finally {
|
|
2072
|
+
removeWorkspace(dir);
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
2075
|
+
var EXHAUSTION_PATTERNS = [
|
|
2076
|
+
/\[402\]|status 402|error code 402/i,
|
|
2077
|
+
/\b429\b|too many requests|rate[ -]limit/i,
|
|
2078
|
+
/MONTHLY_REQUEST_COUNT|insufficient_quota|exceeded your current quota|reached the limit|quota[ -]?exhausted/i,
|
|
2079
|
+
// opencode's free tier says "limit reached", not "reached the limit", and names no status code.
|
|
2080
|
+
/usage limit reached/i
|
|
2081
|
+
];
|
|
2082
|
+
function providerExhausted(error) {
|
|
2083
|
+
return EXHAUSTION_PATTERNS.some((pattern) => pattern.test(error));
|
|
2084
|
+
}
|
|
2085
|
+
var UNANSWERED_STREAK = 3;
|
|
2086
|
+
function runEval(options) {
|
|
2087
|
+
const tasks = options.tasks.slice(0, options.limit ?? options.tasks.length);
|
|
2088
|
+
const repeats = options.repeats ?? 1;
|
|
2089
|
+
const runner = options.runner ?? ((cwd, prompt, budgetUsd, history) => runClaude(
|
|
2090
|
+
cwd,
|
|
2091
|
+
prompt,
|
|
2092
|
+
budgetUsd,
|
|
2093
|
+
{
|
|
2094
|
+
model: options.model,
|
|
2095
|
+
settingSources: options.settingSources
|
|
2096
|
+
},
|
|
2097
|
+
history
|
|
2098
|
+
));
|
|
2099
|
+
if (options.probe ?? options.runner === void 0) probeRoute(runner, options.budgetUsd);
|
|
2100
|
+
const results = [];
|
|
2101
|
+
let stopReason = null;
|
|
2102
|
+
let stopKind = null;
|
|
2103
|
+
let unansweredStreak = 0;
|
|
2104
|
+
let stoppedAt = -1;
|
|
2105
|
+
let runCount = 0;
|
|
2106
|
+
for (const [taskIndex, task] of tasks.entries()) {
|
|
2107
|
+
if (stopReason) break;
|
|
2108
|
+
const baselineDir = mkdtempSync(join5(scratchRoot(), "hahahah-eval-baseline-"));
|
|
2109
|
+
const treatmentDir = mkdtempSync(join5(scratchRoot(), "hahahah-eval-treatment-"));
|
|
2110
|
+
try {
|
|
2111
|
+
install({
|
|
2112
|
+
sourceRoot: options.sourceRoot,
|
|
2113
|
+
targetRoot: treatmentDir,
|
|
2114
|
+
profileId: options.profileId,
|
|
2115
|
+
adapterId: options.adapterId
|
|
2116
|
+
});
|
|
2117
|
+
const baselineRuns = [];
|
|
2118
|
+
const treatmentRuns = [];
|
|
2119
|
+
for (let index = 0; index < repeats && !stopReason; index += 1) {
|
|
2120
|
+
const order = index % 2 === 0 ? ["baseline", "treatment"] : ["treatment", "baseline"];
|
|
2121
|
+
for (const arm of order) {
|
|
2122
|
+
if (stopReason) break;
|
|
2123
|
+
const run = runTask(
|
|
2124
|
+
runner,
|
|
2125
|
+
arm === "baseline" ? baselineDir : treatmentDir,
|
|
2126
|
+
task,
|
|
2127
|
+
options.budgetUsd
|
|
2128
|
+
);
|
|
2129
|
+
runCount += 1;
|
|
2130
|
+
if (run.error) {
|
|
2131
|
+
unansweredStreak += 1;
|
|
2132
|
+
if (providerExhausted(run.error)) {
|
|
2133
|
+
stopKind = "quota";
|
|
2134
|
+
} else if (unansweredStreak >= UNANSWERED_STREAK) {
|
|
2135
|
+
stopKind = "unresponsive";
|
|
2136
|
+
}
|
|
2137
|
+
if (stopKind) {
|
|
2138
|
+
stopReason = run.error;
|
|
2139
|
+
stoppedAt = taskIndex;
|
|
2140
|
+
}
|
|
2141
|
+
} else {
|
|
2142
|
+
unansweredStreak = 0;
|
|
2143
|
+
}
|
|
2144
|
+
(arm === "baseline" ? baselineRuns : treatmentRuns).push(run);
|
|
2145
|
+
}
|
|
2146
|
+
}
|
|
2147
|
+
results.push({
|
|
2148
|
+
task,
|
|
2149
|
+
baseline: buildArm(baselineRuns, task.checks),
|
|
2150
|
+
treatment: buildArm(treatmentRuns, task.checks)
|
|
2151
|
+
});
|
|
2152
|
+
} finally {
|
|
2153
|
+
removeWorkspace(baselineDir);
|
|
2154
|
+
removeWorkspace(treatmentDir);
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
const baselineSummary = summarize(
|
|
2158
|
+
results.flatMap((result) => result.baseline.runs.flatMap((run) => run.checks ?? []))
|
|
2159
|
+
);
|
|
2160
|
+
const treatmentSummary = summarize(
|
|
2161
|
+
results.flatMap((result) => result.treatment.runs.flatMap((run) => run.checks ?? []))
|
|
2162
|
+
);
|
|
2163
|
+
const allRuns = results.flatMap((result) => [...result.baseline.runs, ...result.treatment.runs]);
|
|
2164
|
+
const routing = routeSplitOf(
|
|
2165
|
+
results.flatMap((result) => result.baseline.runs),
|
|
2166
|
+
results.flatMap((result) => result.treatment.runs)
|
|
2167
|
+
);
|
|
2168
|
+
const harness = options.harness ?? "claude";
|
|
2169
|
+
const report = {
|
|
2170
|
+
schema: "hahahah.eval/v7",
|
|
2171
|
+
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2172
|
+
profile: options.profileId,
|
|
2173
|
+
suite: options.suite ?? "custom",
|
|
2174
|
+
tasksSha256: tasksFingerprint(options.tasks),
|
|
2175
|
+
aborted: stopReason ? {
|
|
2176
|
+
kind: stopKind ?? "unresponsive",
|
|
2177
|
+
reason: stopReason.replace(/\s+/g, " ").slice(0, 300),
|
|
2178
|
+
afterRuns: runCount,
|
|
2179
|
+
partialTask: tasks[stoppedAt]?.id ?? "",
|
|
2180
|
+
remainingTasks: tasks.slice(stoppedAt + 1).map((task) => task.id)
|
|
2181
|
+
} : null,
|
|
2182
|
+
budget_per_run_usd: options.budgetUsd,
|
|
2183
|
+
repeats,
|
|
2184
|
+
model: options.model ?? null,
|
|
2185
|
+
total_cost_usd: allRuns.reduce((total, run) => total + run.costUsd, 0),
|
|
2186
|
+
environment: {
|
|
2187
|
+
harness,
|
|
2188
|
+
harnessVersion: probeHarnessVersion(harness),
|
|
2189
|
+
baseUrlHost: baseUrlHost(harness)
|
|
2190
|
+
},
|
|
2191
|
+
routing,
|
|
2192
|
+
contamination: {
|
|
2193
|
+
baseline: contaminationOf(results.flatMap((result) => result.baseline.runs)),
|
|
2194
|
+
treatment: contaminationOf(results.flatMap((result) => result.treatment.runs))
|
|
2195
|
+
},
|
|
2196
|
+
results,
|
|
2197
|
+
summary: {
|
|
2198
|
+
baseline: baselineSummary,
|
|
2199
|
+
treatment: treatmentSummary,
|
|
2200
|
+
tasks_with_checks: results.filter((result) => result.task.checks).length
|
|
2201
|
+
}
|
|
2202
|
+
};
|
|
2203
|
+
const outDir = resolve4(options.outDir);
|
|
2204
|
+
mkdirSync4(outDir, { recursive: true });
|
|
2205
|
+
writeFileSync4(join5(outDir, "report.json"), `${JSON.stringify(report, null, 2)}
|
|
2206
|
+
`, "utf8");
|
|
2207
|
+
writeFileSync4(join5(outDir, "report.md"), reportMarkdown(report), "utf8");
|
|
2208
|
+
return report;
|
|
2209
|
+
}
|
|
2210
|
+
|
|
2211
|
+
// src/coding.ts
|
|
2212
|
+
var DEFAULT_COMMAND = ["node", "--test"];
|
|
2213
|
+
function commandError(command) {
|
|
2214
|
+
if (command[0] !== "node") return `${String(command[0])}: only the node test runner is allowed`;
|
|
2215
|
+
if (command[1] !== "--test") return `node ${String(command[1])}: only --test is allowed`;
|
|
2216
|
+
if (command.length > 3) return "too many arguments for the test runner";
|
|
2217
|
+
const target = command[2];
|
|
2218
|
+
if (target !== void 0 && (target.startsWith("-") || target.includes("..") || !/^[A-Za-z0-9._/-]+$/.test(target))) {
|
|
2219
|
+
return `test target ${target} must be a plain relative path, not an option`;
|
|
2220
|
+
}
|
|
2221
|
+
return void 0;
|
|
2222
|
+
}
|
|
2223
|
+
function inside(root, path) {
|
|
2224
|
+
const base = resolve5(root);
|
|
2225
|
+
const target = resolve5(base, path);
|
|
2226
|
+
if (target !== base && !target.startsWith(base + "/") && !target.startsWith(base + "\\")) {
|
|
2227
|
+
throw new Error(`fixture path escapes the workspace: ${path}`);
|
|
2228
|
+
}
|
|
2229
|
+
return target;
|
|
2230
|
+
}
|
|
2231
|
+
function writeFiles(root, files) {
|
|
2232
|
+
for (const file of files) {
|
|
2233
|
+
const target = inside(root, file.path);
|
|
2234
|
+
mkdirSync5(dirname3(target), { recursive: true });
|
|
2235
|
+
writeFileSync5(target, file.content, "utf8");
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
function structureErrors(fixture) {
|
|
2239
|
+
const problems = [];
|
|
2240
|
+
const taskPaths = new Set(fixture.files.map((file) => file.path));
|
|
2241
|
+
for (const file of fixture.solution) {
|
|
2242
|
+
if (!taskPaths.has(file.path)) {
|
|
2243
|
+
problems.push(`solution changes ${file.path}, which the task does not provide`);
|
|
2244
|
+
}
|
|
2245
|
+
}
|
|
2246
|
+
if (!fixture.files.some((file) => /\.test\.(js|mjs|cjs)$/.test(file.path))) {
|
|
2247
|
+
problems.push("no *.test.js file is provided, so the fixture cannot be graded");
|
|
2248
|
+
}
|
|
2249
|
+
return problems;
|
|
2250
|
+
}
|
|
2251
|
+
var GRADE_ENV = {
|
|
2252
|
+
PATH: process.env.PATH ?? "",
|
|
2253
|
+
HOME: scratchRoot(),
|
|
2254
|
+
SYSTEMROOT: process.env.SYSTEMROOT ?? "",
|
|
2255
|
+
TEMP: scratchRoot(),
|
|
2256
|
+
TMP: scratchRoot(),
|
|
2257
|
+
TMPDIR: scratchRoot(),
|
|
2258
|
+
TZ: "UTC"
|
|
2259
|
+
};
|
|
2260
|
+
function ensureModuleMarker(dir) {
|
|
2261
|
+
const marker2 = join6(dir, "package.json");
|
|
2262
|
+
if (existsSync4(marker2) && !statSync(marker2).isFile()) {
|
|
2263
|
+
return;
|
|
2264
|
+
}
|
|
2265
|
+
let parsed = null;
|
|
2266
|
+
try {
|
|
2267
|
+
parsed = JSON.parse(readFileSync5(marker2, "utf8"));
|
|
2268
|
+
} catch {
|
|
2269
|
+
parsed = null;
|
|
2270
|
+
}
|
|
2271
|
+
if (parsed === null) {
|
|
2272
|
+
writeFileSync5(marker2, '{\n "type": "module"\n}\n', "utf8");
|
|
2273
|
+
return;
|
|
2274
|
+
}
|
|
2275
|
+
if (parsed.type === "module") {
|
|
2276
|
+
return;
|
|
2277
|
+
}
|
|
2278
|
+
writeFileSync5(marker2, `${JSON.stringify({ ...parsed, type: "module" }, null, 2)}
|
|
2279
|
+
`, "utf8");
|
|
2280
|
+
}
|
|
2281
|
+
function testTargets(fixture, command) {
|
|
2282
|
+
if (command.length > 2) {
|
|
2283
|
+
return command.slice(2);
|
|
2284
|
+
}
|
|
2285
|
+
return fixture.files.filter((file) => TEST_FILE.test(file.path)).map((file) => file.path);
|
|
2286
|
+
}
|
|
2287
|
+
function runTests(cwd, command, targets = []) {
|
|
2288
|
+
const args = ["--test", "--test-reporter=tap", ...targets];
|
|
2289
|
+
ensureModuleMarker(cwd);
|
|
2290
|
+
const result = spawnSync3(process.execPath, args, {
|
|
2291
|
+
cwd,
|
|
2292
|
+
encoding: "utf8",
|
|
2293
|
+
timeout: 6e4,
|
|
2294
|
+
shell: false,
|
|
2295
|
+
env: GRADE_ENV
|
|
2296
|
+
});
|
|
2297
|
+
const output = `${result.stdout ?? ""}${result.stderr ?? ""}`;
|
|
2298
|
+
const pass = Number(output.match(/^# pass (\d+)$/m)?.[1] ?? "0");
|
|
2299
|
+
const fail = Number(output.match(/^# fail (\d+)$/m)?.[1] ?? "0");
|
|
2300
|
+
if (result.error) return { passed: false, pass, fail, error: result.error.message };
|
|
2301
|
+
return { passed: result.status === 0 && fail === 0 && pass > 0, pass, fail };
|
|
2302
|
+
}
|
|
2303
|
+
function loadFixtures(root, dir = "fixtures") {
|
|
2304
|
+
const folder = join6(root, dir);
|
|
2305
|
+
const names = readdirSync3(folder).filter((name) => name.endsWith(".yaml") || name.endsWith(".yml")).sort();
|
|
2306
|
+
const fixtures = [];
|
|
2307
|
+
const problems = [];
|
|
2308
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2309
|
+
for (const name of names) {
|
|
2310
|
+
let data;
|
|
2311
|
+
try {
|
|
2312
|
+
data = parse3(readFileSync5(join6(folder, name), "utf8"));
|
|
2313
|
+
validate("codingFixture", data, `${dir}/${name}`, root);
|
|
2314
|
+
} catch (error) {
|
|
2315
|
+
problems.push(`${name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2316
|
+
continue;
|
|
2317
|
+
}
|
|
2318
|
+
const fixture = data;
|
|
2319
|
+
if (seen.has(fixture.id)) problems.push(`${name}: duplicate fixture id ${fixture.id}`);
|
|
2320
|
+
seen.add(fixture.id);
|
|
2321
|
+
const command = commandError(fixture.runner?.command ?? DEFAULT_COMMAND);
|
|
2322
|
+
if (command) problems.push(`${name}: ${command}`);
|
|
2323
|
+
for (const problem of structureErrors(fixture)) problems.push(`${name}: ${problem}`);
|
|
2324
|
+
fixtures.push(fixture);
|
|
2325
|
+
}
|
|
2326
|
+
if (problems.length > 0) {
|
|
2327
|
+
throw new Error(`Invalid coding fixtures:
|
|
2328
|
+
${problems.map((line) => `- ${line}`).join("\n")}`);
|
|
2329
|
+
}
|
|
2330
|
+
return fixtures;
|
|
2331
|
+
}
|
|
2332
|
+
function checkFixture(fixture) {
|
|
2333
|
+
const workspace = mkdtempSync2(join6(scratchRoot(), "hahahah-fixture-"));
|
|
2334
|
+
try {
|
|
2335
|
+
const command = fixture.runner?.command ?? DEFAULT_COMMAND;
|
|
2336
|
+
const targets = testTargets(fixture, command);
|
|
2337
|
+
writeFiles(workspace, fixture.files);
|
|
2338
|
+
const broken = runTests(workspace, command, targets);
|
|
2339
|
+
writeFiles(workspace, fixture.solution);
|
|
2340
|
+
const fixed = runTests(workspace, command, targets);
|
|
2341
|
+
const problems = [];
|
|
2342
|
+
if (broken.passed) problems.push("the task state already passes, so nothing is being asked");
|
|
2343
|
+
if (!fixed.passed)
|
|
2344
|
+
problems.push(`the reference solution does not pass: ${fixed.error ?? `fail=${fixed.fail}`}`);
|
|
2345
|
+
if (fixed.pass !== fixture.tests) {
|
|
2346
|
+
problems.push(`declared ${fixture.tests} tests, the runner counted ${fixed.pass}`);
|
|
2347
|
+
}
|
|
2348
|
+
if (broken.pass + broken.fail !== fixture.tests) {
|
|
2349
|
+
problems.push(
|
|
2350
|
+
`task state ran ${broken.pass + broken.fail} of ${fixture.tests} tests, so the suite is crashing rather than asserting`
|
|
2351
|
+
);
|
|
2352
|
+
}
|
|
2353
|
+
return {
|
|
2354
|
+
id: fixture.id,
|
|
2355
|
+
ok: problems.length === 0,
|
|
2356
|
+
detail: problems.length === 0 ? `${fixed.pass} tests pass with the solution; ${broken.fail} fail in the task state` : problems.join("; "),
|
|
2357
|
+
hidden: fixture.hidden === true,
|
|
2358
|
+
testsFailingWhenBroken: broken.fail,
|
|
2359
|
+
testsPassingWithSolution: fixed.pass
|
|
2360
|
+
};
|
|
2361
|
+
} finally {
|
|
2362
|
+
removeWorkspace(workspace);
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
function checkAll(root, dir = "fixtures") {
|
|
2366
|
+
return loadFixtures(root, dir).map(checkFixture);
|
|
2367
|
+
}
|
|
2368
|
+
var TEST_FILE = /\.test\.(cjs|js|mjs)$/;
|
|
2369
|
+
var RUNNER_OWNED = /* @__PURE__ */ new Set(["CLAUDE.md", "package.json"]);
|
|
2370
|
+
var TASK_NOTES = {
|
|
2371
|
+
id: [
|
|
2372
|
+
"Kerjakan di direktori kerja ini.",
|
|
2373
|
+
"Test ada di file test/*.test.js. Buat semuanya lulus.",
|
|
2374
|
+
"Jangan mengubah file test dan jangan menambah file test baru."
|
|
2375
|
+
],
|
|
2376
|
+
en: [
|
|
2377
|
+
"Work in this working directory.",
|
|
2378
|
+
"The tests live in test/*.test.js. Make all of them pass.",
|
|
2379
|
+
"Do not modify a test file and do not add a new test file."
|
|
2380
|
+
]
|
|
2381
|
+
};
|
|
2382
|
+
function fixturePrompt(fixture) {
|
|
2383
|
+
const notes = TASK_NOTES[fixture.language ?? "id"];
|
|
2384
|
+
return [fixture.brief.trim(), "", ...notes].join("\n");
|
|
2385
|
+
}
|
|
2386
|
+
function listWorkspaceFiles(root, relativeDir = ".") {
|
|
2387
|
+
const found = [];
|
|
2388
|
+
for (const entry of readdirSync3(join6(root, relativeDir), { withFileTypes: true })) {
|
|
2389
|
+
if (entry.isDirectory()) {
|
|
2390
|
+
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
2391
|
+
found.push(...listWorkspaceFiles(root, join6(relativeDir, entry.name)));
|
|
2392
|
+
} else {
|
|
2393
|
+
const absolute = join6(root, relativeDir, entry.name);
|
|
2394
|
+
const path = relative2(root, absolute).replace(/\\/g, "/");
|
|
2395
|
+
if (!RUNNER_OWNED.has(path)) found.push(path);
|
|
2396
|
+
}
|
|
2397
|
+
}
|
|
2398
|
+
return found;
|
|
2399
|
+
}
|
|
2400
|
+
function diffWorkspace(fixture, workspace) {
|
|
2401
|
+
const declared = new Map(fixture.files.map((file) => [file.path, file.content]));
|
|
2402
|
+
const changedFiles = [];
|
|
2403
|
+
const missingFiles = [];
|
|
2404
|
+
for (const [path, content] of declared) {
|
|
2405
|
+
let text;
|
|
2406
|
+
try {
|
|
2407
|
+
text = readFileSync5(inside(workspace, path), "utf8");
|
|
2408
|
+
} catch {
|
|
2409
|
+
text = void 0;
|
|
2410
|
+
}
|
|
2411
|
+
if (text === void 0) missingFiles.push(path);
|
|
2412
|
+
else if (text.replace(/\r\n/g, "\n") !== content.replace(/\r\n/g, "\n"))
|
|
2413
|
+
changedFiles.push(path);
|
|
2414
|
+
}
|
|
2415
|
+
const extraTestFiles = [];
|
|
2416
|
+
const otherNewFiles = [];
|
|
2417
|
+
for (const path of listWorkspaceFiles(workspace)) {
|
|
2418
|
+
if (declared.has(path)) continue;
|
|
2419
|
+
if (TEST_FILE.test(path)) extraTestFiles.push(path);
|
|
2420
|
+
else otherNewFiles.push(path);
|
|
2421
|
+
}
|
|
2422
|
+
return { changedFiles, missingFiles, extraTestFiles, otherNewFiles };
|
|
2423
|
+
}
|
|
2424
|
+
function gradeRun(fixture, workspace, run) {
|
|
2425
|
+
const diff = diffWorkspace(fixture, workspace);
|
|
2426
|
+
const command = fixture.runner?.command ?? DEFAULT_COMMAND;
|
|
2427
|
+
const result = runTests(workspace, command, testTargets(fixture, command));
|
|
2428
|
+
const tampered = diff.extraTestFiles.length > 0 || diff.changedFiles.some((path) => TEST_FILE.test(path)) || diff.missingFiles.some((path) => TEST_FILE.test(path));
|
|
2429
|
+
const untouched = diff.changedFiles.length === 0 && diff.missingFiles.length === 0 && diff.otherNewFiles.length === 0;
|
|
2430
|
+
let grade;
|
|
2431
|
+
if (run.error) grade = "error";
|
|
2432
|
+
else if (tampered) grade = "tampered";
|
|
2433
|
+
else if (untouched) grade = "untouched";
|
|
2434
|
+
else if (result.error) grade = "error";
|
|
2435
|
+
else grade = result.passed && result.pass >= fixture.tests ? "solved" : "failed";
|
|
2436
|
+
return { grade, run, pass: result.pass, fail: result.fail, ...diff };
|
|
2437
|
+
}
|
|
2438
|
+
function summarizeArm(runs) {
|
|
2439
|
+
const numbers = (pick2) => median(
|
|
2440
|
+
runs.map((arm) => arm.run).map(pick2).filter((v) => v !== void 0)
|
|
2441
|
+
);
|
|
2442
|
+
return {
|
|
2443
|
+
runs,
|
|
2444
|
+
grades: runs.map((arm) => arm.grade),
|
|
2445
|
+
solved: runs.filter((arm) => arm.grade === "solved").length,
|
|
2446
|
+
median: {
|
|
2447
|
+
costUsd: numbers((run) => run.costUsd),
|
|
2448
|
+
inputTokens: numbers((run) => promptTokens(run.usage)),
|
|
2449
|
+
outputTokens: numbers((run) => usageNumber(run.usage, "output_tokens")),
|
|
2450
|
+
numTurns: numbers((run) => run.numTurns)
|
|
2451
|
+
}
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2454
|
+
function tally(results, arm) {
|
|
2455
|
+
const grades = results.flatMap((result) => result[arm].grades);
|
|
2456
|
+
const count = (grade) => grades.filter((value) => value === grade).length;
|
|
2457
|
+
return {
|
|
2458
|
+
solved: count("solved"),
|
|
2459
|
+
failed: count("failed"),
|
|
2460
|
+
tampered: count("tampered"),
|
|
2461
|
+
untouched: count("untouched"),
|
|
2462
|
+
error: count("error")
|
|
2463
|
+
};
|
|
2464
|
+
}
|
|
2465
|
+
function projectedCostUsd(count, repeats, budgetUsd, arms = 2) {
|
|
2466
|
+
return count * repeats * arms * budgetUsd;
|
|
2467
|
+
}
|
|
2468
|
+
function safeAgent(runner, cwd, prompt, budget) {
|
|
2469
|
+
try {
|
|
2470
|
+
return runner(cwd, prompt, budget);
|
|
2471
|
+
} catch (error) {
|
|
2472
|
+
return {
|
|
2473
|
+
output: "",
|
|
2474
|
+
costUsd: 0,
|
|
2475
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2476
|
+
};
|
|
2477
|
+
}
|
|
2478
|
+
}
|
|
2479
|
+
function studyMarkdown(report) {
|
|
2480
|
+
const line = (arm) => {
|
|
2481
|
+
const oneLine = (text) => text.replace(/\s+/g, " ").trim();
|
|
2482
|
+
if (arm.runs.length > 0 && arm.runs.every((run) => run.grade === "error"))
|
|
2483
|
+
return `error (${oneLine(arm.runs[0]?.run.error ?? "no runs")})`;
|
|
2484
|
+
return `${arm.grades.join(", ")} \u2014 solved ${arm.solved}/${arm.runs.length}`;
|
|
2485
|
+
};
|
|
2486
|
+
const cell = (arm) => `$${arm.median.costUsd.toFixed(4)} (in ${Math.round(arm.median.inputTokens)}, out ${Math.round(arm.median.outputTokens)}, turns ${arm.median.numTurns || "?"})`;
|
|
2487
|
+
const residue = report.residue.length === 0 ? "none" : `${report.residue.length} workspace dir(s) were still held open by the harness and are left behind: ${report.residue.join(", ")}`;
|
|
2488
|
+
const lines = [
|
|
2489
|
+
"# HAHahah coding fixture study",
|
|
2490
|
+
"",
|
|
2491
|
+
`- Profile: ${report.profile}`,
|
|
2492
|
+
`- Model: ${report.model ?? "harness default"}`,
|
|
2493
|
+
`- Generated: ${report.generated_at}`,
|
|
2494
|
+
`- Fixtures run: ${report.fixtures_run} (${report.hidden_run} hidden holdout)`,
|
|
2495
|
+
`- Repeats per fixture per arm: ${report.repeats}`,
|
|
2496
|
+
`- Arms run: ${report.arms.join(" + ")}${report.arms.length === 1 ? " (single arm \u2014 no baseline/treatment comparison is available from this run)" : ""}`,
|
|
2497
|
+
`- Budget per run: $${report.budget_per_run_usd.toFixed(2)}`,
|
|
2498
|
+
`- Total cost reported: $${report.total_cost_usd.toFixed(4)}`,
|
|
2499
|
+
`- Harness: headless, --permission-mode ${report.harness.permission_mode}, --permission-prompts ${report.harness.permission_prompts}`,
|
|
2500
|
+
`- Workspace residue after the run: ${residue}`,
|
|
2501
|
+
"",
|
|
2502
|
+
`Baseline: solved ${report.summary.baseline.solved}, failed ${report.summary.baseline.failed}, tampered ${report.summary.baseline.tampered}, untouched ${report.summary.baseline.untouched}, error ${report.summary.baseline.error}`,
|
|
2503
|
+
`Treatment: solved ${report.summary.treatment.solved}, failed ${report.summary.treatment.failed}, tampered ${report.summary.treatment.tampered}, untouched ${report.summary.treatment.untouched}, error ${report.summary.treatment.error}`,
|
|
2504
|
+
`Hidden holdout (${report.hidden_run} of ${report.fixtures_run} fixtures) \u2014 baseline solved ${report.summary.holdout.baseline.solved}, treatment solved ${report.summary.holdout.treatment.solved}. Read these apart: the visible set is what the fixtures were designed against, so a holdout that scores worse is the honest signal.`,
|
|
2505
|
+
"",
|
|
2506
|
+
"| Fixture | Baseline | Treatment | Baseline median | Treatment median |",
|
|
2507
|
+
"|---|---|---|---:|---:|",
|
|
2508
|
+
...report.results.map(
|
|
2509
|
+
(result) => `| ${result.fixture.id}${result.fixture.hidden ? " (hidden)" : ""} | ${line(result.baseline)} | ${line(result.treatment)} | ${cell(result.baseline)} | ${cell(result.treatment)} |`
|
|
2510
|
+
),
|
|
2511
|
+
"",
|
|
2512
|
+
"## What this grades",
|
|
2513
|
+
"",
|
|
2514
|
+
"Each run is graded by executing the fixture's own tests in the workspace after the agent finished, so this is a correctness signal rather than a text screen.",
|
|
2515
|
+
"`solved` means the untouched tests all passed. `tampered` means a test file was edited, deleted, or added; those runs are never counted as solved even when the tests then pass.",
|
|
2516
|
+
"`untouched` means the workspace was byte-identical to the task state: the agent replied without changing the code.",
|
|
2517
|
+
"`error` is a run that did not complete. It is reported as an error and never scored as a failure of the persona.",
|
|
2518
|
+
"",
|
|
2519
|
+
"## Limits of this report",
|
|
2520
|
+
"",
|
|
2521
|
+
"The agent ran headless with file edits auto-accepted and every other permission denied, so it could not execute the test suite itself. Pass rates are conditioned on that and are not a claim about interactive use.",
|
|
2522
|
+
"A `tampered` or `untouched` spike may be a harness permission problem rather than a model choice; check a raw transcript before attributing either grade to the persona.",
|
|
2523
|
+
"Costs are medians over the repeats because this provider bills heavy-tailed spikes; one run is not a measurement.",
|
|
2524
|
+
"With this many fixtures the solved counts are descriptive. No non-inferiority or accuracy rate is claimed.",
|
|
2525
|
+
""
|
|
2526
|
+
];
|
|
2527
|
+
return lines.join("\n");
|
|
2528
|
+
}
|
|
2529
|
+
function runFixtureStudy(options) {
|
|
2530
|
+
const repeats = options.repeats ?? 1;
|
|
2531
|
+
const fixtures = options.fixtures.slice(0, options.limit ?? options.fixtures.length);
|
|
2532
|
+
const runner = options.runner ?? ((cwd, prompt, budgetUsd) => runClaude(cwd, prompt, budgetUsd, {
|
|
2533
|
+
model: options.model,
|
|
2534
|
+
settingSources: options.settingSources,
|
|
2535
|
+
// The fixture task is to change source files, so edits are the one thing the
|
|
2536
|
+
// agent must be able to do. Shell and network stay denied: this host is
|
|
2537
|
+
// unsandboxed (TECHNICAL_RISKS R-10/R-14), and the grade comes from the tests
|
|
2538
|
+
// this tool runs afterwards, not from anything the agent ran.
|
|
2539
|
+
agentArgs: ["--permission-mode", "acceptEdits", "--permission-prompts", "none"]
|
|
2540
|
+
}));
|
|
2541
|
+
const results = [];
|
|
2542
|
+
const residue = [];
|
|
2543
|
+
const remove = options.remove ?? removeWorkspace;
|
|
2544
|
+
const arms = options.arm && options.arm !== "both" ? [options.arm] : ["baseline", "treatment"];
|
|
2545
|
+
for (const fixture of fixtures) {
|
|
2546
|
+
const prompt = fixturePrompt(fixture);
|
|
2547
|
+
const baseline = [];
|
|
2548
|
+
const treatment = [];
|
|
2549
|
+
for (let index = 0; index < repeats; index += 1) {
|
|
2550
|
+
let order = ["baseline", "treatment"];
|
|
2551
|
+
if (arms.length === 1) {
|
|
2552
|
+
order = arms;
|
|
2553
|
+
} else if (index % 2 === 1) {
|
|
2554
|
+
order = ["treatment", "baseline"];
|
|
2555
|
+
}
|
|
2556
|
+
for (const arm of order) {
|
|
2557
|
+
const dir = mkdtempSync2(join6(scratchRoot(), `hahahah-study-${arm}-`));
|
|
2558
|
+
try {
|
|
2559
|
+
writeFiles(dir, fixture.files);
|
|
2560
|
+
ensureModuleMarker(dir);
|
|
2561
|
+
if (arm === "treatment") {
|
|
2562
|
+
install({
|
|
2563
|
+
sourceRoot: options.sourceRoot,
|
|
2564
|
+
targetRoot: dir,
|
|
2565
|
+
profileId: options.profileId,
|
|
2566
|
+
adapterId: options.adapterId
|
|
2567
|
+
});
|
|
2568
|
+
}
|
|
2569
|
+
const run = safeAgent(runner, dir, prompt, options.budgetUsd);
|
|
2570
|
+
(arm === "baseline" ? baseline : treatment).push(gradeRun(fixture, dir, run));
|
|
2571
|
+
} finally {
|
|
2572
|
+
if (!remove(dir)) residue.push(dir);
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
}
|
|
2576
|
+
results.push({
|
|
2577
|
+
fixture,
|
|
2578
|
+
baseline: summarizeArm(baseline),
|
|
2579
|
+
treatment: summarizeArm(treatment)
|
|
2580
|
+
});
|
|
2581
|
+
}
|
|
2582
|
+
const allRuns = results.flatMap((result) => [...result.baseline.runs, ...result.treatment.runs]);
|
|
2583
|
+
const holdout = results.filter((result) => result.fixture.hidden);
|
|
2584
|
+
const report = {
|
|
2585
|
+
schema: "hahahah.coding.study/v1",
|
|
2586
|
+
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2587
|
+
profile: options.profileId,
|
|
2588
|
+
model: options.model ?? null,
|
|
2589
|
+
suite: "coding-fixtures",
|
|
2590
|
+
budget_per_run_usd: options.budgetUsd,
|
|
2591
|
+
repeats,
|
|
2592
|
+
arms,
|
|
2593
|
+
fixtures_run: results.length,
|
|
2594
|
+
hidden_run: holdout.length,
|
|
2595
|
+
total_cost_usd: allRuns.reduce((total, arm) => total + arm.run.costUsd, 0),
|
|
2596
|
+
harness: { permission_mode: "acceptEdits", permission_prompts: "none" },
|
|
2597
|
+
residue,
|
|
2598
|
+
summary: {
|
|
2599
|
+
baseline: tally(results, "baseline"),
|
|
2600
|
+
treatment: tally(results, "treatment"),
|
|
2601
|
+
holdout: {
|
|
2602
|
+
baseline: tally(holdout, "baseline"),
|
|
2603
|
+
treatment: tally(holdout, "treatment")
|
|
2604
|
+
}
|
|
2605
|
+
},
|
|
2606
|
+
results
|
|
2607
|
+
};
|
|
2608
|
+
const outDir = resolve5(options.outDir);
|
|
2609
|
+
mkdirSync5(outDir, { recursive: true });
|
|
2610
|
+
writeFileSync5(join6(outDir, "report.json"), `${JSON.stringify(report, null, 2)}
|
|
2611
|
+
`, "utf8");
|
|
2612
|
+
writeFileSync5(join6(outDir, "report.md"), studyMarkdown(report), "utf8");
|
|
2613
|
+
return report;
|
|
2614
|
+
}
|
|
2615
|
+
function diagnoseFixture(options) {
|
|
2616
|
+
const dir = mkdtempSync2(join6(scratchRoot(), `hahahah-diagnose-${options.arm}-`));
|
|
2617
|
+
writeFiles(dir, options.fixture.files);
|
|
2618
|
+
ensureModuleMarker(dir);
|
|
2619
|
+
if (options.arm === "treatment") {
|
|
2620
|
+
install({
|
|
2621
|
+
sourceRoot: options.sourceRoot,
|
|
2622
|
+
targetRoot: dir,
|
|
2623
|
+
profileId: options.profileId
|
|
2624
|
+
});
|
|
2625
|
+
}
|
|
2626
|
+
const stream = options.runner ?? runClaudeStream;
|
|
2627
|
+
const { run, ndjson } = stream(dir, fixturePrompt(options.fixture), options.budgetUsd, {
|
|
2628
|
+
model: options.model,
|
|
2629
|
+
settingSources: options.settingSources
|
|
2630
|
+
});
|
|
2631
|
+
return {
|
|
2632
|
+
summary: summarizeStream(ndjson),
|
|
2633
|
+
graded: gradeRun(options.fixture, dir, run),
|
|
2634
|
+
rawPath: writeRawStream(options.outDir, options.arm, ndjson),
|
|
2635
|
+
workspace: dir
|
|
2636
|
+
};
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2639
|
+
// src/notifier.ts
|
|
2640
|
+
function isNewerVersion(remote, current) {
|
|
2641
|
+
const parse4 = (v) => v.replace(/^v/, "").split(/[-+.]/).map((p) => Number.parseInt(p, 10)).filter((n) => !Number.isNaN(n));
|
|
2642
|
+
const rParts = parse4(remote);
|
|
2643
|
+
const cParts = parse4(current);
|
|
2644
|
+
const maxLen = Math.max(rParts.length, cParts.length, 3);
|
|
2645
|
+
for (let i = 0; i < maxLen; i++) {
|
|
2646
|
+
const r = rParts[i] ?? 0;
|
|
2647
|
+
const c = cParts[i] ?? 0;
|
|
2648
|
+
if (r !== c) {
|
|
2649
|
+
return r > c;
|
|
2650
|
+
}
|
|
2651
|
+
}
|
|
2652
|
+
return false;
|
|
2653
|
+
}
|
|
2654
|
+
function formatUpdateNotice(current, latest) {
|
|
2655
|
+
return [
|
|
2656
|
+
"",
|
|
2657
|
+
"\u250C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510",
|
|
2658
|
+
`\u2502 Pembaruan Anti-Yapping tersedia: v${current} \u2192 v${latest}`.padEnd(57) + "\u2502",
|
|
2659
|
+
"\u2502 Jalankan perintah berikut untuk memperbarui: \u2502",
|
|
2660
|
+
"\u2502 $ npx anti-yapping update \u2502",
|
|
2661
|
+
"\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518",
|
|
2662
|
+
""
|
|
2663
|
+
].join("\n");
|
|
2664
|
+
}
|
|
2665
|
+
async function checkRemoteUpdate(currentVersion) {
|
|
2666
|
+
if (process.env.HAHAHAH_NO_UPDATE_CHECK || process.env.CI) {
|
|
2667
|
+
return null;
|
|
2668
|
+
}
|
|
2669
|
+
try {
|
|
2670
|
+
const res = await fetch(
|
|
2671
|
+
"https://raw.githubusercontent.com/humanoiseLABS/anti-yapping/main/package.json",
|
|
2672
|
+
{
|
|
2673
|
+
signal: AbortSignal.timeout(1500)
|
|
2674
|
+
}
|
|
2675
|
+
);
|
|
2676
|
+
if (!res.ok) return null;
|
|
2677
|
+
const data = await res.json();
|
|
2678
|
+
if (data.version && isNewerVersion(data.version, currentVersion)) {
|
|
2679
|
+
return data.version;
|
|
2680
|
+
}
|
|
2681
|
+
} catch {
|
|
2682
|
+
}
|
|
2683
|
+
return null;
|
|
2684
|
+
}
|
|
2685
|
+
|
|
2686
|
+
// src/opencode.ts
|
|
2687
|
+
import { spawnSync as spawnSync4 } from "node:child_process";
|
|
2688
|
+
import {
|
|
2689
|
+
closeSync,
|
|
2690
|
+
mkdtempSync as mkdtempSync3,
|
|
2691
|
+
openSync,
|
|
2692
|
+
readFileSync as readFileSync6,
|
|
2693
|
+
readSync,
|
|
2694
|
+
statSync as statSync2,
|
|
2695
|
+
writeFileSync as writeFileSync6
|
|
2696
|
+
} from "node:fs";
|
|
2697
|
+
import { homedir as homedir2 } from "node:os";
|
|
2698
|
+
import { join as join7 } from "node:path";
|
|
2699
|
+
function opencodeArgs(cwd, options = {}) {
|
|
2700
|
+
const args = ["run", "--format", "json", "--dir", cwd];
|
|
2701
|
+
if (options.model) args.push("--model", options.model);
|
|
2702
|
+
if (options.agentArgs) args.push(...options.agentArgs);
|
|
2703
|
+
return args;
|
|
2704
|
+
}
|
|
2705
|
+
function opencodeStudyConfig() {
|
|
2706
|
+
return JSON.stringify(
|
|
2707
|
+
{
|
|
2708
|
+
permission: {
|
|
2709
|
+
edit: "allow",
|
|
2710
|
+
bash: "deny",
|
|
2711
|
+
webfetch: "deny",
|
|
2712
|
+
external_directory: "deny",
|
|
2713
|
+
doom_loop: "deny"
|
|
2714
|
+
}
|
|
2715
|
+
},
|
|
2716
|
+
null,
|
|
2717
|
+
2
|
|
2718
|
+
);
|
|
2719
|
+
}
|
|
2720
|
+
function tokensOf(part) {
|
|
2721
|
+
const tokens = part.tokens;
|
|
2722
|
+
if (typeof tokens !== "object" || tokens === null) return null;
|
|
2723
|
+
const record = tokens;
|
|
2724
|
+
if (typeof record.input !== "number" || typeof record.output !== "number") return null;
|
|
2725
|
+
const cache = typeof record.cache === "object" && record.cache !== null ? record.cache : {};
|
|
2726
|
+
return {
|
|
2727
|
+
input_tokens: record.input,
|
|
2728
|
+
output_tokens: record.output,
|
|
2729
|
+
cache_read_input_tokens: typeof cache.read === "number" ? cache.read : 0,
|
|
2730
|
+
cache_creation_input_tokens: typeof cache.write === "number" ? cache.write : 0
|
|
2731
|
+
};
|
|
2732
|
+
}
|
|
2733
|
+
function scanStream(stdout) {
|
|
2734
|
+
const scan = { texts: [], costUsd: 0, turns: 0 };
|
|
2735
|
+
for (const line of stdout.split("\n")) {
|
|
2736
|
+
const trimmed = line.trim();
|
|
2737
|
+
if (!trimmed.startsWith("{")) continue;
|
|
2738
|
+
let event;
|
|
2739
|
+
try {
|
|
2740
|
+
event = JSON.parse(trimmed);
|
|
2741
|
+
} catch {
|
|
2742
|
+
continue;
|
|
2743
|
+
}
|
|
2744
|
+
const part = event.part ?? {};
|
|
2745
|
+
if (event.type === "text" && typeof part.text === "string") {
|
|
2746
|
+
scan.texts.push(part.text);
|
|
2747
|
+
} else if (event.type === "step_finish") {
|
|
2748
|
+
scan.turns += 1;
|
|
2749
|
+
if (typeof part.cost === "number") scan.costUsd = part.cost;
|
|
2750
|
+
const tokens = tokensOf(part);
|
|
2751
|
+
if (tokens) scan.usage = tokens;
|
|
2752
|
+
} else if (event.type === "error") {
|
|
2753
|
+
const detail = event.error;
|
|
2754
|
+
const data = detail?.data;
|
|
2755
|
+
const name = typeof detail?.name === "string" ? detail.name : "opencode error";
|
|
2756
|
+
const message = typeof data?.message === "string" ? data.message : JSON.stringify(detail);
|
|
2757
|
+
scan.error = `${name}: ${message}`;
|
|
2758
|
+
}
|
|
2759
|
+
}
|
|
2760
|
+
return scan;
|
|
2761
|
+
}
|
|
2762
|
+
function parseOpencodeStream(stdout) {
|
|
2763
|
+
const scan = scanStream(stdout);
|
|
2764
|
+
const output = scan.texts.join("");
|
|
2765
|
+
if (scan.error) return { output: "", costUsd: scan.costUsd, error: scan.error };
|
|
2766
|
+
if (!output) return { output: "", costUsd: scan.costUsd, error: "opencode returned no text" };
|
|
2767
|
+
return { output, costUsd: scan.costUsd, usage: scan.usage, numTurns: scan.turns };
|
|
2768
|
+
}
|
|
2769
|
+
var LOG_TAIL_BYTES = 512 * 1024;
|
|
2770
|
+
function opencodeLogPath() {
|
|
2771
|
+
return join7(homedir2(), ".local", "share", "opencode", "log", "opencode.log");
|
|
2772
|
+
}
|
|
2773
|
+
function loggedErrorForRun(logText, directory) {
|
|
2774
|
+
const quotedDirectory = `directory="${directory.replace(/\\/g, "\\\\")}"`;
|
|
2775
|
+
const lines = logText.split("\n");
|
|
2776
|
+
const created = lines.find(
|
|
2777
|
+
(line) => line.includes("message=created") && line.includes(quotedDirectory)
|
|
2778
|
+
);
|
|
2779
|
+
const runId = created === void 0 ? void 0 : / run=(\S+)/.exec(created)?.[1];
|
|
2780
|
+
if (runId === void 0) return null;
|
|
2781
|
+
const errors = lines.filter(
|
|
2782
|
+
(line) => line.includes(` run=${runId} `) && line.includes('message="stream error"')
|
|
2783
|
+
);
|
|
2784
|
+
const answer = errors.find((line) => line.includes("small=false")) ?? errors[0];
|
|
2785
|
+
if (answer === void 0) return null;
|
|
2786
|
+
return /error\.error="(.*)"\s*$/.exec(answer)?.[1] ?? null;
|
|
2787
|
+
}
|
|
2788
|
+
function readLogTail() {
|
|
2789
|
+
try {
|
|
2790
|
+
const path = opencodeLogPath();
|
|
2791
|
+
const size = statSync2(path).size;
|
|
2792
|
+
if (size <= LOG_TAIL_BYTES) return readFileSync6(path, "utf8");
|
|
2793
|
+
const fd = openSync(path, "r");
|
|
2794
|
+
try {
|
|
2795
|
+
const buffer = Buffer.alloc(LOG_TAIL_BYTES);
|
|
2796
|
+
readSync(fd, buffer, 0, LOG_TAIL_BYTES, size - LOG_TAIL_BYTES);
|
|
2797
|
+
const text = buffer.toString("utf8");
|
|
2798
|
+
return text.slice(text.indexOf("\n") + 1);
|
|
2799
|
+
} finally {
|
|
2800
|
+
closeSync(fd);
|
|
2801
|
+
}
|
|
2802
|
+
} catch {
|
|
2803
|
+
return "";
|
|
2804
|
+
}
|
|
2805
|
+
}
|
|
2806
|
+
function runOpencode(cwd, prompt, budgetUsd, options = {}, history) {
|
|
2807
|
+
const command = process.env.HAHAHAH_OPENCODE_BIN ?? "opencode";
|
|
2808
|
+
const input = withTranscript(history, prompt);
|
|
2809
|
+
const startedAt = Date.now();
|
|
2810
|
+
const configDir = mkdtempSync3(join7(scratchRoot(), "hahahah-opencode-cfg-"));
|
|
2811
|
+
const configPath = join7(configDir, "opencode.json");
|
|
2812
|
+
writeFileSync6(configPath, opencodeStudyConfig(), "utf8");
|
|
2813
|
+
const args = opencodeArgs(cwd, options);
|
|
2814
|
+
const invocation = process.platform === "win32" ? (
|
|
2815
|
+
// opencode on this machine is a .cmd shim, which spawnSync cannot exec directly
|
|
2816
|
+
// (measured: `spawnSync opencode ENOENT`). Same wrapper runClaude uses.
|
|
2817
|
+
spawnSync4(
|
|
2818
|
+
process.env.ComSpec ?? "cmd.exe",
|
|
2819
|
+
["/d", "/s", "/c", [command, ...args].join(" ")],
|
|
2820
|
+
{
|
|
2821
|
+
cwd,
|
|
2822
|
+
input,
|
|
2823
|
+
encoding: "utf8",
|
|
2824
|
+
timeout: options.timeoutMs ?? 18e4,
|
|
2825
|
+
env: { ...process.env, OPENCODE_CONFIG: configPath }
|
|
2826
|
+
}
|
|
2827
|
+
)
|
|
2828
|
+
) : spawnSync4(command, args, {
|
|
2829
|
+
cwd,
|
|
2830
|
+
input,
|
|
2831
|
+
encoding: "utf8",
|
|
2832
|
+
timeout: options.timeoutMs ?? 18e4,
|
|
2833
|
+
env: { ...process.env, OPENCODE_CONFIG: configPath }
|
|
2834
|
+
});
|
|
2835
|
+
const elapsed = () => Date.now() - startedAt;
|
|
2836
|
+
if (invocation.error) {
|
|
2837
|
+
const reported = scanStream(invocation.stdout ?? "").error;
|
|
2838
|
+
const logged = loggedErrorForRun(readLogTail(), cwd);
|
|
2839
|
+
const said = reported ?? logged;
|
|
2840
|
+
return {
|
|
2841
|
+
output: "",
|
|
2842
|
+
costUsd: 0,
|
|
2843
|
+
durationMs: elapsed(),
|
|
2844
|
+
error: said === null || said === void 0 ? invocation.error.message : said
|
|
2845
|
+
};
|
|
2846
|
+
}
|
|
2847
|
+
const stdout = (invocation.stdout ?? "").trim();
|
|
2848
|
+
if (invocation.status !== 0 && !stdout) {
|
|
2849
|
+
return {
|
|
2850
|
+
output: "",
|
|
2851
|
+
costUsd: 0,
|
|
2852
|
+
durationMs: elapsed(),
|
|
2853
|
+
error: (invocation.stderr ?? "").trim() || `opencode exited ${String(invocation.status)}`
|
|
2854
|
+
};
|
|
2855
|
+
}
|
|
2856
|
+
const run = parseOpencodeStream(stdout);
|
|
2857
|
+
run.durationMs = elapsed();
|
|
2858
|
+
if (!run.error && run.costUsd > budgetUsd) {
|
|
2859
|
+
return {
|
|
2860
|
+
...run,
|
|
2861
|
+
error: `run cost $${run.costUsd.toFixed(4)} exceeded the $${budgetUsd} ceiling`
|
|
2862
|
+
};
|
|
2863
|
+
}
|
|
2864
|
+
return run;
|
|
2865
|
+
}
|
|
2866
|
+
|
|
2867
|
+
// src/cli.ts
|
|
2868
|
+
var HARNESS_ADAPTERS = {
|
|
2869
|
+
claude: "claude-code",
|
|
2870
|
+
opencode: "opencode"
|
|
2871
|
+
};
|
|
2872
|
+
function harnessOf(value) {
|
|
2873
|
+
const kind = value ?? "claude";
|
|
2874
|
+
if (!Object.keys(HARNESS_ADAPTERS).includes(kind)) {
|
|
2875
|
+
throw new Error(`unknown --harness ${String(value)}: use claude or opencode`);
|
|
2876
|
+
}
|
|
2877
|
+
return kind;
|
|
2878
|
+
}
|
|
2879
|
+
function runnerFor(harness, options) {
|
|
2880
|
+
const kind = harnessOf(harness);
|
|
2881
|
+
if (kind === "opencode") {
|
|
2882
|
+
return (cwd, prompt, budgetUsd, history) => runOpencode(cwd, prompt, budgetUsd, { model: options.model }, history);
|
|
2883
|
+
}
|
|
2884
|
+
return (cwd, prompt, budgetUsd, history) => runClaude(cwd, prompt, budgetUsd, options, history);
|
|
2885
|
+
}
|
|
2886
|
+
function printUsage() {
|
|
2887
|
+
console.log(`Anti-Yapping (+62 Terse & Ponytail Surgical Diffs)
|
|
2888
|
+
|
|
2889
|
+
Usage:
|
|
2890
|
+
npx anti-yapping install [--target <repo>] [--adapter <name>] [--scope workspace|global] [--dry-run] [--force]
|
|
2891
|
+
npx anti-yapping update [--target <repo>] [--dry-run] [--force]
|
|
2892
|
+
npx anti-yapping uninstall [--target <repo>] [--dry-run] [--force]
|
|
2893
|
+
npx anti-yapping doctor [--target <repo>]
|
|
2894
|
+
npx anti-yapping measure [--profile <id>] [--adapter fake|claude-code|opencode|agents-md|antigravity]
|
|
2895
|
+
npx anti-yapping build [--profile <id>] [--adapter fake|claude-code|opencode|agents-md|antigravity] [--out <dir>] [--release]
|
|
2896
|
+
|
|
2897
|
+
Which --adapter to pass, by the agent you run (install into the project you want it active in):
|
|
2898
|
+
Claude Code --adapter claude-code writes CLAUDE.md + .claude/skills/
|
|
2899
|
+
opencode --adapter opencode writes AGENTS.md + .claude/skills/ (opencode reads that tree)
|
|
2900
|
+
Antigravity --adapter antigravity writes AGENTS.md + .agents/rules/*.md; add --scope global with --target ~/.gemini for every project
|
|
2901
|
+
Cursor / Codex / Qoder CLI / anything reading AGENTS.md
|
|
2902
|
+
--adapter agents-md writes AGENTS.md + .agents/skills/
|
|
2903
|
+
Omitting --adapter installs for Claude Code. Uninstall needs only --target; the manifest remembers the rest.
|
|
2904
|
+
`);
|
|
2905
|
+
}
|
|
2906
|
+
function positiveInt(value, flag, fallback) {
|
|
2907
|
+
if (value === void 0) return fallback;
|
|
2908
|
+
const parsed = Number.parseInt(value, 10);
|
|
2909
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
2910
|
+
throw new Error(`${flag} must be a positive integer`);
|
|
2911
|
+
}
|
|
2912
|
+
return parsed;
|
|
2913
|
+
}
|
|
2914
|
+
function positiveNumber(value, flag) {
|
|
2915
|
+
const parsed = Number.parseFloat(value);
|
|
2916
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
2917
|
+
throw new Error(`${flag} must be a positive number`);
|
|
2918
|
+
}
|
|
2919
|
+
return parsed;
|
|
2920
|
+
}
|
|
2921
|
+
function printStudy(report) {
|
|
2922
|
+
const oneLine = (text) => text.replace(/\s+/g, " ").trim();
|
|
2923
|
+
console.log(
|
|
2924
|
+
`coding study: ${report.fixtures_run} fixture(s) x ${report.repeats} repeat(s) x ${report.arms.length} arm(s) [${report.arms.join("+")}], cost $${report.total_cost_usd.toFixed(4)}`
|
|
2925
|
+
);
|
|
2926
|
+
for (const result of report.results) {
|
|
2927
|
+
const state = (arm) => {
|
|
2928
|
+
if (arm.grades.length === 0) return "not run";
|
|
2929
|
+
if (arm.grades.every((grade) => grade === "error")) {
|
|
2930
|
+
return `error (${oneLine(arm.runs[0]?.run.error ?? "no runs")})`;
|
|
2931
|
+
}
|
|
2932
|
+
return `${arm.grades.join(",")} solved ${arm.solved}/${arm.runs.length}`;
|
|
2933
|
+
};
|
|
2934
|
+
console.log(
|
|
2935
|
+
`- ${result.fixture.id}${result.fixture.hidden ? " (hidden)" : ""}: baseline=${state(result.baseline)}, treatment=${state(result.treatment)}`
|
|
2936
|
+
);
|
|
2937
|
+
}
|
|
2938
|
+
const tallyLine = (name) => {
|
|
2939
|
+
const tally2 = report.summary[name];
|
|
2940
|
+
return `grades: ${name} solved ${tally2.solved} failed ${tally2.failed} tampered ${tally2.tampered} untouched ${tally2.untouched} error ${tally2.error}`;
|
|
2941
|
+
};
|
|
2942
|
+
for (const arm of report.arms) {
|
|
2943
|
+
console.log(tallyLine(arm));
|
|
2944
|
+
}
|
|
2945
|
+
if (report.hidden_run > 0) {
|
|
2946
|
+
console.log(
|
|
2947
|
+
`hidden holdout (${report.hidden_run} fixture(s)): baseline solved ${report.summary.holdout.baseline.solved}, treatment solved ${report.summary.holdout.treatment.solved}`
|
|
2948
|
+
);
|
|
2949
|
+
}
|
|
2950
|
+
const errored = report.results.flatMap((result) => [...result.baseline.runs, ...result.treatment.runs]).some((run) => run.grade === "error");
|
|
2951
|
+
if (errored) process.exitCode = 1;
|
|
2952
|
+
}
|
|
2953
|
+
function formatReport(result) {
|
|
2954
|
+
const lines = [
|
|
2955
|
+
`profile: ${result.manifest.profile}`,
|
|
2956
|
+
`persona: ${result.manifest.persona}`,
|
|
2957
|
+
`adapter: ${result.manifest.adapter}`,
|
|
2958
|
+
`tokenizer: ${result.manifest.tokenizer}`,
|
|
2959
|
+
`content_hash: ${result.manifest.content_hash}`
|
|
2960
|
+
];
|
|
2961
|
+
for (const [name, report] of Object.entries(result.manifest.budgets)) {
|
|
2962
|
+
lines.push(
|
|
2963
|
+
`${name}: ${report.tokens}/${report.target} (${report.status}; hard cap ${report.hardCap})`
|
|
2964
|
+
);
|
|
2965
|
+
}
|
|
2966
|
+
if (result.warnings.length > 0) {
|
|
2967
|
+
lines.push("warnings:");
|
|
2968
|
+
for (const warning of result.warnings) {
|
|
2969
|
+
lines.push(`- ${warning}`);
|
|
2970
|
+
}
|
|
2971
|
+
}
|
|
2972
|
+
return lines.join("\n");
|
|
2973
|
+
}
|
|
2974
|
+
function isMainEntry() {
|
|
2975
|
+
const invoked = process.argv[1];
|
|
2976
|
+
if (!invoked) return false;
|
|
2977
|
+
return fileURLToPath2(import.meta.url) === resolve6(invoked);
|
|
2978
|
+
}
|
|
2979
|
+
var packageRoot = resolve6(fileURLToPath2(import.meta.url), "..", "..");
|
|
2980
|
+
function getCliVersion() {
|
|
2981
|
+
try {
|
|
2982
|
+
const pkg = JSON.parse(readFileSync7(join8(packageRoot, "package.json"), "utf8"));
|
|
2983
|
+
return pkg.version ?? "0.1.0";
|
|
2984
|
+
} catch {
|
|
2985
|
+
return "0.1.0";
|
|
2986
|
+
}
|
|
2987
|
+
}
|
|
2988
|
+
if (isMainEntry()) {
|
|
2989
|
+
const version = getCliVersion();
|
|
2990
|
+
const updateCheck = checkRemoteUpdate(version);
|
|
2991
|
+
try {
|
|
2992
|
+
const parsed = parseArgs({
|
|
2993
|
+
allowPositionals: true,
|
|
2994
|
+
strict: true,
|
|
2995
|
+
options: {
|
|
2996
|
+
profile: { type: "string", default: "anti-yapping" },
|
|
2997
|
+
adapter: { type: "string" },
|
|
2998
|
+
target: { type: "string", default: process.cwd() },
|
|
2999
|
+
out: { type: "string" },
|
|
3000
|
+
suite: { type: "string", default: "smoke" },
|
|
3001
|
+
task: { type: "string", multiple: true, default: [] },
|
|
3002
|
+
id: { type: "string", multiple: true, default: [] },
|
|
3003
|
+
limit: { type: "string" },
|
|
3004
|
+
repeats: { type: "string" },
|
|
3005
|
+
"budget-usd": { type: "string", default: "0.10" },
|
|
3006
|
+
"max-total-usd": { type: "string" },
|
|
3007
|
+
model: { type: "string" },
|
|
3008
|
+
harness: { type: "string" },
|
|
3009
|
+
scope: { type: "string" },
|
|
3010
|
+
arm: { type: "string" },
|
|
3011
|
+
"setting-sources": { type: "string" },
|
|
3012
|
+
release: { type: "boolean", default: false },
|
|
3013
|
+
"dry-run": { type: "boolean", default: false },
|
|
3014
|
+
force: { type: "boolean", default: false },
|
|
3015
|
+
strict: { type: "boolean", default: false },
|
|
3016
|
+
live: { type: "boolean", default: false },
|
|
3017
|
+
dir: { type: "string" },
|
|
3018
|
+
help: { type: "boolean", short: "h", default: false }
|
|
3019
|
+
}
|
|
3020
|
+
});
|
|
3021
|
+
const command = parsed.positionals[0];
|
|
3022
|
+
if (!command || parsed.values.help) {
|
|
3023
|
+
printUsage();
|
|
3024
|
+
process.exitCode = command ? 0 : 1;
|
|
3025
|
+
} else if (command === "build" || command === "measure") {
|
|
3026
|
+
const result = build({
|
|
3027
|
+
root: packageRoot,
|
|
3028
|
+
profileId: parsed.values.profile,
|
|
3029
|
+
adapterId: parsed.values.adapter ?? "fake",
|
|
3030
|
+
outDir: parsed.values.out,
|
|
3031
|
+
release: parsed.values.release,
|
|
3032
|
+
write: command === "build"
|
|
3033
|
+
});
|
|
3034
|
+
console.log(formatReport(result));
|
|
3035
|
+
if (command === "build") {
|
|
3036
|
+
console.log(`out: ${result.outDir}`);
|
|
3037
|
+
}
|
|
3038
|
+
} else if (command === "install") {
|
|
3039
|
+
const scope = parsed.values.scope ?? "workspace";
|
|
3040
|
+
const result = install({
|
|
3041
|
+
sourceRoot: packageRoot,
|
|
3042
|
+
targetRoot: parsed.values.target,
|
|
3043
|
+
profileId: parsed.values.profile,
|
|
3044
|
+
adapterId: parsed.values.adapter ?? "claude-code",
|
|
3045
|
+
scope,
|
|
3046
|
+
dryRun: parsed.values["dry-run"],
|
|
3047
|
+
force: parsed.values.force
|
|
3048
|
+
});
|
|
3049
|
+
console.log(
|
|
3050
|
+
`${result.dryRun ? "Dry run: would install" : "Installed"} ${result.manifest.profile} into ${result.targetRoot} (${scope} scope)`
|
|
3051
|
+
);
|
|
3052
|
+
console.log(
|
|
3053
|
+
`files: ${result.manifest.files.length} managed file(s) + ${result.manifest.block.path} block`
|
|
3054
|
+
);
|
|
3055
|
+
for (const warning of result.warnings) {
|
|
3056
|
+
console.log(`warning: ${warning}`);
|
|
3057
|
+
}
|
|
3058
|
+
} else if (command === "update") {
|
|
3059
|
+
const result = update({
|
|
3060
|
+
sourceRoot: packageRoot,
|
|
3061
|
+
targetRoot: parsed.values.target,
|
|
3062
|
+
dryRun: parsed.values["dry-run"],
|
|
3063
|
+
force: parsed.values.force
|
|
3064
|
+
});
|
|
3065
|
+
console.log(
|
|
3066
|
+
`${result.dryRun ? "Dry run: would update" : "Updated"} ${result.manifest.profile} (${result.manifest.adapter}) in ${result.targetRoot}`
|
|
3067
|
+
);
|
|
3068
|
+
console.log(
|
|
3069
|
+
`files: ${result.manifest.files.length} managed file(s) + ${result.manifest.block.path} block`
|
|
3070
|
+
);
|
|
3071
|
+
for (const warning of result.warnings) {
|
|
3072
|
+
console.log(`warning: ${warning}`);
|
|
3073
|
+
}
|
|
3074
|
+
} else if (command === "uninstall") {
|
|
3075
|
+
uninstall({
|
|
3076
|
+
targetRoot: parsed.values.target,
|
|
3077
|
+
dryRun: parsed.values["dry-run"],
|
|
3078
|
+
force: parsed.values.force
|
|
3079
|
+
});
|
|
3080
|
+
console.log(
|
|
3081
|
+
`${parsed.values["dry-run"] ? "Dry run: would uninstall" : "Uninstalled"} from ${parsed.values.target}`
|
|
3082
|
+
);
|
|
3083
|
+
} else if (command === "doctor") {
|
|
3084
|
+
const report = doctor({ targetRoot: parsed.values.target });
|
|
3085
|
+
for (const warning of report.warnings) {
|
|
3086
|
+
console.log(`warning: ${warning}`);
|
|
3087
|
+
}
|
|
3088
|
+
if (!report.ok) {
|
|
3089
|
+
for (const error of report.errors) {
|
|
3090
|
+
console.error(`error: ${error}`);
|
|
3091
|
+
}
|
|
3092
|
+
process.exitCode = 1;
|
|
3093
|
+
} else {
|
|
3094
|
+
console.log(`doctor: ok (${parsed.values.target})`);
|
|
3095
|
+
}
|
|
3096
|
+
} else if (command === "fixtures") {
|
|
3097
|
+
const action = parsed.positionals[1] ?? "check";
|
|
3098
|
+
if (action === "check") {
|
|
3099
|
+
const checks = checkAll(process.cwd(), parsed.values.dir ?? "fixtures");
|
|
3100
|
+
for (const check of checks) {
|
|
3101
|
+
const label = check.ok ? "ok " : "FAIL";
|
|
3102
|
+
const holdout = check.hidden ? " (hidden)" : "";
|
|
3103
|
+
console.log(`${label} ${check.id}${holdout}: ${check.detail}`);
|
|
3104
|
+
}
|
|
3105
|
+
const failed = checks.filter((check) => !check.ok);
|
|
3106
|
+
const hidden = checks.filter((check) => check.hidden).length;
|
|
3107
|
+
console.log(
|
|
3108
|
+
`fixtures: ${checks.length - failed.length}/${checks.length} valid, ${hidden} marked hidden holdout`
|
|
3109
|
+
);
|
|
3110
|
+
if (failed.length > 0) {
|
|
3111
|
+
process.exitCode = 1;
|
|
3112
|
+
}
|
|
3113
|
+
} else if (action === "run") {
|
|
3114
|
+
const all = loadFixtures(process.cwd(), parsed.values.dir ?? "fixtures");
|
|
3115
|
+
let fixtures = all;
|
|
3116
|
+
if (parsed.values.id.length > 0) {
|
|
3117
|
+
const known = new Set(all.map((fixture) => fixture.id));
|
|
3118
|
+
const unknown = parsed.values.id.filter((id) => !known.has(id));
|
|
3119
|
+
if (unknown.length > 0) {
|
|
3120
|
+
throw new Error(`Unknown fixture id(s): ${unknown.join(", ")}`);
|
|
3121
|
+
}
|
|
3122
|
+
fixtures = all.filter((fixture) => parsed.values.id.includes(fixture.id));
|
|
3123
|
+
}
|
|
3124
|
+
const limit = parsed.values.limit ? positiveInt(parsed.values.limit, "--limit", fixtures.length) : void 0;
|
|
3125
|
+
const repeats = positiveInt(parsed.values.repeats, "--repeats", 1);
|
|
3126
|
+
const budget = positiveNumber(parsed.values["budget-usd"], "--budget-usd");
|
|
3127
|
+
const arm = parsed.values.arm ?? "both";
|
|
3128
|
+
if (arm !== "both" && arm !== "baseline" && arm !== "treatment") {
|
|
3129
|
+
throw new Error("--arm must be both, baseline, or treatment");
|
|
3130
|
+
}
|
|
3131
|
+
const armCount = arm === "both" ? 2 : 1;
|
|
3132
|
+
const ceiling = parsed.values["max-total-usd"] ? positiveNumber(parsed.values["max-total-usd"], "--max-total-usd") : 0;
|
|
3133
|
+
const count = limit === void 0 ? fixtures.length : Math.min(limit, fixtures.length);
|
|
3134
|
+
const harness = harnessOf(parsed.values.harness);
|
|
3135
|
+
const projected = projectedCostUsd(count, repeats, budget, armCount);
|
|
3136
|
+
const hidden = fixtures.slice(0, count).filter((fixture) => fixture.hidden).length;
|
|
3137
|
+
console.log(
|
|
3138
|
+
`plan: running ${count} of ${fixtures.length} selected fixture(s) (${hidden} marked hidden holdout), ${repeats} repeat(s), ${armCount} arm(s) [${arm}], $${budget.toFixed(2)} per run, harness ${harness}`
|
|
3139
|
+
);
|
|
3140
|
+
console.log(
|
|
3141
|
+
`projected cost of the caps: $${projected.toFixed(2)} \u2014 not a ceiling: the harness checks its budget between turns, so a capped run overshoots (measured 1.17x and 1.46x of --budget-usd on 2026-11-04)`
|
|
3142
|
+
);
|
|
3143
|
+
if (!parsed.values.live) {
|
|
3144
|
+
console.log(
|
|
3145
|
+
"dry run: no model was called and nothing was spent. Add --live with --max-total-usd to run it."
|
|
3146
|
+
);
|
|
3147
|
+
} else {
|
|
3148
|
+
if (ceiling === 0) {
|
|
3149
|
+
throw new Error(
|
|
3150
|
+
`--live needs --max-total-usd: this campaign projects $${projected.toFixed(2)}, state the approved ceiling`
|
|
3151
|
+
);
|
|
3152
|
+
}
|
|
3153
|
+
if (projected > ceiling) {
|
|
3154
|
+
throw new Error(
|
|
3155
|
+
`projected $${projected.toFixed(2)} exceeds the approved ceiling of $${ceiling.toFixed(2)}: lower --limit, --repeats, or --budget-usd`
|
|
3156
|
+
);
|
|
3157
|
+
}
|
|
3158
|
+
if (armCount === 1) {
|
|
3159
|
+
console.log(
|
|
3160
|
+
'single arm: this run cannot compare baseline against treatment, it only answers "did the harness behave"'
|
|
3161
|
+
);
|
|
3162
|
+
}
|
|
3163
|
+
const report = runFixtureStudy({
|
|
3164
|
+
sourceRoot: process.cwd(),
|
|
3165
|
+
profileId: parsed.values.profile,
|
|
3166
|
+
fixtures,
|
|
3167
|
+
limit,
|
|
3168
|
+
repeats,
|
|
3169
|
+
budgetUsd: budget,
|
|
3170
|
+
arm,
|
|
3171
|
+
model: parsed.values.model,
|
|
3172
|
+
settingSources: parsed.values["setting-sources"],
|
|
3173
|
+
adapterId: HARNESS_ADAPTERS[harness],
|
|
3174
|
+
runner: (cwd, prompt, runBudget) => harness === "opencode" ? runOpencode(cwd, prompt, runBudget, { model: parsed.values.model }) : runClaude(cwd, prompt, runBudget, {
|
|
3175
|
+
model: parsed.values.model,
|
|
3176
|
+
settingSources: parsed.values["setting-sources"],
|
|
3177
|
+
// Edits are the one thing the agent must be able to do; shell and network
|
|
3178
|
+
// stay denied, so the grade comes from the tests this tool runs afterwards.
|
|
3179
|
+
agentArgs: ["--permission-mode", "acceptEdits", "--permission-prompts", "none"]
|
|
3180
|
+
}),
|
|
3181
|
+
outDir: parsed.values.out ?? join8(process.cwd(), ".hahahah", "coding")
|
|
3182
|
+
});
|
|
3183
|
+
printStudy(report);
|
|
3184
|
+
}
|
|
3185
|
+
} else if (action === "diagnose") {
|
|
3186
|
+
const ids = parsed.values.id;
|
|
3187
|
+
if (ids.length !== 1) {
|
|
3188
|
+
throw new Error("fixtures diagnose needs exactly one --id <fixture>");
|
|
3189
|
+
}
|
|
3190
|
+
const arm = parsed.values.arm ?? "treatment";
|
|
3191
|
+
if (arm !== "baseline" && arm !== "treatment") {
|
|
3192
|
+
throw new Error("--arm must be baseline or treatment for a single-arm diagnostic");
|
|
3193
|
+
}
|
|
3194
|
+
const fixture = loadFixtures(process.cwd(), parsed.values.dir ?? "fixtures").find(
|
|
3195
|
+
(candidate) => candidate.id === ids[0]
|
|
3196
|
+
);
|
|
3197
|
+
if (!fixture) {
|
|
3198
|
+
throw new Error(`Unknown fixture id(s): ${ids[0]}`);
|
|
3199
|
+
}
|
|
3200
|
+
const budget = positiveNumber(parsed.values["budget-usd"], "--budget-usd");
|
|
3201
|
+
const projected = projectedCostUsd(1, 1, budget, 1);
|
|
3202
|
+
const ceiling = parsed.values["max-total-usd"] ? positiveNumber(parsed.values["max-total-usd"], "--max-total-usd") : 0;
|
|
3203
|
+
console.log(
|
|
3204
|
+
`diagnostic: ${fixture.id}, one arm [${arm}], $${budget.toFixed(2)} per run, projected $${projected.toFixed(2)} (caps are soft, see docs/phase-3-live-suite-results.md)`
|
|
3205
|
+
);
|
|
3206
|
+
if (!parsed.values.live) {
|
|
3207
|
+
console.log("dry run: no model was called. Add --live with --max-total-usd to run it.");
|
|
3208
|
+
} else {
|
|
3209
|
+
if (ceiling === 0) {
|
|
3210
|
+
throw new Error(
|
|
3211
|
+
`--live needs --max-total-usd: this diagnostic projects $${projected.toFixed(2)}`
|
|
3212
|
+
);
|
|
3213
|
+
}
|
|
3214
|
+
if (projected > ceiling) {
|
|
3215
|
+
throw new Error(
|
|
3216
|
+
`projected $${projected.toFixed(2)} exceeds the approved ceiling of $${ceiling.toFixed(2)}`
|
|
3217
|
+
);
|
|
3218
|
+
}
|
|
3219
|
+
const result = diagnoseFixture({
|
|
3220
|
+
sourceRoot: process.cwd(),
|
|
3221
|
+
profileId: parsed.values.profile,
|
|
3222
|
+
fixture,
|
|
3223
|
+
arm,
|
|
3224
|
+
budgetUsd: budget,
|
|
3225
|
+
model: parsed.values.model,
|
|
3226
|
+
settingSources: parsed.values["setting-sources"],
|
|
3227
|
+
outDir: parsed.values.out ?? join8(process.cwd(), ".hahahah", "diagnostic")
|
|
3228
|
+
});
|
|
3229
|
+
const { summary, graded } = result;
|
|
3230
|
+
const tools = Object.entries(summary.tools).map(([name, count]) => `${name} x${count}`).join(", ");
|
|
3231
|
+
console.log(
|
|
3232
|
+
`stream: subtype=${summary.subtype ?? "?"} turns=${summary.numTurns ?? "?"} cost=$${(summary.totalCostUsd ?? 0).toFixed(4)} events=${summary.lines}`
|
|
3233
|
+
);
|
|
3234
|
+
console.log(`tools attempted: ${tools || "none"}`);
|
|
3235
|
+
for (const denial of summary.denials) {
|
|
3236
|
+
console.log(`DENIED: ${denial}`);
|
|
3237
|
+
}
|
|
3238
|
+
for (const error of summary.toolErrors) {
|
|
3239
|
+
console.log(`tool error: ${error}`);
|
|
3240
|
+
}
|
|
3241
|
+
console.log(
|
|
3242
|
+
`workspace: changed=[${graded.changedFiles.join(", ")}] new=[${graded.otherNewFiles.join(", ")}] missing=[${graded.missingFiles.join(", ")}]`
|
|
3243
|
+
);
|
|
3244
|
+
console.log(`grade=${graded.grade} tests pass=${graded.pass} fail=${graded.fail}`);
|
|
3245
|
+
console.log(`raw stream kept: ${result.rawPath}`);
|
|
3246
|
+
console.log(`workspace kept for inspection: ${result.workspace} (delete when done)`);
|
|
3247
|
+
if (graded.run.error) {
|
|
3248
|
+
process.exitCode = 1;
|
|
3249
|
+
}
|
|
3250
|
+
}
|
|
3251
|
+
} else {
|
|
3252
|
+
printUsage();
|
|
3253
|
+
process.exitCode = 1;
|
|
3254
|
+
}
|
|
3255
|
+
} else if (command === "eval") {
|
|
3256
|
+
const suite = loadSuite(process.cwd(), parsed.values.suite);
|
|
3257
|
+
const limit = parsed.values.limit ? Number.parseInt(parsed.values.limit, 10) : void 0;
|
|
3258
|
+
const repeats = parsed.values.repeats ? Number.parseInt(parsed.values.repeats, 10) : 1;
|
|
3259
|
+
const budget = Number.parseFloat(parsed.values["budget-usd"]);
|
|
3260
|
+
if (limit !== void 0 && (!Number.isInteger(limit) || limit <= 0)) {
|
|
3261
|
+
throw new Error("--limit must be a positive integer");
|
|
3262
|
+
}
|
|
3263
|
+
if (!Number.isInteger(repeats) || repeats < 1) {
|
|
3264
|
+
throw new Error("--repeats must be a positive integer");
|
|
3265
|
+
}
|
|
3266
|
+
if (!Number.isFinite(budget) || budget <= 0) {
|
|
3267
|
+
throw new Error("--budget-usd must be a positive number");
|
|
3268
|
+
}
|
|
3269
|
+
let tasks = suite.tasks;
|
|
3270
|
+
if (parsed.values.task.length > 0) {
|
|
3271
|
+
const known = new Set(suite.tasks.map((task) => task.id));
|
|
3272
|
+
const unknown = parsed.values.task.filter((id) => !known.has(id));
|
|
3273
|
+
if (unknown.length > 0) {
|
|
3274
|
+
throw new Error(`Unknown task id(s): ${unknown.join(", ")} in suite ${suite.suite}`);
|
|
3275
|
+
}
|
|
3276
|
+
tasks = suite.tasks.filter((task) => parsed.values.task.includes(task.id));
|
|
3277
|
+
}
|
|
3278
|
+
const harness = harnessOf(parsed.values.harness);
|
|
3279
|
+
const report = runEval({
|
|
3280
|
+
sourceRoot: process.cwd(),
|
|
3281
|
+
profileId: parsed.values.profile,
|
|
3282
|
+
tasks,
|
|
3283
|
+
suite: suite.suite,
|
|
3284
|
+
limit,
|
|
3285
|
+
repeats,
|
|
3286
|
+
budgetUsd: budget,
|
|
3287
|
+
model: parsed.values.model,
|
|
3288
|
+
settingSources: parsed.values["setting-sources"],
|
|
3289
|
+
adapterId: HARNESS_ADAPTERS[harness],
|
|
3290
|
+
harness,
|
|
3291
|
+
runner: runnerFor(harness, {
|
|
3292
|
+
model: parsed.values.model,
|
|
3293
|
+
settingSources: parsed.values["setting-sources"]
|
|
3294
|
+
}),
|
|
3295
|
+
// The runner is injected here so the harness can be chosen, which would otherwise disable
|
|
3296
|
+
// the auto-probe. A real campaign always talks to a real gateway, so it must always probe.
|
|
3297
|
+
probe: true,
|
|
3298
|
+
outDir: parsed.values.out ?? join8(process.cwd(), ".hahahah", "eval")
|
|
3299
|
+
});
|
|
3300
|
+
const runs = report.results.reduce(
|
|
3301
|
+
(total, result) => total + result.baseline.runs.length + result.treatment.runs.length,
|
|
3302
|
+
0
|
|
3303
|
+
);
|
|
3304
|
+
console.log(
|
|
3305
|
+
`eval: ${report.results.length} task(s) x ${report.repeats} repeat(s) = ${runs} runs, cost $${report.total_cost_usd.toFixed(4)}`
|
|
3306
|
+
);
|
|
3307
|
+
if (report.aborted) {
|
|
3308
|
+
const cause = report.aborted.kind === "quota" ? "the provider refused further requests" : "consecutive runs were answered with nothing at all - the harness or route is not answering";
|
|
3309
|
+
console.log(
|
|
3310
|
+
`ABORTED after ${report.aborted.afterRuns} run(s): ${cause} (${report.aborted.reason.slice(0, 120)}). Not measured: ${report.aborted.remainingTasks.join(", ") || `${report.aborted.partialTask} (partial)`}. Nothing further was spent.`
|
|
3311
|
+
);
|
|
3312
|
+
}
|
|
3313
|
+
for (const result of report.results) {
|
|
3314
|
+
const state = (arm) => {
|
|
3315
|
+
if (arm.consensus === "error") return `ERROR: ${arm.runs[0]?.error ?? "no runs"}`;
|
|
3316
|
+
const passed = arm.runStatuses.filter((status) => status === "pass").length;
|
|
3317
|
+
const findings = [...new Set(arm.screens.findings)];
|
|
3318
|
+
const detail = findings.length > 0 ? ` (${findings.join("; ")})` : "";
|
|
3319
|
+
return `${arm.consensus} ${passed}/${arm.runs.length} med $${arm.median.costUsd.toFixed(4)}${detail}`;
|
|
3320
|
+
};
|
|
3321
|
+
console.log(
|
|
3322
|
+
`- ${result.task.id}: baseline=${state(result.baseline)}, treatment=${state(result.treatment)}`
|
|
3323
|
+
);
|
|
3324
|
+
}
|
|
3325
|
+
const { baseline, treatment } = report.summary;
|
|
3326
|
+
if (report.summary.tasks_with_checks > 0) {
|
|
3327
|
+
console.log(
|
|
3328
|
+
`screens: baseline ${baseline.pass} pass / ${baseline.fail} fail / ${baseline.unknown} unknown; treatment ${treatment.pass} pass / ${treatment.fail} fail / ${treatment.unknown} unknown`
|
|
3329
|
+
);
|
|
3330
|
+
}
|
|
3331
|
+
const failedRun = report.results.flatMap((result) => [...result.baseline.runs, ...result.treatment.runs]).find((run) => run.error);
|
|
3332
|
+
if (failedRun) {
|
|
3333
|
+
process.exitCode = 1;
|
|
3334
|
+
} else if (parsed.values.strict && treatment.fail > 0) {
|
|
3335
|
+
console.error(`strict: ${treatment.fail} treatment screen(s) failed across repeats`);
|
|
3336
|
+
process.exitCode = 1;
|
|
3337
|
+
}
|
|
3338
|
+
} else {
|
|
3339
|
+
printUsage();
|
|
3340
|
+
process.exitCode = 1;
|
|
3341
|
+
}
|
|
3342
|
+
} catch (error) {
|
|
3343
|
+
if (error instanceof BuildError || error instanceof InstallError) {
|
|
3344
|
+
console.error(`Command failed: ${error.message}`);
|
|
3345
|
+
} else {
|
|
3346
|
+
console.error(error);
|
|
3347
|
+
}
|
|
3348
|
+
process.exitCode = 1;
|
|
3349
|
+
} finally {
|
|
3350
|
+
try {
|
|
3351
|
+
const latest = await updateCheck;
|
|
3352
|
+
if (latest) {
|
|
3353
|
+
console.error(formatUpdateNotice(version, latest));
|
|
3354
|
+
}
|
|
3355
|
+
} catch {
|
|
3356
|
+
}
|
|
3357
|
+
}
|
|
3358
|
+
}
|
|
3359
|
+
export {
|
|
3360
|
+
runnerFor
|
|
3361
|
+
};
|