@arjunkhera/atlas 0.3.37 → 0.3.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/door/kit-releases.json +4 -0
- package/package.json +1 -1
- package/tests/contract.mjs +10 -2
package/door/kit-releases.json
CHANGED
|
@@ -36,6 +36,10 @@
|
|
|
36
36
|
{
|
|
37
37
|
"kit_sha256": "b5c36f685ba476e09a54e3207e2444b098cc6a26dded8dca65b01e751fe2cbe4",
|
|
38
38
|
"first_package": "0.3.34"
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"kit_sha256": "c00e87170dd5f1b477d144eca976db76580d94ede6104d76609aac040851d488",
|
|
42
|
+
"first_package": "0.3.37"
|
|
39
43
|
}
|
|
40
44
|
]
|
|
41
45
|
}
|
package/package.json
CHANGED
package/tests/contract.mjs
CHANGED
|
@@ -117,6 +117,12 @@ function load(path, options) {
|
|
|
117
117
|
}
|
|
118
118
|
|
|
119
119
|
const listOf = (x) => (Array.isArray(x) ? x.map(String) : x === undefined ? [] : [String(x)]);
|
|
120
|
+
const oneLine = (t) => String(t ?? '').replace(/\s+/g, ' ').trim();
|
|
121
|
+
// The reason of each run of a repeat that did not pass, for the log: "; run 2: ..., run 3: ...".
|
|
122
|
+
const whyNot = (list) => {
|
|
123
|
+
const lines = list.filter((r) => r.result !== 'pass' && r.result !== 'not here').map((r) => `run ${r.run}: ${oneLine(r.error ?? r.why ?? r.stopped ?? r.result).slice(0, 240)}`);
|
|
124
|
+
return lines.length ? `; ${lines.join('; ')}` : '';
|
|
125
|
+
};
|
|
120
126
|
|
|
121
127
|
export function scenario(path, body, options = {}) {
|
|
122
128
|
const ctx = load(path, options);
|
|
@@ -238,7 +244,7 @@ export function scenario(path, body, options = {}) {
|
|
|
238
244
|
a.result = blocked ? 'blocked' : ok ? 'pass' : unsure && passed + unsure >= repeat.pass ? 'unsure' : 'fail';
|
|
239
245
|
a.proof = {
|
|
240
246
|
repeat: { runs: repeat.runs, pass: repeat.pass, passed }, runs: list,
|
|
241
|
-
...(blocked ? { why: `the repeat stopped in run ${list.length}: ${state.redactor.text(error.message)}` } : ok ? {} : { error: `passed in ${passed} of ${list.length} runs; the rule is ${repeat.pass} of ${repeat.runs}` }),
|
|
247
|
+
...(blocked ? { why: `the repeat stopped in run ${list.length}: ${state.redactor.text(error.message)}` } : ok ? {} : { error: `passed in ${passed} of ${list.length} runs; the rule is ${repeat.pass} of ${repeat.runs}${whyNot(list)}` }),
|
|
242
248
|
};
|
|
243
249
|
}
|
|
244
250
|
ev.write();
|
|
@@ -454,7 +460,9 @@ function makeRun({ ctx, state, way, env, ev, adapter, round = null }) {
|
|
|
454
460
|
if (!key) throw new Blocked(`the judge "${names[0]}" needs the key ${keyName}: list it in secrets of tests.yaml, and give its value`);
|
|
455
461
|
const out = await askJudge({ name: names[0], judge, words: a.words, evidence, key, meter: state.meter, redactor: state.redactor });
|
|
456
462
|
open('run.judge');
|
|
457
|
-
|
|
463
|
+
// A judge that did not pass says why in its own words: the first answer that is not YES.
|
|
464
|
+
const doubt = out.answers.find((x) => x.answer !== 'YES')?.reply;
|
|
465
|
+
ev.setResult(a.fullId, target, out.result, { judge: names[0], answers: out.answers, cost: out.cost, calls, settled_by: way, ...(out.result === 'pass' ? {} : { error: doubt ? `${out.why}: ${state.redactor.text(oneLine(doubt)).slice(0, 200)}` : out.why }) });
|
|
458
466
|
},
|
|
459
467
|
// Money that a product's own model call spent (the router, say), read from its answer.
|
|
460
468
|
// It counts against budget.money with the judge and the agent calls.
|