@skill-harness/core 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/types.d.ts +19 -0
- package/dist/adjudication.d.ts +9 -1
- package/dist/adjudication.js +41 -33
- package/dist/adoption.d.ts +128 -0
- package/dist/adoption.js +102 -0
- package/dist/arms.d.ts +1 -1
- package/dist/capture-trace-types.d.ts +2 -0
- package/dist/command-cost.d.ts +9 -0
- package/dist/command-cost.js +23 -0
- package/dist/comparison.d.ts +1 -0
- package/dist/comparison.js +9 -3
- package/dist/execution-trace.js +9 -1
- package/dist/factory-calibration.d.ts +63 -0
- package/dist/factory-calibration.js +115 -0
- package/dist/grade.d.ts +2 -0
- package/dist/grade.js +2 -1
- package/dist/index.d.ts +21 -0
- package/dist/index.js +21 -0
- package/dist/intervention-results.d.ts +6 -0
- package/dist/intervention-results.js +52 -0
- package/dist/intervention.d.ts +123 -0
- package/dist/intervention.js +218 -0
- package/dist/investigation.d.ts +94 -0
- package/dist/investigation.js +148 -0
- package/dist/journal.d.ts +1 -1
- package/dist/judge-agreement.d.ts +19 -0
- package/dist/judge-agreement.js +31 -0
- package/dist/judge-policy.js +5 -3
- package/dist/lift.js +1 -1
- package/dist/lint.js +1 -1
- package/dist/prompt-normalization.d.ts +6 -0
- package/dist/prompt-normalization.js +14 -0
- package/dist/qualification-capture.d.ts +44 -0
- package/dist/qualification-capture.js +163 -0
- package/dist/qualification-config.d.ts +233 -0
- package/dist/qualification-config.js +694 -0
- package/dist/qualification-lock.d.ts +15 -0
- package/dist/qualification-lock.js +87 -0
- package/dist/qualification-oauth-directory.d.ts +79 -0
- package/dist/qualification-oauth-directory.js +444 -0
- package/dist/qualification-panel-store.d.ts +14 -0
- package/dist/qualification-panel-store.js +188 -0
- package/dist/qualification-panels.d.ts +99 -0
- package/dist/qualification-panels.js +91 -0
- package/dist/qualification-process.d.ts +13 -0
- package/dist/qualification-process.js +190 -0
- package/dist/qualification-runner.d.ts +203 -0
- package/dist/qualification-runner.js +2037 -0
- package/dist/qualification-store.d.ts +187 -0
- package/dist/qualification-store.js +833 -0
- package/dist/qualification-validation.d.ts +12 -0
- package/dist/qualification-validation.js +8 -0
- package/dist/regate.js +42 -10
- package/dist/regrade.d.ts +3 -1
- package/dist/regrade.js +37 -5
- package/dist/reps.d.ts +4 -2
- package/dist/reps.js +13 -4
- package/dist/rescore.js +2 -1
- package/dist/results.d.ts +53 -7
- package/dist/results.js +210 -7
- package/dist/run.js +66 -17
- package/dist/score.d.ts +4 -3
- package/dist/score.js +13 -4
- package/dist/screen.d.ts +31 -0
- package/dist/screen.js +105 -0
- package/dist/seeded.d.ts +2 -1
- package/dist/seeded.js +1 -0
- package/dist/sources.d.ts +7 -1
- package/dist/sources.js +13 -5
- package/dist/spec.d.ts +1 -1
- package/dist/spec.js +1 -1
- package/dist/stability.js +1 -1
- package/dist/trajectory-gates.d.ts +14 -14
- package/dist/trajectory-gates.js +18 -34
- package/dist/trusted-host-supervision.d.ts +25 -0
- package/dist/trusted-host-supervision.js +94 -0
- package/dist/vote-panel.d.ts +33 -0
- package/dist/vote-panel.js +41 -0
- package/dist/work-capture-schema.d.ts +38 -0
- package/dist/work-capture-schema.js +33 -0
- package/dist/work-capture.d.ts +43 -0
- package/dist/work-capture.js +81 -0
- package/dist/work-signals.d.ts +55 -0
- package/dist/work-signals.js +71 -0
- package/dist/workspace.js +4 -1
- package/package.json +3 -3
- package/schemas/qualification-cell-result-v1.schema.json +32 -0
- package/schemas/qualification-config-v1.schema.json +244 -0
- package/schemas/qualification-invocation-request-v1.schema.json +59 -0
- package/schemas/qualification-judge-panel-result-v1.schema.json +42 -0
- package/schemas/results-v2.schema.json +19 -21
- package/schemas/results-v3.schema.json +654 -0
- package/schemas/specification-v1.schema.json +1 -1
- package/schemas/trajectory-event-v1.schema.json +20 -1
package/dist/screen.js
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { deliveryStatusForObservations } from "./results.js";
|
|
2
|
+
import { collapseVotePanel } from "./vote-panel.js";
|
|
3
|
+
function classify(rate) {
|
|
4
|
+
if (rate.n === 0)
|
|
5
|
+
return "UNKNOWN";
|
|
6
|
+
const p = rate.passes / rate.n;
|
|
7
|
+
if (p >= 0.8)
|
|
8
|
+
return "CEILING";
|
|
9
|
+
if (p <= 0.1)
|
|
10
|
+
return "FLOOR";
|
|
11
|
+
if (p >= 0.2 && p <= 0.7)
|
|
12
|
+
return "INFORMATIVE";
|
|
13
|
+
return "UNKNOWN";
|
|
14
|
+
}
|
|
15
|
+
/** Pure/offline: derive only from v3 retained invocation and vote evidence. */
|
|
16
|
+
export function screenResults(results) {
|
|
17
|
+
const rows = new Map();
|
|
18
|
+
const criteria = new Map();
|
|
19
|
+
for (const result of results) {
|
|
20
|
+
const observations = result.schema === 3 ? result.subject_invocations : [];
|
|
21
|
+
for (const scenario of result.scenarios) {
|
|
22
|
+
const rowKey = `${result.skill}\u0000${result.model}\u0000${scenario.id}`;
|
|
23
|
+
const row = rows.get(rowKey) ?? { control: { passes: 0, n: 0 }, treatment: { passes: 0, n: 0 }, not_measured: 0 };
|
|
24
|
+
for (const rep of scenario.rep_judgments ?? []) {
|
|
25
|
+
const repObservations = observations.filter(observation => observation.scenario_id === scenario.id && observation.repetition === rep.repetition);
|
|
26
|
+
const deliveryStatus = deliveryStatusForObservations(repObservations);
|
|
27
|
+
if (deliveryStatus === "NOT-MEASURED")
|
|
28
|
+
row.not_measured++;
|
|
29
|
+
if (deliveryStatus !== "PASS")
|
|
30
|
+
continue;
|
|
31
|
+
const terminalAttempt = Math.max(...repObservations.map(observation => observation.attempt ?? 0));
|
|
32
|
+
const mechanisms = new Set(repObservations.filter(observation => (observation.attempt ?? 0) === terminalAttempt).map(observation => observation.prompt.mechanism));
|
|
33
|
+
if (mechanisms.size !== 1)
|
|
34
|
+
continue;
|
|
35
|
+
const rate = mechanisms.has("none") ? row.control : row.treatment;
|
|
36
|
+
let panelVerdict = rep.recorded_verdict;
|
|
37
|
+
if (rep.judgments.length > 0) {
|
|
38
|
+
const clean = rep.judgments.filter(judgment => !judgment.suspect && (judgment.verdict === "PASS" || judgment.verdict === "FAIL"));
|
|
39
|
+
if (clean.length === 0)
|
|
40
|
+
continue;
|
|
41
|
+
panelVerdict = clean.length === 1 ? clean[0].verdict : collapseVotePanel(rep.judgments).verdict ?? "JUDGE-AMBIGUOUS";
|
|
42
|
+
}
|
|
43
|
+
else if (rep.objective?.status === "FAIL" && rep.recorded_verdict === "FAIL") {
|
|
44
|
+
panelVerdict = "FAIL"; // objective behavioral evidence needs no prose judge
|
|
45
|
+
}
|
|
46
|
+
else
|
|
47
|
+
continue;
|
|
48
|
+
if (scenario.adjudication?.repetition === rep.repetition) {
|
|
49
|
+
if (scenario.adjudication.state === "unresolved" || !scenario.adjudication.verdict)
|
|
50
|
+
continue;
|
|
51
|
+
const adjudicated = collapseVotePanel(scenario.adjudication.judgments);
|
|
52
|
+
if (adjudicated.state !== scenario.adjudication.state || adjudicated.verdict !== scenario.adjudication.verdict)
|
|
53
|
+
continue;
|
|
54
|
+
panelVerdict = adjudicated.verdict;
|
|
55
|
+
}
|
|
56
|
+
const effective = rep.objective?.status === "ERROR" ? "ERROR" : rep.objective?.status === "NOT-MEASURED" ? "NOT-MEASURED" : rep.objective?.status === "FAIL" ? "FAIL" : panelVerdict;
|
|
57
|
+
if (effective !== "PASS" && effective !== "FAIL")
|
|
58
|
+
continue;
|
|
59
|
+
rate.n++;
|
|
60
|
+
if (effective === "PASS")
|
|
61
|
+
rate.passes++;
|
|
62
|
+
for (const judgment of rep.judgments) {
|
|
63
|
+
if (judgment.suspect || (judgment.verdict !== "PASS" && judgment.verdict !== "FAIL"))
|
|
64
|
+
continue;
|
|
65
|
+
for (const vote of judgment.criteria ?? [])
|
|
66
|
+
addCriterion(criteria, result.skill, result.model, scenario.id, vote);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const adjudicatedRep = scenario.adjudication?.repetition;
|
|
70
|
+
const adjudicationDelivery = result.schema === 3 && adjudicatedRep !== undefined
|
|
71
|
+
? deliveryStatusForObservations(observations.filter(observation => observation.scenario_id === scenario.id && observation.repetition === adjudicatedRep))
|
|
72
|
+
: "ERROR";
|
|
73
|
+
for (const judgment of adjudicationDelivery === "PASS" ? scenario.adjudication?.judgments.filter(judgment => judgment.ordinal > 1) ?? [] : []) {
|
|
74
|
+
if (judgment.suspect || (judgment.verdict !== "PASS" && judgment.verdict !== "FAIL"))
|
|
75
|
+
continue;
|
|
76
|
+
for (const vote of judgment.criteria ?? [])
|
|
77
|
+
addCriterion(criteria, result.skill, result.model, scenario.id, vote);
|
|
78
|
+
}
|
|
79
|
+
rows.set(rowKey, row);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
scenarios: [...rows.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([key, row]) => { const [skill, model, id] = key.split("\u0000"); return { skill, model, id, ...row, classification: classify(row.control) }; }),
|
|
84
|
+
criteria: [...criteria.values()].sort((a, b) => a.scenario_id.localeCompare(b.scenario_id) || a.criterion - b.criterion).map(x => ({ ...x, fail_rate: x.failures / x.n })),
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
function addCriterion(map, skill, model, id, vote) {
|
|
88
|
+
if (vote.verdict === "ERROR")
|
|
89
|
+
return;
|
|
90
|
+
const key = `${skill}\u0000${model}\u0000${id}:${vote.index}`, row = map.get(key) ?? { skill, model, scenario_id: id, criterion: vote.index, failures: 0, n: 0 };
|
|
91
|
+
row.n++;
|
|
92
|
+
if (vote.verdict === "FAIL")
|
|
93
|
+
row.failures++;
|
|
94
|
+
map.set(key, row);
|
|
95
|
+
}
|
|
96
|
+
export function formatScreen(report) {
|
|
97
|
+
const lines = ["skill/model/scenario control treatment not-measured class"];
|
|
98
|
+
for (const s of report.scenarios)
|
|
99
|
+
lines.push(`${s.skill}/${s.model}/${s.id} ${s.control.passes}/${s.control.n} ${s.treatment.passes}/${s.treatment.n} ${s.not_measured} ${s.classification}`);
|
|
100
|
+
lines.push("", "skill/model/criterion failures/n fail-rate");
|
|
101
|
+
for (const c of report.criteria)
|
|
102
|
+
lines.push(`${c.skill}/${c.model}/${c.scenario_id}.${c.criterion} ${c.failures}/${c.n} ${(c.fail_rate * 100).toFixed(1)}%`);
|
|
103
|
+
return lines.join("\n");
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=screen.js.map
|
package/dist/seeded.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Scenario } from "./spec.js";
|
|
2
2
|
import type { ExecutionTraceV1 } from "./capture-trace-types.js";
|
|
3
3
|
import type { TrajectoryEventV1 } from "./trajectory-gates.js";
|
|
4
|
-
import type { HarnessAdapter, ModelRef, RunMode } from "./adapters/types.js";
|
|
4
|
+
import type { HarnessAdapter, ModelRef, RunMode, PromptProvenance } from "./adapters/types.js";
|
|
5
5
|
import { type ExecResult } from "./util/exec.js";
|
|
6
6
|
interface SeededOpts {
|
|
7
7
|
skillDir: string;
|
|
@@ -37,6 +37,7 @@ interface SeededOpts {
|
|
|
37
37
|
armExtensions?: string[];
|
|
38
38
|
/** The arm's env for the subject process; `<run-dir>` already substituted by the caller. */
|
|
39
39
|
armEnv?: Record<string, string>;
|
|
40
|
+
onPromptObservation?: (observation: PromptProvenance) => void;
|
|
40
41
|
}
|
|
41
42
|
/**
|
|
42
43
|
* Result of one vitest invocation — deliberately `ExecResult`, not a narrower
|
package/dist/seeded.js
CHANGED
|
@@ -175,6 +175,7 @@ export async function runSeeded(scenario, opts) {
|
|
|
175
175
|
],
|
|
176
176
|
eventSources: scenario.eventSources,
|
|
177
177
|
...(opts.armEnv ? { armEnv: opts.armEnv } : {}),
|
|
178
|
+
...(opts.onPromptObservation ? { onPromptObservation: opts.onPromptObservation } : {}),
|
|
178
179
|
};
|
|
179
180
|
// A trace-gated seeded scenario runs through the structured path so the tool
|
|
180
181
|
// calls are recorded; everything downstream (gates, diff, transcript) is
|
package/dist/sources.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Scenario } from "./spec.js";
|
|
2
|
+
export { PROMPT_NORMALIZATION_SOURCE_DIGEST, PROMPT_NORMALIZATION_SOURCE_KEY } from "./prompt-normalization.js";
|
|
2
3
|
/**
|
|
3
4
|
* What a run measured, as a map of `key -> sha256`, recorded in results.yaml so
|
|
4
5
|
* lint can prove a published result still describes the current inputs.
|
|
@@ -53,7 +54,7 @@ export declare const FIXTURE_PREFIX = "fixture:";
|
|
|
53
54
|
* | `stimulus:<id>` | mode, turns, workspace, remote, agent-file path, fixture path, `assert.vitest`, `post_test` path | the transcripts answer a different question | `run` (model + judge) |
|
|
54
55
|
* | `rubric:<id>` | title, checklist | transcripts fine, verdicts wrong | `grade` (judge only) |
|
|
55
56
|
* | `policy:<id>` | critical, reps, pass_threshold | only the scoring moved | `rescore` (free) |
|
|
56
|
-
* | `gates:<id>` | `diff_contains`, `diff_excludes` | needle wrong, behavior fine | `regate` (
|
|
57
|
+
* | `gates:<id>` | `diff_contains`, `diff_excludes` | needle wrong, behavior fine | `regate` (no subject call; judges fail→pass reps) |
|
|
57
58
|
* | `rubric:__persona` | spec-level `judge_persona` | every verdict in the skill | `grade` per model |
|
|
58
59
|
*
|
|
59
60
|
* Why this matters more than it looks: with one key, lint had exactly one remedy for
|
|
@@ -190,6 +191,11 @@ export declare const SKILL_KEY = "SKILL.md";
|
|
|
190
191
|
*/
|
|
191
192
|
export declare const SKILL_PROMPT_KEY = "skill:prompt";
|
|
192
193
|
export declare const PROMPT_PREFIX = "prompt:";
|
|
194
|
+
/** A prompt document split into its frontmatter text (null when it has none) and its body. */
|
|
195
|
+
export declare function splitPromptDoc(text: string): {
|
|
196
|
+
frontmatter: string | null;
|
|
197
|
+
body: string;
|
|
198
|
+
};
|
|
193
199
|
/**
|
|
194
200
|
* The digest of a prompt document: its body verbatim, plus every frontmatter field the
|
|
195
201
|
* model can actually receive.
|
package/dist/sources.js
CHANGED
|
@@ -2,6 +2,8 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import { readFileSync, readdirSync } from "node:fs";
|
|
3
3
|
import { isAbsolute, join, resolve } from "node:path";
|
|
4
4
|
import yaml from "js-yaml";
|
|
5
|
+
import { PROMPT_NORMALIZATION_SOURCE_DIGEST, PROMPT_NORMALIZATION_SOURCE_KEY } from "./prompt-normalization.js";
|
|
6
|
+
export { PROMPT_NORMALIZATION_SOURCE_DIGEST, PROMPT_NORMALIZATION_SOURCE_KEY } from "./prompt-normalization.js";
|
|
5
7
|
/**
|
|
6
8
|
* What a run measured, as a map of `key -> sha256`, recorded in results.yaml so
|
|
7
9
|
* lint can prove a published result still describes the current inputs.
|
|
@@ -56,7 +58,7 @@ export const FIXTURE_PREFIX = "fixture:";
|
|
|
56
58
|
* | `stimulus:<id>` | mode, turns, workspace, remote, agent-file path, fixture path, `assert.vitest`, `post_test` path | the transcripts answer a different question | `run` (model + judge) |
|
|
57
59
|
* | `rubric:<id>` | title, checklist | transcripts fine, verdicts wrong | `grade` (judge only) |
|
|
58
60
|
* | `policy:<id>` | critical, reps, pass_threshold | only the scoring moved | `rescore` (free) |
|
|
59
|
-
* | `gates:<id>` | `diff_contains`, `diff_excludes` | needle wrong, behavior fine | `regate` (
|
|
61
|
+
* | `gates:<id>` | `diff_contains`, `diff_excludes` | needle wrong, behavior fine | `regate` (no subject call; judges fail→pass reps) |
|
|
60
62
|
* | `rubric:__persona` | spec-level `judge_persona` | every verdict in the skill | `grade` per model |
|
|
61
63
|
*
|
|
62
64
|
* Why this matters more than it looks: with one key, lint had exactly one remedy for
|
|
@@ -182,7 +184,7 @@ function facets(s) {
|
|
|
182
184
|
const _assertExhaustive = restAssert;
|
|
183
185
|
void _assertExhaustive;
|
|
184
186
|
// `traceAssert` is a GATE, not stimulus: it is evaluated against a trace the run
|
|
185
|
-
// already saved, so `regate`
|
|
187
|
+
// already saved, so `regate` can re-answer it without re-running the subject model.
|
|
186
188
|
// Note the asymmetry with `env.extensions` in Phase 3, which IS stimulus — one
|
|
187
189
|
// changes what gets executed, the other only what we conclude from it.
|
|
188
190
|
const hasGates = diff_contains !== undefined || diff_excludes !== undefined || traceAssert !== undefined || trajectoryAssert !== undefined;
|
|
@@ -194,7 +196,7 @@ function facets(s) {
|
|
|
194
196
|
// below. Changing which extensions load changes what the model can DO, so the
|
|
195
197
|
// old transcripts describe a different agent and only a re-run can answer.
|
|
196
198
|
// Changing an assertion only changes what we conclude from evidence already on
|
|
197
|
-
// disk, which `regate` can redo
|
|
199
|
+
// disk, which `regate` can redo without a subject re-run.
|
|
198
200
|
// APPENDED CONDITIONALLY, never as a fixed slot. This tuple is positional and
|
|
199
201
|
// its hash is stored in every published results.yaml, so adding an
|
|
200
202
|
// unconditional element re-hashes every scenario that never used the field —
|
|
@@ -363,6 +365,8 @@ export function sourceHashes(ctx) {
|
|
|
363
365
|
* from drifting: a new key kind is defined once, for both sides.
|
|
364
366
|
*/
|
|
365
367
|
export function currentHashFor(key, ctx) {
|
|
368
|
+
if (key === PROMPT_NORMALIZATION_SOURCE_KEY)
|
|
369
|
+
return PROMPT_NORMALIZATION_SOURCE_DIGEST;
|
|
366
370
|
if (key === SKILL_KEY)
|
|
367
371
|
return fileSha256(resolve(ctx.skillDir, "SKILL.md"));
|
|
368
372
|
if (key === SKILL_PROMPT_KEY)
|
|
@@ -408,6 +412,8 @@ export function currentHashFor(key, ctx) {
|
|
|
408
412
|
}
|
|
409
413
|
/** Human label for a recorded key, used in lint messages. */
|
|
410
414
|
export function describeSourceKey(key) {
|
|
415
|
+
if (key === PROMPT_NORMALIZATION_SOURCE_KEY)
|
|
416
|
+
return "the prompt normalization rule registry";
|
|
411
417
|
if (key === PERSONA_KEY)
|
|
412
418
|
return "the judge persona";
|
|
413
419
|
// Deliberately the same label as the raw-bytes key: only one of the pair is ever
|
|
@@ -439,6 +445,8 @@ export function describeSourceKey(key) {
|
|
|
439
445
|
* in place. Naming the actual remedy is what converts that into a free command.
|
|
440
446
|
*/
|
|
441
447
|
export function remedyForKey(key) {
|
|
448
|
+
if (key === PROMPT_NORMALIZATION_SOURCE_KEY)
|
|
449
|
+
return "re-run — prompt provenance cannot be recomputed without the original model-visible payload";
|
|
442
450
|
if (key === PERSONA_KEY) {
|
|
443
451
|
return "re-grade each model's saved transcripts (`grade <run-dir>`) — judge-only, no model spend";
|
|
444
452
|
}
|
|
@@ -449,7 +457,7 @@ export function remedyForKey(key) {
|
|
|
449
457
|
return "re-score the saved reps (`rescore <run-dir>`) — free, offline";
|
|
450
458
|
}
|
|
451
459
|
if (key.startsWith(GATES_PREFIX)) {
|
|
452
|
-
return "re-evaluate
|
|
460
|
+
return "re-evaluate gates against saved artifacts (`regate <run-dir>`) — no subject call; fail→pass reps require judge calls";
|
|
453
461
|
}
|
|
454
462
|
// A pre-split run recorded one hash over stimulus + rubric + policy + gates, so
|
|
455
463
|
// which of them moved is genuinely unknowable from the record. Naming a cheap
|
|
@@ -498,7 +506,7 @@ const CAPABILITY_KEYS = new Set(["allowed-tools", "tools"]);
|
|
|
498
506
|
/** Opening `---` on line 1 only — a `---` further down is a horizontal rule. */
|
|
499
507
|
const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/;
|
|
500
508
|
/** A prompt document split into its frontmatter text (null when it has none) and its body. */
|
|
501
|
-
function splitPromptDoc(text) {
|
|
509
|
+
export function splitPromptDoc(text) {
|
|
502
510
|
const m = FRONTMATTER_RE.exec(text);
|
|
503
511
|
// Unterminated frontmatter falls through to "all body", the same call
|
|
504
512
|
// instruction-coverage.ts makes: treat the whole file as content rather than lose it.
|
package/dist/spec.d.ts
CHANGED
|
@@ -23,7 +23,7 @@ export interface SeededAssert {
|
|
|
23
23
|
}
|
|
24
24
|
export interface TrajectoryEventSource {
|
|
25
25
|
/** Native format adapter registered by the harness adapter. */
|
|
26
|
-
adapter: "normalized-v1" | "principal-assurance-v1" | "pi-daddy-v1";
|
|
26
|
+
adapter: "normalized-v1" | "principal-assurance-v1" | "pi-daddy-v1" | "pi-daddy-ledger-v3";
|
|
27
27
|
/** Workspace-relative file/glob. Never an absolute path or traversal. */
|
|
28
28
|
path: string;
|
|
29
29
|
/** Missing evidence is an error by default; false is for optional supplemental ledgers. */
|
package/dist/spec.js
CHANGED
|
@@ -97,7 +97,7 @@ function resolveEventSources(env, id, file) {
|
|
|
97
97
|
throw new SpecError(`scenario \`${id}\` env.event_sources must be a non-empty list`, file);
|
|
98
98
|
}
|
|
99
99
|
const allowedAdapters = new Set([
|
|
100
|
-
"normalized-v1", "principal-assurance-v1", "pi-daddy-v1",
|
|
100
|
+
"normalized-v1", "principal-assurance-v1", "pi-daddy-v1", "pi-daddy-ledger-v3",
|
|
101
101
|
]);
|
|
102
102
|
return raw.map((entry, index) => {
|
|
103
103
|
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
package/dist/stability.js
CHANGED
|
@@ -9,7 +9,7 @@ function conclusive(v) {
|
|
|
9
9
|
// Same rule lift.ts applies, for the same reason: ERROR is a harness failure and an
|
|
10
10
|
// unresolved misfire is a judge failure. Counting either as a side of a flip would
|
|
11
11
|
// report infrastructure noise as behavioural instability.
|
|
12
|
-
return !v.suspect && v.verdict !== "ERROR" && v.verdict !== "JUDGE-AMBIGUOUS";
|
|
12
|
+
return !v.suspect && v.verdict !== "ERROR" && v.verdict !== "NOT-MEASURED" && v.verdict !== "JUDGE-AMBIGUOUS";
|
|
13
13
|
}
|
|
14
14
|
function pointFor(r, s, verdict) {
|
|
15
15
|
const reps = s.reps ?? 1;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { type ArgPredicate, type CountConstraint } from "./trace-gates.js";
|
|
2
2
|
/** Adapter-neutral event contract used by objective trajectory assertions. */
|
|
3
|
-
export declare const
|
|
3
|
+
export declare const LEGACY_TRAJECTORY_EVENT_VERSION: "1.0";
|
|
4
|
+
export declare const TRAJECTORY_EVENT_VERSION: "1.1";
|
|
5
|
+
export type TrajectoryEventVersion = typeof LEGACY_TRAJECTORY_EVENT_VERSION | typeof TRAJECTORY_EVENT_VERSION;
|
|
4
6
|
export declare const TRAJECTORY_ASSERT_VERSION: "1.0";
|
|
5
7
|
export interface TrajectoryApproval {
|
|
6
8
|
id?: string;
|
|
@@ -26,7 +28,7 @@ export interface TrajectoryDigests {
|
|
|
26
28
|
* needs to know whether the source was pi, pi-daddy, or principal assurance.
|
|
27
29
|
*/
|
|
28
30
|
export interface TrajectoryEventV1 {
|
|
29
|
-
event_version:
|
|
31
|
+
event_version: TrajectoryEventVersion;
|
|
30
32
|
seq: number;
|
|
31
33
|
type: string;
|
|
32
34
|
source: string;
|
|
@@ -38,6 +40,16 @@ export interface TrajectoryEventV1 {
|
|
|
38
40
|
finding_id?: string;
|
|
39
41
|
parent_id?: string;
|
|
40
42
|
child_id?: string;
|
|
43
|
+
/** Unique pi-daddy v3 execution occurrence; never synthesized for v2/0.17. */
|
|
44
|
+
execution_id?: string;
|
|
45
|
+
/** Explicit v3 execution parent. Null is a meaningful root identity. */
|
|
46
|
+
parent_execution_id?: string | null;
|
|
47
|
+
/** Unique v3 occurrence whose output composed this task. */
|
|
48
|
+
task_from_execution_id?: string;
|
|
49
|
+
/** Identity of a pi-daddy v3 workflow_fact occurrence. */
|
|
50
|
+
workflow_fact_id?: string;
|
|
51
|
+
/** Immutable v3 nonterminal lifecycle deadline. */
|
|
52
|
+
deadline_at?: string;
|
|
41
53
|
phase?: string;
|
|
42
54
|
tool?: string;
|
|
43
55
|
capability?: string;
|
|
@@ -123,15 +135,3 @@ export declare function deserializeTrajectoryEvents(text: string): TrajectoryEve
|
|
|
123
135
|
/** Evaluate a closed, declarative assertion set over saved normalized events. */
|
|
124
136
|
export declare function evaluateTrajectoryGates(assert: TrajectoryAssert, input: TrajectoryEventV1[]): TrajectoryGateResult;
|
|
125
137
|
export declare function parseTrajectoryAssert(raw: unknown, ctx: string): TrajectoryAssert;
|
|
126
|
-
export interface MutationSelfTestCase {
|
|
127
|
-
id: string;
|
|
128
|
-
detected: boolean;
|
|
129
|
-
status: TrajectoryAssertionStatus;
|
|
130
|
-
detail: string;
|
|
131
|
-
}
|
|
132
|
-
export interface MutationSelfTestReport {
|
|
133
|
-
baseline: "PASS";
|
|
134
|
-
cases: MutationSelfTestCase[];
|
|
135
|
-
}
|
|
136
|
-
/** Fast, deterministic proof that every trajectory assertion class can turn red. */
|
|
137
|
-
export declare function runTrajectoryMutationSelfTest(): MutationSelfTestReport;
|
package/dist/trajectory-gates.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { parsePredicate, testPredicate } from "./trace-gates.js";
|
|
3
3
|
/** Adapter-neutral event contract used by objective trajectory assertions. */
|
|
4
|
-
export const
|
|
4
|
+
export const LEGACY_TRAJECTORY_EVENT_VERSION = "1.0";
|
|
5
|
+
export const TRAJECTORY_EVENT_VERSION = "1.1";
|
|
5
6
|
export const TRAJECTORY_ASSERT_VERSION = "1.0";
|
|
6
7
|
export function trajectoryEventsSha256(events) {
|
|
7
8
|
return createHash("sha256").update(stableStringify(events)).digest("hex");
|
|
@@ -428,42 +429,14 @@ function parseCount(raw, ctx) {
|
|
|
428
429
|
throw new Error(`${ctx}.min exceeds max`);
|
|
429
430
|
return out;
|
|
430
431
|
}
|
|
431
|
-
/** Fast, deterministic proof that every trajectory assertion class can turn red. */
|
|
432
|
-
export function runTrajectoryMutationSelfTest() {
|
|
433
|
-
const e = (seq, type, extra = {}) => ({ event_version: TRAJECTORY_EVENT_VERSION, seq, type, source: "mutation-self-test", at: `2026-01-01T00:00:${String(seq).padStart(2, "0")}Z`, ...extra });
|
|
434
|
-
const h = "a".repeat(40), t = "b".repeat(40), p = "1".repeat(64);
|
|
435
|
-
const cases = [
|
|
436
|
-
{ id: "remove-required-event", assertion: { version: "1.0", require: [{ event: "risk_classified" }] }, good: [e(1, "risk_classified")], mutate: () => [e(1, "unrelated")] },
|
|
437
|
-
{ id: "add-forbidden-tool-side-effect", assertion: { version: "1.0", forbid: [{ event: "tool_called", where: { tool: { equals: "rm" } } }, { event: "side_effect_performed" }] }, good: [e(1, "tool_called", { tool: "read" })], mutate: (events) => [...events, e(2, "side_effect_performed", { attributes: { destructive: true } })] },
|
|
438
|
-
{ id: "reorder-transition", assertion: { version: "1.0", ordered: [[{ event: "phase_started" }, { event: "phase_completed" }]] }, good: [e(1, "phase_started"), e(2, "phase_completed")], mutate: () => [e(2, "phase_started"), e(1, "phase_completed")] },
|
|
439
|
-
{ id: "substitute-workspace-id", assertion: { version: "1.0", correlate: [{ left: { event: "code_changed" }, right: { event: "evidence_recorded" }, same: ["workspace_id"] }] }, good: [e(1, "code_changed", { workspace_id: "ws-1" }), e(2, "evidence_recorded", { workspace_id: "ws-1" })], mutate: (events) => events.map((x) => x.type === "evidence_recorded" ? { ...x, workspace_id: "ws-2" } : x) },
|
|
440
|
-
{ id: "concurrent-writer", assertion: { version: "1.0", forbid: [{ event: "writer_lease_conflict" }] }, good: [e(1, "writer_lease_acquired", { workspace_id: "ws-1" })], mutate: (events) => [...events, e(2, "writer_lease_conflict", { workspace_id: "ws-1" })] },
|
|
441
|
-
{ id: "approval-expired-or-mismatched", assertion: { version: "1.0", approvals: [{ grant: { event: "approval_granted" }, use: { event: "approval_used" }, same: ["approval.id", "approval.capability"], unexpired: true }] }, good: [e(1, "approval_granted", { approval: { id: "a1", capability: "push", expires_at: "2026-01-01T01:00:00Z" } }), e(2, "approval_used", { approval: { id: "a1", capability: "push", used_at: "2026-01-01T00:01:00Z" } })], mutate: (events) => events.map((x) => x.type === "approval_used" ? { ...x, approval: { ...x.approval, used_at: "2026-01-01T02:00:00Z" } } : x) },
|
|
442
|
-
{ id: "evidence-before-change", assertion: { version: "1.0", freshness: [{ subject: { event: "evidence_recorded" }, after: [{ event: "code_changed" }] }] }, good: [e(1, "code_changed"), e(2, "evidence_recorded")], mutate: () => [e(2, "code_changed"), e(1, "evidence_recorded")] },
|
|
443
|
-
{ id: "evidence-before-authority", assertion: { version: "1.0", freshness: [{ subject: { event: "evidence_recorded" }, after: [{ event: "plan_recorded" }] }] }, good: [e(1, "plan_recorded", { digests: { plan: p } }), e(2, "evidence_recorded")], mutate: () => [e(2, "plan_recorded", { digests: { plan: p } }), e(1, "evidence_recorded")] },
|
|
444
|
-
{ id: "evidence-before-build-completion", assertion: { version: "1.0", freshness: [{ subject: { event: "evidence_recorded" }, after: [{ event: "phase_completed", where: { phase: { equals: "build" } } }] }] }, good: [e(1, "phase_completed", { phase: "build" }), e(2, "evidence_recorded")], mutate: () => [e(2, "phase_completed", { phase: "build" }), e(1, "evidence_recorded")] },
|
|
445
|
-
{ id: "head-equal-tree-different", assertion: { version: "1.0", correlate: [{ left: { event: "code_changed" }, right: { event: "evidence_recorded" }, same: ["digests.head", "digests.tree"] }] }, good: [e(1, "code_changed", { digests: { head: h, tree: t } }), e(2, "evidence_recorded", { digests: { head: h, tree: t } })], mutate: (events) => events.map((x) => x.type === "evidence_recorded" ? { ...x, digests: { head: h, tree: "c".repeat(40) } } : x) },
|
|
446
|
-
{ id: "command-receipt-nonzero", assertion: { version: "1.0", require: [{ event: "evidence_recorded", where: { exit_code: { equals: 0 } } }] }, good: [e(1, "evidence_recorded", { exit_code: 0 })], mutate: (events) => events.map((x) => ({ ...x, exit_code: 1 })) },
|
|
447
|
-
{ id: "remove-requirement-coverage", assertion: { version: "1.0", coverage: [{ requirements: ["AUTH-7"], events: { event: "evidence_recorded" } }] }, good: [e(1, "evidence_recorded", { requirements: ["AUTH-7"] })], mutate: (events) => events.map((x) => ({ ...x, requirements: [] })) },
|
|
448
|
-
{ id: "mutate-superseded-task", assertion: { version: "1.0", forbid_after: [{ anchor: { event: "task_packet_superseded" }, forbidden: [{ event: "code_changed" }, { event: "repair_started" }], same: ["task_id"] }] }, good: [e(1, "task_packet_superseded", { task_id: "old" }), e(2, "code_changed", { task_id: "new" })], mutate: (events) => events.map((x) => x.type === "code_changed" ? { ...x, task_id: "old" } : x) },
|
|
449
|
-
{ id: "reuse-context-id", assertion: { version: "1.0", unique: [{ events: { event: "review_recorded" }, fields: ["context_id"] }] }, good: [e(1, "review_recorded", { context_id: "c1" }), e(2, "review_recorded", { context_id: "c2" })], mutate: (events) => events.map((x) => x.seq === 2 ? { ...x, context_id: "c1" } : x) },
|
|
450
|
-
{ id: "mismatch-finalization-identity", assertion: { version: "1.0", correlate: [{ left: { event: "gate_evaluated", where: { "attributes.gate": { equals: "finalize" } } }, right: { event: "finalization_completed" }, same: ["digests.head", "digests.tree"], order: "before" }] }, good: [e(1, "gate_evaluated", { digests: { head: h, tree: t }, attributes: { gate: "finalize", result: "pass" } }), e(2, "finalization_completed", { digests: { head: h, tree: t } })], mutate: (events) => events.map((x) => x.type === "finalization_completed" ? { ...x, digests: { head: h, tree: "d".repeat(40) } } : x) },
|
|
451
|
-
];
|
|
452
|
-
const results = cases.map((testCase) => {
|
|
453
|
-
const baseline = evaluateTrajectoryGates(testCase.assertion, testCase.good);
|
|
454
|
-
if (baseline.status !== "PASS")
|
|
455
|
-
throw new Error(`mutation self-test baseline ${testCase.id} is ${baseline.status}`);
|
|
456
|
-
const mutated = evaluateTrajectoryGates(testCase.assertion, testCase.mutate(structuredClone(testCase.good)));
|
|
457
|
-
return { id: testCase.id, detected: mutated.status !== "PASS", status: mutated.status, detail: mutated.assertions.find((result) => result.status !== "PASS")?.detail ?? "mutation was not detected" };
|
|
458
|
-
});
|
|
459
|
-
return { baseline: "PASS", cases: results };
|
|
460
|
-
}
|
|
461
432
|
// ---------------------------------------------------------------------------
|
|
462
433
|
// Internals
|
|
463
434
|
// ---------------------------------------------------------------------------
|
|
435
|
+
const V11_EVENT_KEYS = new Set(["execution_id", "parent_execution_id", "task_from_execution_id", "workflow_fact_id", "deadline_at"]);
|
|
464
436
|
const EVENT_KEYS = new Set([
|
|
465
437
|
"event_version", "seq", "type", "source", "at", "run_id", "task_id", "workspace_id", "context_id", "finding_id",
|
|
466
|
-
"parent_id", "child_id",
|
|
438
|
+
"parent_id", "child_id", ...V11_EVENT_KEYS,
|
|
439
|
+
"phase", "tool", "capability", "requested_capabilities", "effective_capabilities", "refusal_code",
|
|
467
440
|
"exit_code", "digests", "approval", "requirements", "attributes",
|
|
468
441
|
]);
|
|
469
442
|
const APPROVAL_KEYS = new Set(["id", "capability", "subject", "source", "scope", "approved_at", "expires_at", "used_at"]);
|
|
@@ -479,8 +452,14 @@ function validateEvent(event) {
|
|
|
479
452
|
const unknown = Object.keys(object).find((key) => !EVENT_KEYS.has(key));
|
|
480
453
|
if (unknown)
|
|
481
454
|
return `unknown field ${unknown}`;
|
|
482
|
-
if (event.event_version !== TRAJECTORY_EVENT_VERSION)
|
|
455
|
+
if (event.event_version !== LEGACY_TRAJECTORY_EVENT_VERSION && event.event_version !== TRAJECTORY_EVENT_VERSION) {
|
|
483
456
|
return `unsupported event_version ${String(event.event_version)}`;
|
|
457
|
+
}
|
|
458
|
+
if (event.event_version === LEGACY_TRAJECTORY_EVENT_VERSION) {
|
|
459
|
+
const versionedField = Object.keys(object).find((key) => V11_EVENT_KEYS.has(key));
|
|
460
|
+
if (versionedField)
|
|
461
|
+
return `${versionedField} requires event_version ${TRAJECTORY_EVENT_VERSION}`;
|
|
462
|
+
}
|
|
484
463
|
if (!Number.isInteger(event.seq) || event.seq < 1)
|
|
485
464
|
return "seq must be a positive integer";
|
|
486
465
|
if (typeof event.type !== "string" || !event.type)
|
|
@@ -489,10 +468,15 @@ function validateEvent(event) {
|
|
|
489
468
|
return "source must be a non-empty string";
|
|
490
469
|
if (event.at !== undefined && !validDate(event.at))
|
|
491
470
|
return "at must be an RFC 3339 date-time";
|
|
492
|
-
for (const field of ["run_id", "task_id", "workspace_id", "context_id", "finding_id", "parent_id", "child_id"]) {
|
|
471
|
+
for (const field of ["run_id", "task_id", "workspace_id", "context_id", "finding_id", "parent_id", "child_id", "execution_id", "task_from_execution_id", "workflow_fact_id"]) {
|
|
493
472
|
if (event[field] !== undefined && (typeof event[field] !== "string" || !ID_RE.test(event[field])))
|
|
494
473
|
return `${field} is not a valid bounded identifier`;
|
|
495
474
|
}
|
|
475
|
+
if (event.parent_execution_id !== undefined && event.parent_execution_id !== null &&
|
|
476
|
+
(typeof event.parent_execution_id !== "string" || !ID_RE.test(event.parent_execution_id)))
|
|
477
|
+
return "parent_execution_id is not a valid bounded identifier or null";
|
|
478
|
+
if (event.deadline_at !== undefined && !validDate(event.deadline_at))
|
|
479
|
+
return "deadline_at must be an RFC 3339 date-time";
|
|
496
480
|
for (const field of ["phase", "tool", "capability"]) {
|
|
497
481
|
if (event[field] !== undefined && (typeof event[field] !== "string" || !event[field]))
|
|
498
482
|
return `${field} must be a non-empty string`;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { ChildProcess } from 'node:child_process';
|
|
2
|
+
import { type QualificationProcessIdentity } from './qualification-process.js';
|
|
3
|
+
export interface TrustedHostSupervisionLimits {
|
|
4
|
+
wallMs: number;
|
|
5
|
+
settlementMs: number;
|
|
6
|
+
maxOutputBytes: number;
|
|
7
|
+
signal: AbortSignal;
|
|
8
|
+
}
|
|
9
|
+
export interface TrustedHostProcessObservation {
|
|
10
|
+
outcome: 'completed' | 'failed' | 'aborted' | 'timed-out' | 'output-limit' | 'unknown';
|
|
11
|
+
identity: QualificationProcessIdentity;
|
|
12
|
+
exitCode: number | null;
|
|
13
|
+
signal: NodeJS.Signals | null;
|
|
14
|
+
stdoutBytes: number;
|
|
15
|
+
stderrBytes: number;
|
|
16
|
+
truncated: boolean;
|
|
17
|
+
cleanupAttempted: boolean;
|
|
18
|
+
acceptance: 'not-assessed';
|
|
19
|
+
}
|
|
20
|
+
/** Observe an ALREADY admitted, caller-owned detached Linux child, after its spawn event.
|
|
21
|
+
* No spawn/authorization/accounting/retry is performed here. The parent event loop must
|
|
22
|
+
* remain responsive. Occurrence-safe same-group cleanup is not hostile-code containment,
|
|
23
|
+
* survivor/crash recovery, hard aggregate RAM/CPU, or proof of remote provider termination.
|
|
24
|
+
* Bytes are discarded, not retained or hashed; the existing worker journal owns artifacts. */
|
|
25
|
+
export declare function superviseTrustedHost(child: ChildProcess, limits: TrustedHostSupervisionLimits): Promise<TrustedHostProcessObservation>;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { qualificationProcessIdentity, cleanupQualificationProcessGroupAfterLeaderExit, terminateQualificationProcess } from './qualification-process.js';
|
|
3
|
+
const monitored = new WeakSet();
|
|
4
|
+
/** Observe an ALREADY admitted, caller-owned detached Linux child, after its spawn event.
|
|
5
|
+
* No spawn/authorization/accounting/retry is performed here. The parent event loop must
|
|
6
|
+
* remain responsive. Occurrence-safe same-group cleanup is not hostile-code containment,
|
|
7
|
+
* survivor/crash recovery, hard aggregate RAM/CPU, or proof of remote provider termination.
|
|
8
|
+
* Bytes are discarded, not retained or hashed; the existing worker journal owns artifacts. */
|
|
9
|
+
export async function superviseTrustedHost(child, limits) {
|
|
10
|
+
if (process.platform !== 'linux')
|
|
11
|
+
throw Error('trusted host supervision requires Linux occurrence identity');
|
|
12
|
+
if (!(limits.signal instanceof AbortSignal) || !Number.isSafeInteger(limits.wallMs) || limits.wallMs < 1 || limits.wallMs > 150000 || !Number.isSafeInteger(limits.settlementMs) || limits.settlementMs < 100 || limits.settlementMs > 5000 || !Number.isSafeInteger(limits.maxOutputBytes) || limits.maxOutputBytes < 1 || limits.maxOutputBytes > 1048576)
|
|
13
|
+
throw Error('bounded trusted host limits required');
|
|
14
|
+
if (monitored.has(child))
|
|
15
|
+
throw Error('original child already monitored; no retry');
|
|
16
|
+
if (!child.pid || !child.stdout || !child.stderr || child.exitCode !== null || child.signalCode !== null)
|
|
17
|
+
throw Error('live caller-owned child with output pipes required');
|
|
18
|
+
const identity = qualificationProcessIdentity(child.pid), stat = readFileSync(`/proc/${child.pid}/stat`, 'utf8'), fields = stat.slice(stat.lastIndexOf(')') + 2).trim().split(/\s+/);
|
|
19
|
+
if (Number(fields[2]) !== child.pid || fields[19] !== identity.start_ticks)
|
|
20
|
+
throw Error('original detached process group required');
|
|
21
|
+
monitored.add(child);
|
|
22
|
+
const stdout = child.stdout, stderr = child.stderr;
|
|
23
|
+
return new Promise(resolve => {
|
|
24
|
+
let reason = null, done = false, exited = false, closed = false, exitCode = null, signal = null, stdoutBytes = 0, stderrBytes = 0, cleanupAttempted = false;
|
|
25
|
+
let settling, cleanup = null;
|
|
26
|
+
const finish = (outcome) => {
|
|
27
|
+
if (done)
|
|
28
|
+
return;
|
|
29
|
+
done = true;
|
|
30
|
+
clearTimeout(wall);
|
|
31
|
+
clearTimeout(settling);
|
|
32
|
+
limits.signal.removeEventListener('abort', abort);
|
|
33
|
+
stdout.off('data', out);
|
|
34
|
+
stderr.off('data', err);
|
|
35
|
+
stdout.resume();
|
|
36
|
+
stderr.resume();
|
|
37
|
+
child.off('exit', exit);
|
|
38
|
+
child.off('close', close);
|
|
39
|
+
child.off('error', error);
|
|
40
|
+
// On unknown, the caller still owns this child and occurrence. Never claim settlement.
|
|
41
|
+
resolve({ outcome, identity, exitCode, signal, stdoutBytes, stderrBytes, truncated: stdoutBytes + stderrBytes > limits.maxOutputBytes, cleanupAttempted, acceptance: 'not-assessed' });
|
|
42
|
+
};
|
|
43
|
+
const boundSettlement = () => { settling ??= setTimeout(() => finish('unknown'), limits.settlementMs); };
|
|
44
|
+
const stop = (why) => {
|
|
45
|
+
if (done)
|
|
46
|
+
return;
|
|
47
|
+
reason ??= why;
|
|
48
|
+
boundSettlement();
|
|
49
|
+
if (cleanupAttempted)
|
|
50
|
+
return;
|
|
51
|
+
cleanupAttempted = true;
|
|
52
|
+
try {
|
|
53
|
+
terminateQualificationProcess(child, identity);
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
finish('unknown');
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
const out = (b) => { stdoutBytes += b.length; if (stdoutBytes + stderrBytes > limits.maxOutputBytes)
|
|
60
|
+
stop('output-limit'); };
|
|
61
|
+
const err = (b) => { stderrBytes += b.length; if (stdoutBytes + stderrBytes > limits.maxOutputBytes)
|
|
62
|
+
stop('output-limit'); };
|
|
63
|
+
const abort = () => stop('aborted');
|
|
64
|
+
const error = () => stop('failed');
|
|
65
|
+
const complete = () => {
|
|
66
|
+
if (!exited || !closed || done)
|
|
67
|
+
return;
|
|
68
|
+
void (cleanup ?? Promise.resolve()).then(() => finish(reason ?? (exitCode === 0 ? 'completed' : 'failed')), () => finish('unknown'));
|
|
69
|
+
};
|
|
70
|
+
const exit = (code, sig) => {
|
|
71
|
+
exited = true;
|
|
72
|
+
exitCode = code;
|
|
73
|
+
signal = sig;
|
|
74
|
+
clearTimeout(wall);
|
|
75
|
+
boundSettlement();
|
|
76
|
+
cleanupAttempted = true;
|
|
77
|
+
cleanup = cleanupQualificationProcessGroupAfterLeaderExit(child.pid, identity);
|
|
78
|
+
// Attach rejection handling immediately, even if descendant-held pipes never close.
|
|
79
|
+
void cleanup.catch(() => finish('unknown'));
|
|
80
|
+
complete();
|
|
81
|
+
};
|
|
82
|
+
const close = () => { closed = true; complete(); };
|
|
83
|
+
const wall = setTimeout(() => stop('timed-out'), limits.wallMs);
|
|
84
|
+
stdout.on('data', out);
|
|
85
|
+
stderr.on('data', err);
|
|
86
|
+
child.once('error', error);
|
|
87
|
+
child.once('exit', exit);
|
|
88
|
+
child.once('close', close);
|
|
89
|
+
limits.signal.addEventListener('abort', abort, { once: true });
|
|
90
|
+
if (limits.signal.aborted)
|
|
91
|
+
abort();
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=trusted-host-supervision.js.map
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Verdict } from "./score.js";
|
|
2
|
+
/** Domain-neutral vote shape shared by ordinary adjudication and qualification panels. */
|
|
3
|
+
export interface PanelVote {
|
|
4
|
+
ordinal: number;
|
|
5
|
+
verdict: Verdict;
|
|
6
|
+
suspect: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface VotePanelCollapse {
|
|
9
|
+
state: "confirmed" | "tie_broken" | "unresolved";
|
|
10
|
+
verdict?: "PASS" | "FAIL";
|
|
11
|
+
clean_votes: number;
|
|
12
|
+
pass_votes: number;
|
|
13
|
+
fail_votes: number;
|
|
14
|
+
/** True only when the first two answers are both clean and oppose one another. */
|
|
15
|
+
split: boolean;
|
|
16
|
+
/** Minority clean votes / all clean votes; zero when there is no minority. */
|
|
17
|
+
minority_rate: number;
|
|
18
|
+
}
|
|
19
|
+
export declare function isCleanPanelVote(vote: PanelVote): vote is PanelVote & {
|
|
20
|
+
verdict: "PASS" | "FAIL";
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* One collapse rule for both skill-harness adjudication and qualification.
|
|
24
|
+
* Invalid, ambiguous, and suspect answers remain evidence but are never votes.
|
|
25
|
+
*/
|
|
26
|
+
export declare function collapseVotePanel(votes: readonly PanelVote[]): VotePanelCollapse;
|
|
27
|
+
export declare function runVotePanel<T extends PanelVote>(options: {
|
|
28
|
+
cast: (ordinal: 1 | 2 | 3) => Promise<T>;
|
|
29
|
+
}): Promise<{
|
|
30
|
+
votes: T[];
|
|
31
|
+
collapse: VotePanelCollapse;
|
|
32
|
+
calls_made: number;
|
|
33
|
+
}>;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export function isCleanPanelVote(vote) {
|
|
2
|
+
return !vote.suspect && (vote.verdict === "PASS" || vote.verdict === "FAIL");
|
|
3
|
+
}
|
|
4
|
+
/**
|
|
5
|
+
* One collapse rule for both skill-harness adjudication and qualification.
|
|
6
|
+
* Invalid, ambiguous, and suspect answers remain evidence but are never votes.
|
|
7
|
+
*/
|
|
8
|
+
export function collapseVotePanel(votes) {
|
|
9
|
+
const clean = votes.filter(isCleanPanelVote);
|
|
10
|
+
const passVotes = clean.filter((vote) => vote.verdict === "PASS").length;
|
|
11
|
+
const failVotes = clean.length - passVotes;
|
|
12
|
+
const firstTwo = votes.filter((vote) => vote.ordinal === 1 || vote.ordinal === 2);
|
|
13
|
+
const split = firstTwo.length === 2 && firstTwo.every(isCleanPanelVote) && firstTwo[0].verdict !== firstTwo[1].verdict;
|
|
14
|
+
const minority = Math.min(passVotes, failVotes);
|
|
15
|
+
const base = {
|
|
16
|
+
clean_votes: clean.length,
|
|
17
|
+
pass_votes: passVotes,
|
|
18
|
+
fail_votes: failVotes,
|
|
19
|
+
split,
|
|
20
|
+
minority_rate: clean.length === 0 ? 0 : minority / clean.length,
|
|
21
|
+
};
|
|
22
|
+
if (clean.length < 2)
|
|
23
|
+
return { ...base, state: "unresolved" };
|
|
24
|
+
if (passVotes === 0 || failVotes === 0)
|
|
25
|
+
return { ...base, state: "confirmed", verdict: clean[0].verdict };
|
|
26
|
+
if (passVotes > failVotes)
|
|
27
|
+
return { ...base, state: "tie_broken", verdict: "PASS" };
|
|
28
|
+
if (failVotes > passVotes)
|
|
29
|
+
return { ...base, state: "tie_broken", verdict: "FAIL" };
|
|
30
|
+
return { ...base, state: "unresolved" };
|
|
31
|
+
}
|
|
32
|
+
export async function runVotePanel(options) {
|
|
33
|
+
const votes = [await options.cast(1), await options.cast(2)];
|
|
34
|
+
let collapse = collapseVotePanel(votes);
|
|
35
|
+
if (collapse.split) {
|
|
36
|
+
votes.push(await options.cast(3));
|
|
37
|
+
collapse = collapseVotePanel(votes);
|
|
38
|
+
}
|
|
39
|
+
return { votes, collapse, calls_made: votes.length };
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=vote-panel.js.map
|