@markstanden/opencode-test-container 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +263 -2
- package/container/Containerfile +24 -0
- package/container/README.md +29 -0
- package/dist/adapters/opencode/events.d.ts +90 -0
- package/dist/adapters/opencode/events.js +213 -0
- package/dist/adapters/opencode/runtime.d.ts +87 -0
- package/dist/adapters/opencode/runtime.js +136 -0
- package/dist/adapters/podman/container.d.ts +62 -0
- package/dist/adapters/podman/container.js +119 -0
- package/dist/adapters/podman/image.d.ts +50 -0
- package/dist/adapters/podman/image.js +124 -0
- package/dist/adapters/process.d.ts +38 -0
- package/dist/adapters/process.js +71 -0
- package/dist/campaign/assess.d.ts +45 -0
- package/dist/campaign/assess.js +208 -0
- package/dist/campaign/close.d.ts +17 -0
- package/dist/campaign/close.js +112 -0
- package/dist/campaign/evidence-guard.d.ts +24 -0
- package/dist/campaign/evidence-guard.js +49 -0
- package/dist/campaign/freeze.d.ts +94 -0
- package/dist/campaign/freeze.js +297 -0
- package/dist/campaign/inventory.d.ts +62 -0
- package/dist/campaign/inventory.js +192 -0
- package/dist/campaign/judge.d.ts +209 -0
- package/dist/campaign/judge.js +1285 -0
- package/dist/campaign/ledger.d.ts +64 -0
- package/dist/campaign/ledger.js +149 -0
- package/dist/campaign/prepare.d.ts +129 -0
- package/dist/campaign/prepare.js +483 -0
- package/dist/campaign/primary.d.ts +96 -0
- package/dist/campaign/primary.js +189 -0
- package/dist/campaign/run.d.ts +151 -0
- package/dist/campaign/run.js +562 -0
- package/dist/campaign/status.d.ts +76 -0
- package/dist/campaign/status.js +237 -0
- package/dist/cli/otc.d.ts +51 -0
- package/dist/cli/otc.js +435 -0
- package/dist/contracts/canonical.d.ts +21 -0
- package/dist/contracts/canonical.js +44 -0
- package/dist/contracts/types.d.ts +261 -0
- package/dist/contracts/types.js +9 -0
- package/dist/contracts/validate.d.ts +72 -0
- package/dist/contracts/validate.js +1021 -0
- package/dist/contracts/version.d.ts +44 -0
- package/dist/contracts/version.js +147 -0
- package/dist/evaluators/docs/analysis/accounting.d.ts +140 -0
- package/dist/evaluators/docs/analysis/accounting.js +237 -0
- package/dist/evaluators/docs/analysis/blinding.d.ts +40 -0
- package/dist/evaluators/docs/analysis/blinding.js +80 -0
- package/dist/evaluators/docs/analysis/bounds.d.ts +40 -0
- package/dist/evaluators/docs/analysis/bounds.js +31 -0
- package/dist/evaluators/docs/analysis/budget.d.ts +137 -0
- package/dist/evaluators/docs/analysis/budget.js +305 -0
- package/dist/evaluators/docs/analysis/closure.d.ts +175 -0
- package/dist/evaluators/docs/analysis/closure.js +251 -0
- package/dist/evaluators/docs/analysis/decision.d.ts +59 -0
- package/dist/evaluators/docs/analysis/decision.js +161 -0
- package/dist/evaluators/docs/analysis/freeze.d.ts +201 -0
- package/dist/evaluators/docs/analysis/freeze.js +395 -0
- package/dist/evaluators/docs/analysis/guardrails.d.ts +51 -0
- package/dist/evaluators/docs/analysis/guardrails.js +194 -0
- package/dist/evaluators/docs/analysis/journal.d.ts +150 -0
- package/dist/evaluators/docs/analysis/journal.js +475 -0
- package/dist/evaluators/docs/analysis/ledger.d.ts +78 -0
- package/dist/evaluators/docs/analysis/ledger.js +145 -0
- package/dist/evaluators/docs/analysis/measures.d.ts +302 -0
- package/dist/evaluators/docs/analysis/measures.js +1069 -0
- package/dist/evaluators/docs/analysis/model.d.ts +236 -0
- package/dist/evaluators/docs/analysis/model.js +740 -0
- package/dist/evaluators/docs/analysis/pairing.d.ts +68 -0
- package/dist/evaluators/docs/analysis/pairing.js +95 -0
- package/dist/evaluators/docs/analysis/records.d.ts +229 -0
- package/dist/evaluators/docs/analysis/records.js +935 -0
- package/dist/evaluators/docs/analysis/recovery.d.ts +225 -0
- package/dist/evaluators/docs/analysis/recovery.js +507 -0
- package/dist/evaluators/docs/analysis/render.d.ts +235 -0
- package/dist/evaluators/docs/analysis/render.js +566 -0
- package/dist/evaluators/docs/analysis/replay.d.ts +15 -0
- package/dist/evaluators/docs/analysis/replay.js +1272 -0
- package/dist/evaluators/docs/analysis/screening/inventory.d.ts +136 -0
- package/dist/evaluators/docs/analysis/screening/inventory.js +307 -0
- package/dist/evaluators/docs/analysis/screening/noteVersion.d.ts +15 -0
- package/dist/evaluators/docs/analysis/screening/noteVersion.js +46 -0
- package/dist/evaluators/docs/analysis/screening/reconcile.d.ts +38 -0
- package/dist/evaluators/docs/analysis/screening/reconcile.js +40 -0
- package/dist/evaluators/docs/analysis/screening/rules.d.ts +163 -0
- package/dist/evaluators/docs/analysis/screening/rules.js +330 -0
- package/dist/evaluators/docs/analysis/screening/scan.d.ts +72 -0
- package/dist/evaluators/docs/analysis/screening/scan.js +362 -0
- package/dist/evaluators/docs/analysis/selector.d.ts +68 -0
- package/dist/evaluators/docs/analysis/selector.js +123 -0
- package/dist/evaluators/docs/assess.d.ts +129 -0
- package/dist/evaluators/docs/assess.js +433 -0
- package/dist/evaluators/docs/campaign.d.ts +133 -0
- package/dist/evaluators/docs/campaign.js +222 -0
- package/dist/evaluators/docs/index.d.ts +25 -0
- package/dist/evaluators/docs/index.js +25 -0
- package/dist/evaluators/docs/journal.d.ts +150 -0
- package/dist/evaluators/docs/journal.js +475 -0
- package/dist/evaluators/docs/scoring.d.ts +367 -0
- package/dist/evaluators/docs/scoring.js +662 -0
- package/dist/evaluators/skills/assess.d.ts +94 -0
- package/dist/evaluators/skills/assess.js +207 -0
- package/dist/index.d.ts +46 -0
- package/dist/index.js +29 -0
- package/dist/legacy/prod014.d.ts +86 -0
- package/dist/legacy/prod014.js +209 -0
- package/dist/reporting/report.d.ts +79 -0
- package/dist/reporting/report.js +242 -0
- package/docs/architecture.md +106 -0
- package/docs/capability-matrix.md +84 -0
- package/docs/offline-rehearsal.md +75 -0
- package/docs/parity-record.md +81 -0
- package/docs/pr-description.md +103 -0
- package/docs/schema.md +97 -0
- package/examples/skills/README.md +13 -0
- package/examples/skills/campaign.json +73 -0
- package/examples/skills/skills/otel-helper/SKILL.md +10 -0
- package/examples/skills/suite.json +29 -0
- package/package.json +45 -5
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** The frozen evaluator-semantics version. */
|
|
2
|
+
export declare const EVALUATOR_VERSION = 1;
|
|
3
|
+
/** The package name recorded in every manifest and freeze receipt. */
|
|
4
|
+
export declare const PACKAGE_NAME = "@markstanden/opencode-test-container";
|
|
5
|
+
/** The installed package root, two levels above this module. */
|
|
6
|
+
export declare function packageRoot(): string;
|
|
7
|
+
/**
|
|
8
|
+
* Read the installed package version, or `unknown` when the metadata is
|
|
9
|
+
* unavailable. This never reads the caller's working directory.
|
|
10
|
+
* @returns The version string.
|
|
11
|
+
*/
|
|
12
|
+
export declare function packageVersion(): string;
|
|
13
|
+
/**
|
|
14
|
+
* The content identity of the executing runtime tree.
|
|
15
|
+
* @param root The package root; defaults to the running installation.
|
|
16
|
+
* @param moduleFile The executing module file; defaults to this module.
|
|
17
|
+
* @returns A stable SHA-256 over sorted paths and file bytes.
|
|
18
|
+
*/
|
|
19
|
+
export declare function artifactSha256(root?: string, moduleFile?: string): string;
|
|
20
|
+
/** The tool identity every manifest and freeze receipt carries. */
|
|
21
|
+
export interface ToolIdentity {
|
|
22
|
+
/** The package name. */
|
|
23
|
+
name: string;
|
|
24
|
+
/** The installed package version, or `unknown`. */
|
|
25
|
+
version: string;
|
|
26
|
+
/** The checked evaluator-semantics version. */
|
|
27
|
+
evaluatorVersion: number;
|
|
28
|
+
/** The executing runtime content hash. */
|
|
29
|
+
artifactSha256: string;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Build the current tool identity. The default installation is cached; an
|
|
33
|
+
* explicit root is always recomputed.
|
|
34
|
+
* @param root The package root; defaults to the running installation.
|
|
35
|
+
* @returns The identity.
|
|
36
|
+
*/
|
|
37
|
+
export declare function toolIdentity(root?: string): ToolIdentity;
|
|
38
|
+
/**
|
|
39
|
+
* Compare the frozen identity with the installation that is executing now.
|
|
40
|
+
* @param frozen The identity recorded at freeze time.
|
|
41
|
+
* @param current The executing identity.
|
|
42
|
+
* @returns Differences; empty means the same artefact.
|
|
43
|
+
*/
|
|
44
|
+
export declare function toolIdentityProblems(frozen: ToolIdentity, current?: ToolIdentity): string[];
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The package and evaluator identity frozen with every campaign.
|
|
3
|
+
*
|
|
4
|
+
* The version comes from the installed package metadata and the artifact hash
|
|
5
|
+
* from the executing runtime tree, so a tarball can be traced to the artefact
|
|
6
|
+
* that produced its evidence without a Git checkout. The artifact hash covers
|
|
7
|
+
* executable JavaScript (or TypeScript when running from source) and the
|
|
8
|
+
* package manifest, so a changed build under the same version is detectable.
|
|
9
|
+
*/
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
|
12
|
+
import { dirname, isAbsolute, join, relative, sep } from "node:path";
|
|
13
|
+
import { fileURLToPath } from "node:url";
|
|
14
|
+
/** The frozen evaluator-semantics version. */
|
|
15
|
+
export const EVALUATOR_VERSION = 1;
|
|
16
|
+
/** The package name recorded in every manifest and freeze receipt. */
|
|
17
|
+
export const PACKAGE_NAME = "@markstanden/opencode-test-container";
|
|
18
|
+
/** The installed package root, two levels above this module. */
|
|
19
|
+
export function packageRoot() {
|
|
20
|
+
return join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Read the installed package version, or `unknown` when the metadata is
|
|
24
|
+
* unavailable. This never reads the caller's working directory.
|
|
25
|
+
* @returns The version string.
|
|
26
|
+
*/
|
|
27
|
+
export function packageVersion() {
|
|
28
|
+
const root = packageRoot();
|
|
29
|
+
for (const candidate of [
|
|
30
|
+
join(root, "package.json"),
|
|
31
|
+
join(root, "..", "package.json"),
|
|
32
|
+
]) {
|
|
33
|
+
try {
|
|
34
|
+
const parsed = JSON.parse(readFileSync(candidate, "utf8"));
|
|
35
|
+
if (typeof parsed === "object" &&
|
|
36
|
+
parsed !== null &&
|
|
37
|
+
"version" in parsed &&
|
|
38
|
+
typeof parsed.version === "string") {
|
|
39
|
+
return parsed.version;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
// try the next candidate
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return "unknown";
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The runtime tree that is actually executing, derived from the module's own
|
|
50
|
+
* location rather than the mere presence of a build.
|
|
51
|
+
* @param root The package root.
|
|
52
|
+
* @param moduleFile The executing module file.
|
|
53
|
+
* @returns The `src` or `dist` tree under `root`.
|
|
54
|
+
*/
|
|
55
|
+
function runtimeRoot(root, moduleFile) {
|
|
56
|
+
const rel = relative(root, moduleFile);
|
|
57
|
+
if (!rel.startsWith("..") && !isAbsolute(rel)) {
|
|
58
|
+
const top = rel.split(sep)[0];
|
|
59
|
+
if (top === "dist" || top === "src")
|
|
60
|
+
return join(root, top);
|
|
61
|
+
}
|
|
62
|
+
return existsSync(join(root, "src"))
|
|
63
|
+
? join(root, "src")
|
|
64
|
+
: join(root, "dist");
|
|
65
|
+
}
|
|
66
|
+
/** Every file under a root, sorted by relative path. */
|
|
67
|
+
function filesUnder(root) {
|
|
68
|
+
const files = [];
|
|
69
|
+
const walk = (directory) => {
|
|
70
|
+
for (const entry of readdirSync(directory, {
|
|
71
|
+
withFileTypes: true,
|
|
72
|
+
}).sort((left, right) => left.name.localeCompare(right.name))) {
|
|
73
|
+
const path = join(directory, entry.name);
|
|
74
|
+
if (entry.isDirectory())
|
|
75
|
+
walk(path);
|
|
76
|
+
else if (entry.isFile() && /\.(?:js|ts)$/.test(entry.name)) {
|
|
77
|
+
files.push(path);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
if (existsSync(root))
|
|
82
|
+
walk(root);
|
|
83
|
+
return files.sort();
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* The content identity of the executing runtime tree.
|
|
87
|
+
* @param root The package root; defaults to the running installation.
|
|
88
|
+
* @param moduleFile The executing module file; defaults to this module.
|
|
89
|
+
* @returns A stable SHA-256 over sorted paths and file bytes.
|
|
90
|
+
*/
|
|
91
|
+
export function artifactSha256(root = packageRoot(), moduleFile = fileURLToPath(import.meta.url)) {
|
|
92
|
+
const hash = createHash("sha256");
|
|
93
|
+
for (const path of filesUnder(runtimeRoot(root, moduleFile))) {
|
|
94
|
+
hash.update(relative(root, path).split(sep).join("/"));
|
|
95
|
+
hash.update("\0");
|
|
96
|
+
hash.update(readFileSync(path));
|
|
97
|
+
hash.update("\0");
|
|
98
|
+
}
|
|
99
|
+
const manifest = join(root, "package.json");
|
|
100
|
+
if (existsSync(manifest) && statSync(manifest).isFile()) {
|
|
101
|
+
hash.update("package.json\0");
|
|
102
|
+
hash.update(readFileSync(manifest));
|
|
103
|
+
}
|
|
104
|
+
return hash.digest("hex");
|
|
105
|
+
}
|
|
106
|
+
let cachedIdentity = null;
|
|
107
|
+
/**
|
|
108
|
+
* Build the current tool identity. The default installation is cached; an
|
|
109
|
+
* explicit root is always recomputed.
|
|
110
|
+
* @param root The package root; defaults to the running installation.
|
|
111
|
+
* @returns The identity.
|
|
112
|
+
*/
|
|
113
|
+
export function toolIdentity(root) {
|
|
114
|
+
if (root === undefined && cachedIdentity !== null)
|
|
115
|
+
return cachedIdentity;
|
|
116
|
+
const identity = {
|
|
117
|
+
name: PACKAGE_NAME,
|
|
118
|
+
version: packageVersion(),
|
|
119
|
+
evaluatorVersion: EVALUATOR_VERSION,
|
|
120
|
+
artifactSha256: artifactSha256(root),
|
|
121
|
+
};
|
|
122
|
+
if (root === undefined)
|
|
123
|
+
cachedIdentity = identity;
|
|
124
|
+
return identity;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Compare the frozen identity with the installation that is executing now.
|
|
128
|
+
* @param frozen The identity recorded at freeze time.
|
|
129
|
+
* @param current The executing identity.
|
|
130
|
+
* @returns Differences; empty means the same artefact.
|
|
131
|
+
*/
|
|
132
|
+
export function toolIdentityProblems(frozen, current = toolIdentity()) {
|
|
133
|
+
const problems = [];
|
|
134
|
+
if (frozen.name !== current.name) {
|
|
135
|
+
problems.push(`frozen tool name ${frozen.name} differs from executing ${current.name}`);
|
|
136
|
+
}
|
|
137
|
+
if (frozen.version !== current.version) {
|
|
138
|
+
problems.push(`frozen tool version ${frozen.version} differs from executing ${current.version}`);
|
|
139
|
+
}
|
|
140
|
+
if (frozen.evaluatorVersion !== current.evaluatorVersion) {
|
|
141
|
+
problems.push(`frozen evaluator version ${frozen.evaluatorVersion} differs from executing ${current.evaluatorVersion}`);
|
|
142
|
+
}
|
|
143
|
+
if (frozen.artifactSha256 !== current.artifactSha256) {
|
|
144
|
+
problems.push(`frozen tool artifact ${frozen.artifactSha256} differs from executing ${current.artifactSha256}`);
|
|
145
|
+
}
|
|
146
|
+
return problems;
|
|
147
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Trial accounting for the analysis framework.
|
|
3
|
+
*
|
|
4
|
+
* One scheduled slot becomes one {@link TrialScored}: the shared
|
|
5
|
+
* `ac-scoring` scored-trial shape plus the utility its loaded evidence
|
|
6
|
+
* supports. Counting keeps planned slots, recorded launches, unproven
|
|
7
|
+
* directories, completed and usable runs, assessment states, gate outcomes
|
|
8
|
+
* and semantic outcomes as separate measures — a blocked gate is never a
|
|
9
|
+
* semantic failure, and an unassessed trial is never a zero.
|
|
10
|
+
*/
|
|
11
|
+
import { type DocumentationOutcome, type ScoredTrial, type UtilityWeights } from "../scoring.ts";
|
|
12
|
+
import type { AssessmentStatus, AttemptState, TrialRecord } from "./records.ts";
|
|
13
|
+
import type { Disposition, SlotSpec } from "./model.ts";
|
|
14
|
+
/** How one observation relates to the frozen schedule. */
|
|
15
|
+
export type { Disposition };
|
|
16
|
+
/** The declared identity a scored trial carries. */
|
|
17
|
+
export interface ScoredIdentity {
|
|
18
|
+
/** The scheduled slot number, or null for an undeclared observation. */
|
|
19
|
+
slot: number | null;
|
|
20
|
+
/** The declared pair id, when the slot belongs to a pair. */
|
|
21
|
+
pairId: string | undefined;
|
|
22
|
+
/** The scenario the slot declared. */
|
|
23
|
+
scenario: string;
|
|
24
|
+
/** The arm the slot declared. */
|
|
25
|
+
arm: string;
|
|
26
|
+
/** The repetition within scenario and arm. */
|
|
27
|
+
repetition: number;
|
|
28
|
+
/** The trial stamp the evidence was loaded under. */
|
|
29
|
+
stamp: string;
|
|
30
|
+
}
|
|
31
|
+
/** One scored trial: identity, shared scoring shape and derived utility. */
|
|
32
|
+
export interface TrialScored {
|
|
33
|
+
/** The declared identity. */
|
|
34
|
+
identity: ScoredIdentity;
|
|
35
|
+
/** How the observation relates to the schedule. */
|
|
36
|
+
disposition: Disposition;
|
|
37
|
+
/** What the trial directory proved about the launch. */
|
|
38
|
+
attemptState: AttemptState;
|
|
39
|
+
/** The shared ac-scoring trial shape. */
|
|
40
|
+
trial: ScoredTrial;
|
|
41
|
+
/** The utility and its bounds. */
|
|
42
|
+
utility: {
|
|
43
|
+
score: number;
|
|
44
|
+
low: number;
|
|
45
|
+
high: number;
|
|
46
|
+
assessed: boolean;
|
|
47
|
+
};
|
|
48
|
+
/** The selected judgement layer's status. */
|
|
49
|
+
assessmentStatus: AssessmentStatus;
|
|
50
|
+
/** The recorded gate status, or `missing`. */
|
|
51
|
+
gateStatus: string;
|
|
52
|
+
}
|
|
53
|
+
/** Options for scoring one loaded trial against its declared slot. */
|
|
54
|
+
export interface ScoreOptions {
|
|
55
|
+
/** The frozen utility weights. */
|
|
56
|
+
weights?: UtilityWeights;
|
|
57
|
+
/** Whether a non-compensable guardrail breach is established. */
|
|
58
|
+
guardrailBreach?: boolean;
|
|
59
|
+
/** How the observation relates to the schedule. */
|
|
60
|
+
disposition?: Disposition;
|
|
61
|
+
}
|
|
62
|
+
/** One set of accounting counts. */
|
|
63
|
+
export interface AccountingCounts {
|
|
64
|
+
/** Scheduled slots in the declared population. */
|
|
65
|
+
planned: number;
|
|
66
|
+
/** Slots whose directory proves a recorded launch. */
|
|
67
|
+
recordedLaunches: number;
|
|
68
|
+
/** Directories or reservations that prove no launch either way. */
|
|
69
|
+
unprovenDirectories: number;
|
|
70
|
+
/** Planned slots with no directory at all. */
|
|
71
|
+
absent: number;
|
|
72
|
+
/** Recorded runs whose evidence is usable. */
|
|
73
|
+
completed: number;
|
|
74
|
+
/** Slots whose usability cannot be established. */
|
|
75
|
+
usableUnknown: number;
|
|
76
|
+
/** Recorded runs whose evidence is unusable. */
|
|
77
|
+
unusable: number;
|
|
78
|
+
/** Trials with a known documentation outcome (documentation coverage). */
|
|
79
|
+
documented: number;
|
|
80
|
+
/** Trials whose utility is fully known (scored population). */
|
|
81
|
+
assessed: number;
|
|
82
|
+
/** Observations excluded from the measured population. */
|
|
83
|
+
excluded: number;
|
|
84
|
+
/** Count per assessment status. */
|
|
85
|
+
assessmentStatuses: Record<AssessmentStatus, number>;
|
|
86
|
+
/** Primary-task results. */
|
|
87
|
+
primary: {
|
|
88
|
+
pass: number;
|
|
89
|
+
fail: number;
|
|
90
|
+
unknown: number;
|
|
91
|
+
};
|
|
92
|
+
/** Gate outcomes, kept separate from semantic outcomes. */
|
|
93
|
+
gate: {
|
|
94
|
+
pass: number;
|
|
95
|
+
blocked: number;
|
|
96
|
+
other: number;
|
|
97
|
+
missing: number;
|
|
98
|
+
};
|
|
99
|
+
/** Count per documentation outcome. */
|
|
100
|
+
outcomes: Record<DocumentationOutcome, number>;
|
|
101
|
+
/** Sum of assessed trial utilities. */
|
|
102
|
+
utilityTotal: number;
|
|
103
|
+
/** Full-population lower bound: assessed total plus missing lows. */
|
|
104
|
+
utilityLow: number;
|
|
105
|
+
/** Full-population upper bound: assessed total plus missing highs. */
|
|
106
|
+
utilityHigh: number;
|
|
107
|
+
/** Mean over planned slots, or null without a denominator. */
|
|
108
|
+
meanPerPlanned: number | null;
|
|
109
|
+
/** Mean over assessed trials, or null without a denominator. */
|
|
110
|
+
meanPerAssessed: number | null;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Score one loaded trial against its declared slot.
|
|
114
|
+
* @param record The loaded trial record.
|
|
115
|
+
* @param slot The manifest slot the record belongs to.
|
|
116
|
+
* @param options Weights, guardrail state and disposition overrides.
|
|
117
|
+
* @returns The scored trial with identity and utility bounds.
|
|
118
|
+
*/
|
|
119
|
+
export declare function scoreTrial(record: TrialRecord, slot: SlotSpec, options?: ScoreOptions): TrialScored;
|
|
120
|
+
/**
|
|
121
|
+
* Aggregate one measured population.
|
|
122
|
+
* @param scored The scored trials, one per scheduled slot.
|
|
123
|
+
* @param planned The number of scheduled slots in the population.
|
|
124
|
+
* @returns The counts, totals and bounds.
|
|
125
|
+
*/
|
|
126
|
+
export declare function accountTrials(scored: readonly TrialScored[], planned: number): AccountingCounts;
|
|
127
|
+
/**
|
|
128
|
+
* Aggregate one population per declared arm.
|
|
129
|
+
* @param scored The scored trials.
|
|
130
|
+
* @param slots The declared schedule slots, for planned denominators.
|
|
131
|
+
* @returns Counts keyed by arm id, sorted.
|
|
132
|
+
*/
|
|
133
|
+
export declare function accountByArm(scored: readonly TrialScored[], slots: readonly SlotSpec[]): Record<string, AccountingCounts>;
|
|
134
|
+
/**
|
|
135
|
+
* Aggregate one population per arm and scenario.
|
|
136
|
+
* @param scored The scored trials.
|
|
137
|
+
* @param slots The declared schedule slots, for planned denominators.
|
|
138
|
+
* @returns Counts keyed by arm then scenario, with only present combinations.
|
|
139
|
+
*/
|
|
140
|
+
export declare function accountByScenarioArm(scored: readonly TrialScored[], slots: readonly SlotSpec[]): Record<string, Record<string, AccountingCounts>>;
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Trial accounting for the analysis framework.
|
|
3
|
+
*
|
|
4
|
+
* One scheduled slot becomes one {@link TrialScored}: the shared
|
|
5
|
+
* `ac-scoring` scored-trial shape plus the utility its loaded evidence
|
|
6
|
+
* supports. Counting keeps planned slots, recorded launches, unproven
|
|
7
|
+
* directories, completed and usable runs, assessment states, gate outcomes
|
|
8
|
+
* and semantic outcomes as separate measures — a blocked gate is never a
|
|
9
|
+
* semantic failure, and an unassessed trial is never a zero.
|
|
10
|
+
*/
|
|
11
|
+
import { DEFAULT_WEIGHTS, trialUtility, } from "../scoring.js";
|
|
12
|
+
/** The assessment states every accounting record counts. */
|
|
13
|
+
const ASSESSMENT_STATUSES = [
|
|
14
|
+
"judged",
|
|
15
|
+
"invalid-verdict",
|
|
16
|
+
"judge-error",
|
|
17
|
+
"unassessed",
|
|
18
|
+
"not-run",
|
|
19
|
+
"unknown",
|
|
20
|
+
];
|
|
21
|
+
/** The documentation outcomes every accounting record counts. */
|
|
22
|
+
const OUTCOME_KEYS = [
|
|
23
|
+
"useful-create",
|
|
24
|
+
"useful-update",
|
|
25
|
+
"transient",
|
|
26
|
+
"duplicate",
|
|
27
|
+
"misleading",
|
|
28
|
+
"none",
|
|
29
|
+
];
|
|
30
|
+
/** Start a zeroed accounting record. */
|
|
31
|
+
function emptyCounts(planned) {
|
|
32
|
+
const assessmentStatuses = {};
|
|
33
|
+
for (const status of ASSESSMENT_STATUSES)
|
|
34
|
+
assessmentStatuses[status] = 0;
|
|
35
|
+
const outcomes = {};
|
|
36
|
+
for (const outcome of OUTCOME_KEYS)
|
|
37
|
+
outcomes[outcome] = 0;
|
|
38
|
+
return {
|
|
39
|
+
planned,
|
|
40
|
+
recordedLaunches: 0,
|
|
41
|
+
unprovenDirectories: 0,
|
|
42
|
+
absent: 0,
|
|
43
|
+
completed: 0,
|
|
44
|
+
usableUnknown: 0,
|
|
45
|
+
unusable: 0,
|
|
46
|
+
documented: 0,
|
|
47
|
+
assessed: 0,
|
|
48
|
+
excluded: 0,
|
|
49
|
+
assessmentStatuses,
|
|
50
|
+
primary: { pass: 0, fail: 0, unknown: 0 },
|
|
51
|
+
gate: { pass: 0, blocked: 0, other: 0, missing: 0 },
|
|
52
|
+
outcomes,
|
|
53
|
+
utilityTotal: 0,
|
|
54
|
+
utilityLow: 0,
|
|
55
|
+
utilityHigh: 0,
|
|
56
|
+
meanPerPlanned: null,
|
|
57
|
+
meanPerAssessed: null,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Score one loaded trial against its declared slot.
|
|
62
|
+
* @param record The loaded trial record.
|
|
63
|
+
* @param slot The manifest slot the record belongs to.
|
|
64
|
+
* @param options Weights, guardrail state and disposition overrides.
|
|
65
|
+
* @returns The scored trial with identity and utility bounds.
|
|
66
|
+
*/
|
|
67
|
+
export function scoreTrial(record, slot, options = {}) {
|
|
68
|
+
const disposition = options.disposition ?? slot.disposition ?? "first-attempt";
|
|
69
|
+
const trial = {
|
|
70
|
+
arm: slot.arm,
|
|
71
|
+
scenario: slot.scenario,
|
|
72
|
+
planned: disposition !== "excluded",
|
|
73
|
+
usable: record.attempt.usable === true,
|
|
74
|
+
outcome: record.assessment.outcome,
|
|
75
|
+
primary: record.primary.status,
|
|
76
|
+
guardrailBreach: options.guardrailBreach ?? false,
|
|
77
|
+
};
|
|
78
|
+
return {
|
|
79
|
+
identity: {
|
|
80
|
+
slot: slot.slot,
|
|
81
|
+
pairId: slot.pairId,
|
|
82
|
+
scenario: slot.scenario,
|
|
83
|
+
arm: slot.arm,
|
|
84
|
+
repetition: slot.repetition,
|
|
85
|
+
stamp: slot.stamp,
|
|
86
|
+
},
|
|
87
|
+
disposition,
|
|
88
|
+
attemptState: record.attempt.state,
|
|
89
|
+
trial,
|
|
90
|
+
utility: trialUtility(trial, options.weights ?? DEFAULT_WEIGHTS),
|
|
91
|
+
assessmentStatus: record.assessment.status,
|
|
92
|
+
gateStatus: record.gate.state === "available"
|
|
93
|
+
? record.gate.value.status
|
|
94
|
+
: "missing",
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Count what one trial's attempt record proves.
|
|
99
|
+
* @param counts The counts to update.
|
|
100
|
+
* @param row The scored trial.
|
|
101
|
+
*/
|
|
102
|
+
function addAttempt(counts, row) {
|
|
103
|
+
if (row.attemptState === "recorded") {
|
|
104
|
+
counts.recordedLaunches += 1;
|
|
105
|
+
}
|
|
106
|
+
else if (row.attemptState === "absent") {
|
|
107
|
+
counts.absent += 1;
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
counts.unprovenDirectories += 1;
|
|
111
|
+
}
|
|
112
|
+
if (row.attemptState !== "recorded") {
|
|
113
|
+
counts.usableUnknown += 1;
|
|
114
|
+
}
|
|
115
|
+
else if (row.trial.usable) {
|
|
116
|
+
counts.completed += 1;
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
counts.unusable += 1;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Count one trial's assessment, gate and primary outcomes.
|
|
124
|
+
* @param counts The counts to update.
|
|
125
|
+
* @param row The scored trial.
|
|
126
|
+
*/
|
|
127
|
+
function addAssessment(counts, row) {
|
|
128
|
+
counts.assessmentStatuses[row.assessmentStatus] += 1;
|
|
129
|
+
if (row.trial.primary === "pass")
|
|
130
|
+
counts.primary.pass += 1;
|
|
131
|
+
else if (row.trial.primary === "fail")
|
|
132
|
+
counts.primary.fail += 1;
|
|
133
|
+
else
|
|
134
|
+
counts.primary.unknown += 1;
|
|
135
|
+
if (row.gateStatus === "pass")
|
|
136
|
+
counts.gate.pass += 1;
|
|
137
|
+
else if (row.gateStatus === "blocked")
|
|
138
|
+
counts.gate.blocked += 1;
|
|
139
|
+
else if (row.gateStatus === "missing")
|
|
140
|
+
counts.gate.missing += 1;
|
|
141
|
+
else
|
|
142
|
+
counts.gate.other += 1;
|
|
143
|
+
if (row.assessmentStatus === "judged" && row.trial.outcome !== undefined) {
|
|
144
|
+
counts.documented += 1;
|
|
145
|
+
counts.outcomes[row.trial.outcome] += 1;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Aggregate one measured population.
|
|
150
|
+
* @param scored The scored trials, one per scheduled slot.
|
|
151
|
+
* @param planned The number of scheduled slots in the population.
|
|
152
|
+
* @returns The counts, totals and bounds.
|
|
153
|
+
*/
|
|
154
|
+
export function accountTrials(scored, planned) {
|
|
155
|
+
const counts = emptyCounts(planned);
|
|
156
|
+
let missingLow = 0;
|
|
157
|
+
let missingHigh = 0;
|
|
158
|
+
for (const row of scored) {
|
|
159
|
+
if (row.disposition === "excluded") {
|
|
160
|
+
counts.excluded += 1;
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
addAttempt(counts, row);
|
|
164
|
+
addAssessment(counts, row);
|
|
165
|
+
if (row.utility.assessed) {
|
|
166
|
+
counts.assessed += 1;
|
|
167
|
+
counts.utilityTotal += row.utility.score;
|
|
168
|
+
}
|
|
169
|
+
else {
|
|
170
|
+
missingLow += row.utility.low;
|
|
171
|
+
missingHigh += row.utility.high;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
counts.utilityLow = counts.utilityTotal + missingLow;
|
|
175
|
+
counts.utilityHigh = counts.utilityTotal + missingHigh;
|
|
176
|
+
counts.meanPerPlanned =
|
|
177
|
+
planned === 0 ? null : counts.utilityTotal / planned;
|
|
178
|
+
counts.meanPerAssessed =
|
|
179
|
+
counts.assessed === 0 ? null : counts.utilityTotal / counts.assessed;
|
|
180
|
+
return counts;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Aggregate one population per declared arm.
|
|
184
|
+
* @param scored The scored trials.
|
|
185
|
+
* @param slots The declared schedule slots, for planned denominators.
|
|
186
|
+
* @returns Counts keyed by arm id, sorted.
|
|
187
|
+
*/
|
|
188
|
+
export function accountByArm(scored, slots) {
|
|
189
|
+
const arms = [
|
|
190
|
+
...new Set([
|
|
191
|
+
...slots.map((slot) => slot.arm),
|
|
192
|
+
...scored.map((row) => row.trial.arm),
|
|
193
|
+
]),
|
|
194
|
+
].sort();
|
|
195
|
+
const byArm = {};
|
|
196
|
+
for (const arm of arms) {
|
|
197
|
+
byArm[arm] = accountTrials(scored.filter((row) => row.trial.arm === arm), slots.filter((slot) => slot.arm === arm).length);
|
|
198
|
+
}
|
|
199
|
+
return byArm;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Aggregate one population per arm and scenario.
|
|
203
|
+
* @param scored The scored trials.
|
|
204
|
+
* @param slots The declared schedule slots, for planned denominators.
|
|
205
|
+
* @returns Counts keyed by arm then scenario, with only present combinations.
|
|
206
|
+
*/
|
|
207
|
+
export function accountByScenarioArm(scored, slots) {
|
|
208
|
+
const byScenario = {};
|
|
209
|
+
for (const [arm, armSlots] of Object.entries(groupSlotsByArm(slots)).sort(([left], [right]) => left.localeCompare(right))) {
|
|
210
|
+
const scenarios = [
|
|
211
|
+
...new Set([
|
|
212
|
+
...armSlots.map((slot) => slot.scenario),
|
|
213
|
+
...scored
|
|
214
|
+
.filter((row) => row.trial.arm === arm)
|
|
215
|
+
.map((row) => row.trial.scenario),
|
|
216
|
+
]),
|
|
217
|
+
].sort();
|
|
218
|
+
byScenario[arm] = {};
|
|
219
|
+
for (const scenario of scenarios) {
|
|
220
|
+
byScenario[arm][scenario] = accountTrials(scored.filter((row) => row.trial.arm === arm &&
|
|
221
|
+
row.trial.scenario === scenario), armSlots.filter((slot) => slot.scenario === scenario).length);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return byScenario;
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Group declared slots by arm.
|
|
228
|
+
* @param slots The declared schedule slots.
|
|
229
|
+
* @returns Slots keyed by arm id.
|
|
230
|
+
*/
|
|
231
|
+
function groupSlotsByArm(slots) {
|
|
232
|
+
const grouped = {};
|
|
233
|
+
for (const slot of slots) {
|
|
234
|
+
grouped[slot.arm] = [...(grouped[slot.arm] ?? []), slot];
|
|
235
|
+
}
|
|
236
|
+
return grouped;
|
|
237
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Blinding checks for the trial child and the judge packet.
|
|
3
|
+
*
|
|
4
|
+
* The A/B description difference is the treatment and must survive. What must
|
|
5
|
+
* not cross either boundary is the arm's identity or mapping: an arm label, an
|
|
6
|
+
* arm-specific worktree path or commit, or a revealing stamp such as
|
|
7
|
+
* `pd-A-r0-S02`.
|
|
8
|
+
*/
|
|
9
|
+
/** One detected exposure. */
|
|
10
|
+
export interface BlindingFinding {
|
|
11
|
+
/** The field or value that carried the leak. */
|
|
12
|
+
field: string;
|
|
13
|
+
/** The matched fragment, never a whole secret. */
|
|
14
|
+
fragment: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Find arm-revealing fragments in one value.
|
|
18
|
+
* @param field The field or path the value came from.
|
|
19
|
+
* @param value The inspected text.
|
|
20
|
+
* @returns One finding per matched pattern, with only the matched fragment.
|
|
21
|
+
*/
|
|
22
|
+
export declare function blindingFindings(field: string, value: string): BlindingFinding[];
|
|
23
|
+
/**
|
|
24
|
+
* Inspect trial-child environment entries.
|
|
25
|
+
*
|
|
26
|
+
* Provenance variables differ per arm (the plugin commit and dirty flag), so
|
|
27
|
+
* they are arm-revealing at the trial boundary; the caller keeps them in the
|
|
28
|
+
* outer harness instead. Image and fixture identities are identical for both
|
|
29
|
+
* arms and allowed.
|
|
30
|
+
* @param env The environment the trial container receives.
|
|
31
|
+
* @returns Every exposure found.
|
|
32
|
+
*/
|
|
33
|
+
export declare function trialEnvFindings(env: Readonly<Record<string, string | undefined>>): BlindingFinding[];
|
|
34
|
+
/**
|
|
35
|
+
* Inspect every string field of a nested judge-packet object.
|
|
36
|
+
* @param packet The packet value, already parsed when it is JSON.
|
|
37
|
+
* @param prefix The dotted path accumulated so far.
|
|
38
|
+
* @returns Every exposure found in keys or values.
|
|
39
|
+
*/
|
|
40
|
+
export declare function packetFindings(packet: unknown, prefix?: string): BlindingFinding[];
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Blinding checks for the trial child and the judge packet.
|
|
3
|
+
*
|
|
4
|
+
* The A/B description difference is the treatment and must survive. What must
|
|
5
|
+
* not cross either boundary is the arm's identity or mapping: an arm label, an
|
|
6
|
+
* arm-specific worktree path or commit, or a revealing stamp such as
|
|
7
|
+
* `pd-A-r0-S02`.
|
|
8
|
+
*/
|
|
9
|
+
/** Patterns that reveal an arm rather than the treatment. */
|
|
10
|
+
const ARM_PATTERNS = [
|
|
11
|
+
{ label: "stamp", pattern: /pd-[AB]-r\d/ },
|
|
12
|
+
{
|
|
13
|
+
label: "arm-path",
|
|
14
|
+
pattern: /(?:^|[^A-Za-z0-9])arms?[/_-][ab](?:[^A-Za-z0-9]|$)/i,
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
label: "arm-flag",
|
|
18
|
+
pattern: /(?:^|[^A-Za-z0-9])arm[-_ ]?[ab](?:[^A-Za-z0-9]|$)/i,
|
|
19
|
+
},
|
|
20
|
+
];
|
|
21
|
+
/**
|
|
22
|
+
* Find arm-revealing fragments in one value.
|
|
23
|
+
* @param field The field or path the value came from.
|
|
24
|
+
* @param value The inspected text.
|
|
25
|
+
* @returns One finding per matched pattern, with only the matched fragment.
|
|
26
|
+
*/
|
|
27
|
+
export function blindingFindings(field, value) {
|
|
28
|
+
const findings = [];
|
|
29
|
+
for (const { label, pattern } of ARM_PATTERNS) {
|
|
30
|
+
const match = pattern.exec(value);
|
|
31
|
+
if (match === null)
|
|
32
|
+
continue;
|
|
33
|
+
findings.push({ field, fragment: `${label}:${match[0].trim()}` });
|
|
34
|
+
}
|
|
35
|
+
return findings;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Inspect trial-child environment entries.
|
|
39
|
+
*
|
|
40
|
+
* Provenance variables differ per arm (the plugin commit and dirty flag), so
|
|
41
|
+
* they are arm-revealing at the trial boundary; the caller keeps them in the
|
|
42
|
+
* outer harness instead. Image and fixture identities are identical for both
|
|
43
|
+
* arms and allowed.
|
|
44
|
+
* @param env The environment the trial container receives.
|
|
45
|
+
* @returns Every exposure found.
|
|
46
|
+
*/
|
|
47
|
+
export function trialEnvFindings(env) {
|
|
48
|
+
const findings = [];
|
|
49
|
+
for (const [key, value] of Object.entries(env)) {
|
|
50
|
+
if ((key === "PLACEMENT_PLUGIN_COMMIT" ||
|
|
51
|
+
key === "PLACEMENT_PLUGIN_DIRTY") &&
|
|
52
|
+
value !== undefined &&
|
|
53
|
+
value !== "") {
|
|
54
|
+
findings.push({ field: key, fragment: "arm-specific provenance" });
|
|
55
|
+
}
|
|
56
|
+
if (value !== undefined && value !== "") {
|
|
57
|
+
findings.push(...blindingFindings(key, value));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return findings;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Inspect every string field of a nested judge-packet object.
|
|
64
|
+
* @param packet The packet value, already parsed when it is JSON.
|
|
65
|
+
* @param prefix The dotted path accumulated so far.
|
|
66
|
+
* @returns Every exposure found in keys or values.
|
|
67
|
+
*/
|
|
68
|
+
export function packetFindings(packet, prefix = "packet") {
|
|
69
|
+
if (typeof packet === "string")
|
|
70
|
+
return blindingFindings(prefix, packet);
|
|
71
|
+
if (Array.isArray(packet)) {
|
|
72
|
+
return packet.flatMap((entry, index) => packetFindings(entry, `${prefix}[${String(index)}]`));
|
|
73
|
+
}
|
|
74
|
+
if (typeof packet !== "object" || packet === null)
|
|
75
|
+
return [];
|
|
76
|
+
return Object.entries(packet).flatMap(([key, value]) => [
|
|
77
|
+
...blindingFindings(`${prefix}.${key}`, key),
|
|
78
|
+
...packetFindings(value, `${prefix}.${key}`),
|
|
79
|
+
]);
|
|
80
|
+
}
|