@markstanden/opencode-test-container 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +263 -2
- package/container/Containerfile +24 -0
- package/container/README.md +29 -0
- package/dist/adapters/opencode/events.d.ts +90 -0
- package/dist/adapters/opencode/events.js +213 -0
- package/dist/adapters/opencode/runtime.d.ts +87 -0
- package/dist/adapters/opencode/runtime.js +136 -0
- package/dist/adapters/podman/container.d.ts +62 -0
- package/dist/adapters/podman/container.js +119 -0
- package/dist/adapters/podman/image.d.ts +50 -0
- package/dist/adapters/podman/image.js +124 -0
- package/dist/adapters/process.d.ts +38 -0
- package/dist/adapters/process.js +71 -0
- package/dist/campaign/assess.d.ts +45 -0
- package/dist/campaign/assess.js +208 -0
- package/dist/campaign/close.d.ts +17 -0
- package/dist/campaign/close.js +112 -0
- package/dist/campaign/evidence-guard.d.ts +24 -0
- package/dist/campaign/evidence-guard.js +49 -0
- package/dist/campaign/freeze.d.ts +94 -0
- package/dist/campaign/freeze.js +297 -0
- package/dist/campaign/inventory.d.ts +62 -0
- package/dist/campaign/inventory.js +192 -0
- package/dist/campaign/judge.d.ts +209 -0
- package/dist/campaign/judge.js +1285 -0
- package/dist/campaign/ledger.d.ts +64 -0
- package/dist/campaign/ledger.js +149 -0
- package/dist/campaign/prepare.d.ts +129 -0
- package/dist/campaign/prepare.js +483 -0
- package/dist/campaign/primary.d.ts +96 -0
- package/dist/campaign/primary.js +189 -0
- package/dist/campaign/run.d.ts +151 -0
- package/dist/campaign/run.js +562 -0
- package/dist/campaign/status.d.ts +76 -0
- package/dist/campaign/status.js +237 -0
- package/dist/cli/otc.d.ts +51 -0
- package/dist/cli/otc.js +435 -0
- package/dist/contracts/canonical.d.ts +21 -0
- package/dist/contracts/canonical.js +44 -0
- package/dist/contracts/types.d.ts +261 -0
- package/dist/contracts/types.js +9 -0
- package/dist/contracts/validate.d.ts +72 -0
- package/dist/contracts/validate.js +1021 -0
- package/dist/contracts/version.d.ts +44 -0
- package/dist/contracts/version.js +147 -0
- package/dist/evaluators/docs/analysis/accounting.d.ts +140 -0
- package/dist/evaluators/docs/analysis/accounting.js +237 -0
- package/dist/evaluators/docs/analysis/blinding.d.ts +40 -0
- package/dist/evaluators/docs/analysis/blinding.js +80 -0
- package/dist/evaluators/docs/analysis/bounds.d.ts +40 -0
- package/dist/evaluators/docs/analysis/bounds.js +31 -0
- package/dist/evaluators/docs/analysis/budget.d.ts +137 -0
- package/dist/evaluators/docs/analysis/budget.js +305 -0
- package/dist/evaluators/docs/analysis/closure.d.ts +175 -0
- package/dist/evaluators/docs/analysis/closure.js +251 -0
- package/dist/evaluators/docs/analysis/decision.d.ts +59 -0
- package/dist/evaluators/docs/analysis/decision.js +161 -0
- package/dist/evaluators/docs/analysis/freeze.d.ts +201 -0
- package/dist/evaluators/docs/analysis/freeze.js +395 -0
- package/dist/evaluators/docs/analysis/guardrails.d.ts +51 -0
- package/dist/evaluators/docs/analysis/guardrails.js +194 -0
- package/dist/evaluators/docs/analysis/journal.d.ts +150 -0
- package/dist/evaluators/docs/analysis/journal.js +475 -0
- package/dist/evaluators/docs/analysis/ledger.d.ts +78 -0
- package/dist/evaluators/docs/analysis/ledger.js +145 -0
- package/dist/evaluators/docs/analysis/measures.d.ts +302 -0
- package/dist/evaluators/docs/analysis/measures.js +1069 -0
- package/dist/evaluators/docs/analysis/model.d.ts +236 -0
- package/dist/evaluators/docs/analysis/model.js +740 -0
- package/dist/evaluators/docs/analysis/pairing.d.ts +68 -0
- package/dist/evaluators/docs/analysis/pairing.js +95 -0
- package/dist/evaluators/docs/analysis/records.d.ts +229 -0
- package/dist/evaluators/docs/analysis/records.js +935 -0
- package/dist/evaluators/docs/analysis/recovery.d.ts +225 -0
- package/dist/evaluators/docs/analysis/recovery.js +507 -0
- package/dist/evaluators/docs/analysis/render.d.ts +235 -0
- package/dist/evaluators/docs/analysis/render.js +566 -0
- package/dist/evaluators/docs/analysis/replay.d.ts +15 -0
- package/dist/evaluators/docs/analysis/replay.js +1272 -0
- package/dist/evaluators/docs/analysis/screening/inventory.d.ts +136 -0
- package/dist/evaluators/docs/analysis/screening/inventory.js +307 -0
- package/dist/evaluators/docs/analysis/screening/noteVersion.d.ts +15 -0
- package/dist/evaluators/docs/analysis/screening/noteVersion.js +46 -0
- package/dist/evaluators/docs/analysis/screening/reconcile.d.ts +38 -0
- package/dist/evaluators/docs/analysis/screening/reconcile.js +40 -0
- package/dist/evaluators/docs/analysis/screening/rules.d.ts +163 -0
- package/dist/evaluators/docs/analysis/screening/rules.js +330 -0
- package/dist/evaluators/docs/analysis/screening/scan.d.ts +72 -0
- package/dist/evaluators/docs/analysis/screening/scan.js +362 -0
- package/dist/evaluators/docs/analysis/selector.d.ts +68 -0
- package/dist/evaluators/docs/analysis/selector.js +123 -0
- package/dist/evaluators/docs/assess.d.ts +129 -0
- package/dist/evaluators/docs/assess.js +433 -0
- package/dist/evaluators/docs/campaign.d.ts +133 -0
- package/dist/evaluators/docs/campaign.js +222 -0
- package/dist/evaluators/docs/index.d.ts +25 -0
- package/dist/evaluators/docs/index.js +25 -0
- package/dist/evaluators/docs/journal.d.ts +150 -0
- package/dist/evaluators/docs/journal.js +475 -0
- package/dist/evaluators/docs/scoring.d.ts +367 -0
- package/dist/evaluators/docs/scoring.js +662 -0
- package/dist/evaluators/skills/assess.d.ts +94 -0
- package/dist/evaluators/skills/assess.js +207 -0
- package/dist/index.d.ts +46 -0
- package/dist/index.js +29 -0
- package/dist/legacy/prod014.d.ts +86 -0
- package/dist/legacy/prod014.js +209 -0
- package/dist/reporting/report.d.ts +79 -0
- package/dist/reporting/report.js +242 -0
- package/docs/architecture.md +106 -0
- package/docs/capability-matrix.md +84 -0
- package/docs/offline-rehearsal.md +75 -0
- package/docs/parity-record.md +81 -0
- package/docs/pr-description.md +103 -0
- package/docs/schema.md +97 -0
- package/examples/skills/README.md +13 -0
- package/examples/skills/campaign.json +73 -0
- package/examples/skills/skills/otel-helper/SKILL.md +10 -0
- package/examples/skills/suite.json +29 -0
- package/package.json +45 -5
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import type { SuiteCase } from "../../contracts/types.ts";
|
|
2
|
+
import { type Usage } from "../../adapters/opencode/events.ts";
|
|
3
|
+
/** The evidence states the evaluator can establish. */
|
|
4
|
+
export type SkillValidity = "valid" | "invalid" | "unknown" | "unrun";
|
|
5
|
+
/** What the trail showed, independent of the expected outcome. */
|
|
6
|
+
export type SkillObservation = "target-activation" | "attempted-target-load" | "failed-target-load" | "other-skill-only" | "no-invocation";
|
|
7
|
+
/** The criterion verdict. */
|
|
8
|
+
export type SkillOutcome = "pass" | "fail" | "unknown";
|
|
9
|
+
/** Exact counts, never collapsed into a single score. */
|
|
10
|
+
export interface SkillCounts {
|
|
11
|
+
/** The requested target value. */
|
|
12
|
+
requestedTarget: number;
|
|
13
|
+
/** The successful target value. */
|
|
14
|
+
successfulTarget: number;
|
|
15
|
+
/** The failed target value. */
|
|
16
|
+
failedTarget: number;
|
|
17
|
+
/** The other skill value. */
|
|
18
|
+
otherSkill: number;
|
|
19
|
+
/** The total skill calls value. */
|
|
20
|
+
totalSkillCalls: number;
|
|
21
|
+
}
|
|
22
|
+
/** The deterministic assessment of one skill-trigger trial. */
|
|
23
|
+
export interface SkillAssessment {
|
|
24
|
+
/** The schema version value. */
|
|
25
|
+
schemaVersion: 1;
|
|
26
|
+
/** The kind value. */
|
|
27
|
+
kind: "skill-activation";
|
|
28
|
+
/** The campaign id value. */
|
|
29
|
+
campaignId: string;
|
|
30
|
+
/** The slot id value. */
|
|
31
|
+
slotId: string;
|
|
32
|
+
/** The case id value. */
|
|
33
|
+
caseId: string;
|
|
34
|
+
/** The arm id value. */
|
|
35
|
+
armId: string;
|
|
36
|
+
/** The repetition id value. */
|
|
37
|
+
repetitionId: string;
|
|
38
|
+
/** The attempt id value. */
|
|
39
|
+
attemptId: string;
|
|
40
|
+
/** The target skill id value. */
|
|
41
|
+
targetSkillId: string;
|
|
42
|
+
/** The expected value. */
|
|
43
|
+
expected: "activation" | "no-activation";
|
|
44
|
+
/** The validity value. */
|
|
45
|
+
validity: SkillValidity;
|
|
46
|
+
/** The observed value. */
|
|
47
|
+
observed: SkillObservation | "unrun";
|
|
48
|
+
/** The outcome value. */
|
|
49
|
+
outcome: SkillOutcome;
|
|
50
|
+
/** The counts value. */
|
|
51
|
+
counts: SkillCounts;
|
|
52
|
+
/** The body routes value. */
|
|
53
|
+
bodyRoutes: string[];
|
|
54
|
+
/** The contaminated value. */
|
|
55
|
+
contaminated: boolean;
|
|
56
|
+
/** The reasons value. */
|
|
57
|
+
reasons: string[];
|
|
58
|
+
/** The usage value. */
|
|
59
|
+
usage: Usage | null;
|
|
60
|
+
/** The answered value. */
|
|
61
|
+
answered: boolean;
|
|
62
|
+
/** The assessed at value. */
|
|
63
|
+
assessedAt: string;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Assess one skill-trigger trial from checked evidence. Capture validity and
|
|
67
|
+
* criterion outcome are distinct: a complete run with no target activation is
|
|
68
|
+
* an observed negative; an unreadable or failed load is unknown, never a
|
|
69
|
+
* passing negative.
|
|
70
|
+
* @param options Trial identity, case definition and its evidence directory.
|
|
71
|
+
* @returns The assessment.
|
|
72
|
+
* @param options.campaignId The campaignId value.
|
|
73
|
+
* @param options.slotId The slotId value.
|
|
74
|
+
* @param options.caseId The caseId value.
|
|
75
|
+
* @param options.armId The armId value.
|
|
76
|
+
* @param options.repetitionId The repetitionId value.
|
|
77
|
+
* @param options.testCase The testCase value.
|
|
78
|
+
* @param options.trialDir The trialDir value.
|
|
79
|
+
* @param options.attemptId The attemptId value.
|
|
80
|
+
* @param options.assessedAt The assessedAt value.
|
|
81
|
+
*/
|
|
82
|
+
export declare function assessSkillTrial(options: {
|
|
83
|
+
campaignId: string;
|
|
84
|
+
slotId: string;
|
|
85
|
+
caseId: string;
|
|
86
|
+
armId: string;
|
|
87
|
+
repetitionId: string;
|
|
88
|
+
testCase: Extract<SuiteCase, {
|
|
89
|
+
kind: "skill-trigger";
|
|
90
|
+
}>;
|
|
91
|
+
trialDir: string;
|
|
92
|
+
attemptId: string;
|
|
93
|
+
assessedAt: string;
|
|
94
|
+
}): SkillAssessment;
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { bodyRoutes, captureProblems, decodeTrail, skillCalls, } from "../../adapters/opencode/events.js";
|
|
4
|
+
import { isRecord } from "../../contracts/validate.js";
|
|
5
|
+
/** The zeroed count block. */
|
|
6
|
+
function emptyCounts() {
|
|
7
|
+
return {
|
|
8
|
+
requestedTarget: 0,
|
|
9
|
+
successfulTarget: 0,
|
|
10
|
+
failedTarget: 0,
|
|
11
|
+
otherSkill: 0,
|
|
12
|
+
totalSkillCalls: 0,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
/** Split skill calls into target and other counts. */
|
|
16
|
+
function countsOf(calls, targetSkillId) {
|
|
17
|
+
const counts = emptyCounts();
|
|
18
|
+
for (const call of calls) {
|
|
19
|
+
counts.totalSkillCalls += 1;
|
|
20
|
+
if (call.requestedId !== targetSkillId) {
|
|
21
|
+
if (call.loaded)
|
|
22
|
+
counts.otherSkill += 1;
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
25
|
+
counts.requestedTarget += 1;
|
|
26
|
+
if (call.loaded)
|
|
27
|
+
counts.successfulTarget += 1;
|
|
28
|
+
else
|
|
29
|
+
counts.failedTarget += 1;
|
|
30
|
+
}
|
|
31
|
+
return counts;
|
|
32
|
+
}
|
|
33
|
+
/** Read a trial meta record without trusting it. */
|
|
34
|
+
function readMeta(path) {
|
|
35
|
+
if (!existsSync(path))
|
|
36
|
+
return null;
|
|
37
|
+
try {
|
|
38
|
+
const value = JSON.parse(readFileSync(path, "utf8"));
|
|
39
|
+
return isRecord(value) ? value : null;
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** Read the usage record, or null when absent/malformed. */
|
|
46
|
+
function readUsage(path) {
|
|
47
|
+
if (!existsSync(path))
|
|
48
|
+
return null;
|
|
49
|
+
try {
|
|
50
|
+
const value = JSON.parse(readFileSync(path, "utf8"));
|
|
51
|
+
return isRecord(value) ? value : null;
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** An unrun assessment with no usable evidence. */
|
|
58
|
+
function unrunAssessment(base, reason, assessedAt) {
|
|
59
|
+
return {
|
|
60
|
+
...base,
|
|
61
|
+
validity: "unrun",
|
|
62
|
+
observed: "unrun",
|
|
63
|
+
outcome: "unknown",
|
|
64
|
+
counts: emptyCounts(),
|
|
65
|
+
bodyRoutes: [],
|
|
66
|
+
contaminated: false,
|
|
67
|
+
reasons: [reason],
|
|
68
|
+
usage: null,
|
|
69
|
+
answered: false,
|
|
70
|
+
assessedAt,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/** Capture integrity problems from meta and the decoded trail. */
|
|
74
|
+
function skillIntegrity(meta, trail) {
|
|
75
|
+
const integrity = [...captureProblems(trail)];
|
|
76
|
+
if (meta.captureValid !== true)
|
|
77
|
+
integrity.push(...(meta.captureProblems ?? []));
|
|
78
|
+
if (meta.exit !== 0)
|
|
79
|
+
integrity.push(`container exited ${String(meta.exit)}`);
|
|
80
|
+
if (meta.timedOut)
|
|
81
|
+
integrity.push("container timed out");
|
|
82
|
+
return integrity;
|
|
83
|
+
}
|
|
84
|
+
/** The observed state independent of expectation. */
|
|
85
|
+
function observationOf(counts) {
|
|
86
|
+
if (counts.successfulTarget > 0)
|
|
87
|
+
return "target-activation";
|
|
88
|
+
if (counts.otherSkill > 0)
|
|
89
|
+
return "other-skill-only";
|
|
90
|
+
return "no-invocation";
|
|
91
|
+
}
|
|
92
|
+
/** The criterion outcome for a valid observation. */
|
|
93
|
+
function outcomeOf(expected, counts) {
|
|
94
|
+
const activated = counts.successfulTarget > 0;
|
|
95
|
+
return (expected === "activation") === activated ? "pass" : "fail";
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Assess one skill-trigger trial from checked evidence. Capture validity and
|
|
99
|
+
* criterion outcome are distinct: a complete run with no target activation is
|
|
100
|
+
* an observed negative; an unreadable or failed load is unknown, never a
|
|
101
|
+
* passing negative.
|
|
102
|
+
* @param options Trial identity, case definition and its evidence directory.
|
|
103
|
+
* @returns The assessment.
|
|
104
|
+
* @param options.campaignId The campaignId value.
|
|
105
|
+
* @param options.slotId The slotId value.
|
|
106
|
+
* @param options.caseId The caseId value.
|
|
107
|
+
* @param options.armId The armId value.
|
|
108
|
+
* @param options.repetitionId The repetitionId value.
|
|
109
|
+
* @param options.testCase The testCase value.
|
|
110
|
+
* @param options.trialDir The trialDir value.
|
|
111
|
+
* @param options.attemptId The attemptId value.
|
|
112
|
+
* @param options.assessedAt The assessedAt value.
|
|
113
|
+
*/
|
|
114
|
+
export function assessSkillTrial(options) {
|
|
115
|
+
const { testCase, trialDir } = options;
|
|
116
|
+
const meta = readMeta(join(trialDir, "meta.json"));
|
|
117
|
+
const base = {
|
|
118
|
+
schemaVersion: 1,
|
|
119
|
+
kind: "skill-activation",
|
|
120
|
+
campaignId: options.campaignId,
|
|
121
|
+
slotId: options.slotId,
|
|
122
|
+
caseId: options.caseId,
|
|
123
|
+
armId: options.armId,
|
|
124
|
+
repetitionId: options.repetitionId,
|
|
125
|
+
attemptId: options.attemptId,
|
|
126
|
+
targetSkillId: testCase.targetSkillId,
|
|
127
|
+
expected: testCase.expected,
|
|
128
|
+
};
|
|
129
|
+
if (meta === null) {
|
|
130
|
+
return unrunAssessment(base, "trial meta is missing or unreadable", options.assessedAt);
|
|
131
|
+
}
|
|
132
|
+
if (!existsSync(join(trialDir, "events.jsonl"))) {
|
|
133
|
+
return unrunAssessment(base, "trial event stream is missing", options.assessedAt);
|
|
134
|
+
}
|
|
135
|
+
const trail = decodeTrail(readFileSync(join(trialDir, "events.jsonl"), "utf8"));
|
|
136
|
+
const calls = skillCalls(trail);
|
|
137
|
+
const counts = countsOf(calls, testCase.targetSkillId);
|
|
138
|
+
const routes = bodyRoutes(trail, testCase.targetSkillId);
|
|
139
|
+
const contamination = routes.length > 0;
|
|
140
|
+
const integrity = skillIntegrity(meta, trail);
|
|
141
|
+
if (integrity.length > 0) {
|
|
142
|
+
return {
|
|
143
|
+
...base,
|
|
144
|
+
validity: "invalid",
|
|
145
|
+
observed: "unrun",
|
|
146
|
+
outcome: "unknown",
|
|
147
|
+
counts,
|
|
148
|
+
bodyRoutes: routes,
|
|
149
|
+
contaminated: contamination,
|
|
150
|
+
reasons: integrity,
|
|
151
|
+
usage: readUsage(join(trialDir, "usage.json")),
|
|
152
|
+
answered: trail.text.trim() !== "",
|
|
153
|
+
assessedAt: options.assessedAt,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
if (contamination && testCase.contamination !== "record") {
|
|
157
|
+
return {
|
|
158
|
+
...base,
|
|
159
|
+
validity: "invalid",
|
|
160
|
+
observed: counts.successfulTarget > 0
|
|
161
|
+
? "target-activation"
|
|
162
|
+
: "no-invocation",
|
|
163
|
+
outcome: "unknown",
|
|
164
|
+
counts,
|
|
165
|
+
bodyRoutes: routes,
|
|
166
|
+
contaminated: true,
|
|
167
|
+
reasons: [
|
|
168
|
+
`the target body was reached through ${routes.join(", ")}, not a skill-tool activation`,
|
|
169
|
+
],
|
|
170
|
+
usage: readUsage(join(trialDir, "usage.json")),
|
|
171
|
+
answered: trail.text.trim() !== "",
|
|
172
|
+
assessedAt: options.assessedAt,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
if (counts.failedTarget > 0) {
|
|
176
|
+
return {
|
|
177
|
+
...base,
|
|
178
|
+
validity: "unknown",
|
|
179
|
+
observed: "failed-target-load",
|
|
180
|
+
outcome: "unknown",
|
|
181
|
+
counts,
|
|
182
|
+
bodyRoutes: routes,
|
|
183
|
+
contaminated: contamination,
|
|
184
|
+
reasons: [
|
|
185
|
+
"the target skill load failed; successful activation is unknown",
|
|
186
|
+
],
|
|
187
|
+
usage: readUsage(join(trialDir, "usage.json")),
|
|
188
|
+
answered: trail.text.trim() !== "",
|
|
189
|
+
assessedAt: options.assessedAt,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
const observed = observationOf(counts);
|
|
193
|
+
const outcome = outcomeOf(testCase.expected, counts);
|
|
194
|
+
return {
|
|
195
|
+
...base,
|
|
196
|
+
validity: "valid",
|
|
197
|
+
observed,
|
|
198
|
+
outcome,
|
|
199
|
+
counts,
|
|
200
|
+
bodyRoutes: routes,
|
|
201
|
+
contaminated: contamination,
|
|
202
|
+
reasons: [],
|
|
203
|
+
usage: readUsage(join(trialDir, "usage.json")),
|
|
204
|
+
answered: trail.text.trim() !== "",
|
|
205
|
+
assessedAt: options.assessedAt,
|
|
206
|
+
};
|
|
207
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @markstanden/opencode-test-container
|
|
3
|
+
*
|
|
4
|
+
* Public modules are re-exported here for programmatic use. The supported
|
|
5
|
+
* consumer surface is the `otc` CLI; these exports exist so tests and
|
|
6
|
+
* evaluators can exercise the same lifecycle without a shell.
|
|
7
|
+
*/
|
|
8
|
+
export { canonicalJson, sha256Bytes, sha256Of } from "./contracts/canonical.ts";
|
|
9
|
+
export { EVALUATOR_VERSION, PACKAGE_NAME, packageVersion, toolIdentity, } from "./contracts/version.ts";
|
|
10
|
+
export type { ToolIdentity } from "./contracts/version.ts";
|
|
11
|
+
export type { Arm, AssessmentKind, Campaign, CampaignBundle, DocsTaskCase, ExpectedDocs, LifecycleState, PrimaryCheckSpec, ProviderRoute, RecoveryDeclaration, RuntimeSpec, SkillTriggerCase, Slot, Suite, SuiteCase, } from "./contracts/types.ts";
|
|
12
|
+
export { ID_PATTERN, isId, isSafeRelativePath, loadCampaignBundle, validateBundle, validateCampaign, validateSuite, } from "./contracts/validate.ts";
|
|
13
|
+
export { DEFAULT_PLUGIN_INCLUDE, loadPreparedManifest, manifestSha256, normaliseCampaign, prepareCampaign, preparedDifferences, } from "./campaign/prepare.ts";
|
|
14
|
+
export type { PreparedManifest, StagedArm } from "./campaign/prepare.ts";
|
|
15
|
+
export { deliveryCheck, freezeCampaign, freezeDifferences, freezeIdentity, loadFrozen, } from "./campaign/freeze.ts";
|
|
16
|
+
export type { FrozenRecord } from "./campaign/freeze.ts";
|
|
17
|
+
export { appendLedgerEvent, launchRefusal, parseLedger, readLedger, trialStarts, } from "./campaign/ledger.ts";
|
|
18
|
+
export type { LedgerEvent, LedgerRead } from "./campaign/ledger.ts";
|
|
19
|
+
export { launchSha256, planTrial, runCampaign, trialDirectory, } from "./campaign/run.ts";
|
|
20
|
+
export type { RunSummary, TrialMeta } from "./campaign/run.ts";
|
|
21
|
+
export { judgeCampaign, buildJudgePacketForSlot, checkedSelection, checkedSelectionForSlot, readJudgeAttempts, readSelectedJudgement, selectionPath, } from "./campaign/judge.ts";
|
|
22
|
+
export type { CheckedSelection, JudgeAttemptMeta, JudgeSummary, SelectedJudgement, StoredJudgement, } from "./campaign/judge.ts";
|
|
23
|
+
export { executePrimaryCheck, primaryCheckPlan, readPrimaryCheck, } from "./campaign/primary.ts";
|
|
24
|
+
export type { PrimaryCheckRecord } from "./campaign/primary.ts";
|
|
25
|
+
export { assessCampaign, assessmentPath, readAssessments, } from "./campaign/assess.ts";
|
|
26
|
+
export type { StoredAssessment } from "./campaign/assess.ts";
|
|
27
|
+
export { closeCampaign } from "./campaign/close.ts";
|
|
28
|
+
export { statusOf } from "./campaign/status.ts";
|
|
29
|
+
export type { StatusReport } from "./campaign/status.ts";
|
|
30
|
+
export { analyseCampaign, reportIdentity, reportIntact, } from "./reporting/report.ts";
|
|
31
|
+
export type { Report } from "./reporting/report.ts";
|
|
32
|
+
export { decodeTrail, captureProblems, skillCalls, } from "./adapters/opencode/events.ts";
|
|
33
|
+
export type { DecodedTrail, SkillCall, Usage, } from "./adapters/opencode/events.ts";
|
|
34
|
+
export { containerArgv, mountProblems, runContainer, } from "./adapters/podman/container.ts";
|
|
35
|
+
export type { ContainerPlan, Mount } from "./adapters/podman/container.ts";
|
|
36
|
+
export { buildImage, containerfileText, parseOpencodeVersion, resolveImage, } from "./adapters/podman/image.ts";
|
|
37
|
+
export { assessDocsTrial, docsVerbOf } from "./evaluators/docs/assess.ts";
|
|
38
|
+
export type { DocsAssessment } from "./evaluators/docs/assess.ts";
|
|
39
|
+
export * from "./evaluators/docs/index.ts";
|
|
40
|
+
export { hostDispatchesFromTrail, isServedRecord, journalIdentityFromTrail, journalMacValid, verifyOperationsJournal, } from "./evaluators/docs/journal.ts";
|
|
41
|
+
export type { JournalDispatch, JournalVerification, } from "./evaluators/docs/journal.ts";
|
|
42
|
+
export { assessSkillTrial } from "./evaluators/skills/assess.ts";
|
|
43
|
+
export type { SkillAssessment } from "./evaluators/skills/assess.ts";
|
|
44
|
+
export { readLegacyManifest, readLegacySchedule } from "./legacy/prod014.ts";
|
|
45
|
+
export type { LegacyManifest, LegacySchedule, LegacySlot, } from "./legacy/prod014.ts";
|
|
46
|
+
export { main, parseArgs, USAGE } from "./cli/otc.ts";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @markstanden/opencode-test-container
|
|
3
|
+
*
|
|
4
|
+
* Public modules are re-exported here for programmatic use. The supported
|
|
5
|
+
* consumer surface is the `otc` CLI; these exports exist so tests and
|
|
6
|
+
* evaluators can exercise the same lifecycle without a shell.
|
|
7
|
+
*/
|
|
8
|
+
export { canonicalJson, sha256Bytes, sha256Of } from "./contracts/canonical.js";
|
|
9
|
+
export { EVALUATOR_VERSION, PACKAGE_NAME, packageVersion, toolIdentity, } from "./contracts/version.js";
|
|
10
|
+
export { ID_PATTERN, isId, isSafeRelativePath, loadCampaignBundle, validateBundle, validateCampaign, validateSuite, } from "./contracts/validate.js";
|
|
11
|
+
export { DEFAULT_PLUGIN_INCLUDE, loadPreparedManifest, manifestSha256, normaliseCampaign, prepareCampaign, preparedDifferences, } from "./campaign/prepare.js";
|
|
12
|
+
export { deliveryCheck, freezeCampaign, freezeDifferences, freezeIdentity, loadFrozen, } from "./campaign/freeze.js";
|
|
13
|
+
export { appendLedgerEvent, launchRefusal, parseLedger, readLedger, trialStarts, } from "./campaign/ledger.js";
|
|
14
|
+
export { launchSha256, planTrial, runCampaign, trialDirectory, } from "./campaign/run.js";
|
|
15
|
+
export { judgeCampaign, buildJudgePacketForSlot, checkedSelection, checkedSelectionForSlot, readJudgeAttempts, readSelectedJudgement, selectionPath, } from "./campaign/judge.js";
|
|
16
|
+
export { executePrimaryCheck, primaryCheckPlan, readPrimaryCheck, } from "./campaign/primary.js";
|
|
17
|
+
export { assessCampaign, assessmentPath, readAssessments, } from "./campaign/assess.js";
|
|
18
|
+
export { closeCampaign } from "./campaign/close.js";
|
|
19
|
+
export { statusOf } from "./campaign/status.js";
|
|
20
|
+
export { analyseCampaign, reportIdentity, reportIntact, } from "./reporting/report.js";
|
|
21
|
+
export { decodeTrail, captureProblems, skillCalls, } from "./adapters/opencode/events.js";
|
|
22
|
+
export { containerArgv, mountProblems, runContainer, } from "./adapters/podman/container.js";
|
|
23
|
+
export { buildImage, containerfileText, parseOpencodeVersion, resolveImage, } from "./adapters/podman/image.js";
|
|
24
|
+
export { assessDocsTrial, docsVerbOf } from "./evaluators/docs/assess.js";
|
|
25
|
+
export * from "./evaluators/docs/index.js";
|
|
26
|
+
export { hostDispatchesFromTrail, isServedRecord, journalIdentityFromTrail, journalMacValid, verifyOperationsJournal, } from "./evaluators/docs/journal.js";
|
|
27
|
+
export { assessSkillTrial } from "./evaluators/skills/assess.js";
|
|
28
|
+
export { readLegacyManifest, readLegacySchedule } from "./legacy/prod014.js";
|
|
29
|
+
export { main, parseArgs, USAGE } from "./cli/otc.js";
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Explicit readers for the frozen prod014 layout.
|
|
3
|
+
*
|
|
4
|
+
* Historical scenario ids, stamps, pair ids and frozen schedule bytes are
|
|
5
|
+
* records, not templates. These adapters translate the old fields into the
|
|
6
|
+
* new vocabulary without renaming an id or rewriting the source file. A
|
|
7
|
+
* legacy campaign is never resumed by the new engine: the reader exists for
|
|
8
|
+
* comparison and provenance, and new work gets a fresh OTC root.
|
|
9
|
+
*/
|
|
10
|
+
import type { Slot } from "../contracts/types.ts";
|
|
11
|
+
/** One translated legacy slot, retaining every historical identity. */
|
|
12
|
+
export interface LegacySlot {
|
|
13
|
+
/** The new slot identity: the frozen stamp, unchanged. */
|
|
14
|
+
slot: Slot;
|
|
15
|
+
/** The historical numeric slot position. */
|
|
16
|
+
legacySlot: number;
|
|
17
|
+
/** The frozen pair identity. */
|
|
18
|
+
pairId: string;
|
|
19
|
+
/** The historical scenario id, unchanged. */
|
|
20
|
+
legacyScenario: string;
|
|
21
|
+
/** The historical stamp, unchanged. */
|
|
22
|
+
legacyStamp: string;
|
|
23
|
+
/** The historical arm label, unchanged. */
|
|
24
|
+
legacyArm: string;
|
|
25
|
+
}
|
|
26
|
+
/** A translated legacy schedule. */
|
|
27
|
+
export interface LegacySchedule {
|
|
28
|
+
/** The kind value. */
|
|
29
|
+
kind: "prod014-schedule";
|
|
30
|
+
/** The legacy campaign id value. */
|
|
31
|
+
legacyCampaignId: string;
|
|
32
|
+
/** The generator value. */
|
|
33
|
+
generator: string | null;
|
|
34
|
+
/** The slots value. */
|
|
35
|
+
slots: LegacySlot[];
|
|
36
|
+
}
|
|
37
|
+
/** A parsed legacy record, or every reason it is unusable. */
|
|
38
|
+
export type LegacyParse<T> = {
|
|
39
|
+
ok: true;
|
|
40
|
+
value: T;
|
|
41
|
+
} | {
|
|
42
|
+
ok: false;
|
|
43
|
+
problems: string[];
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* Translate one frozen prod014 `schedule.json` value.
|
|
47
|
+
* @param value The parsed JSON value.
|
|
48
|
+
* @returns The translated schedule, or explicit problems.
|
|
49
|
+
*/
|
|
50
|
+
export declare function readLegacySchedule(value: unknown): LegacyParse<LegacySchedule>;
|
|
51
|
+
/** One translated legacy manifest arm. */
|
|
52
|
+
export interface LegacyArm {
|
|
53
|
+
/** The arm id value. */
|
|
54
|
+
armId: string;
|
|
55
|
+
/** The label value. */
|
|
56
|
+
label: string;
|
|
57
|
+
}
|
|
58
|
+
/** One translated legacy manifest scenario. */
|
|
59
|
+
export interface LegacyScenario {
|
|
60
|
+
/** The case id value. */
|
|
61
|
+
caseId: string;
|
|
62
|
+
/** The kind value. */
|
|
63
|
+
kind: string;
|
|
64
|
+
/** The old mutation/preservation blocks are preserved verbatim. */
|
|
65
|
+
legacy: Record<string, unknown>;
|
|
66
|
+
}
|
|
67
|
+
/** A translated legacy campaign manifest. */
|
|
68
|
+
export interface LegacyManifest {
|
|
69
|
+
/** The kind value. */
|
|
70
|
+
kind: "prod014-manifest";
|
|
71
|
+
/** The legacy campaign id value. */
|
|
72
|
+
legacyCampaignId: string;
|
|
73
|
+
/** The question value. */
|
|
74
|
+
question: string | null;
|
|
75
|
+
/** The arms value. */
|
|
76
|
+
arms: LegacyArm[];
|
|
77
|
+
/** The scenarios value. */
|
|
78
|
+
scenarios: LegacyScenario[];
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Translate one frozen prod014 `manifest.json` value. Unknown fields remain
|
|
82
|
+
* in `legacy` so an old fact is never silently discarded.
|
|
83
|
+
* @param value The parsed JSON value.
|
|
84
|
+
* @returns The translated manifest, or explicit problems.
|
|
85
|
+
*/
|
|
86
|
+
export declare function readLegacyManifest(value: unknown): LegacyParse<LegacyManifest>;
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Explicit readers for the frozen prod014 layout.
|
|
3
|
+
*
|
|
4
|
+
* Historical scenario ids, stamps, pair ids and frozen schedule bytes are
|
|
5
|
+
* records, not templates. These adapters translate the old fields into the
|
|
6
|
+
* new vocabulary without renaming an id or rewriting the source file. A
|
|
7
|
+
* legacy campaign is never resumed by the new engine: the reader exists for
|
|
8
|
+
* comparison and provenance, and new work gets a fresh OTC root.
|
|
9
|
+
*/
|
|
10
|
+
import { isRecord } from "../contracts/validate.js";
|
|
11
|
+
/** A required non-blank string field. */
|
|
12
|
+
function requiredLegacyText(value, field, where, problems) {
|
|
13
|
+
if (typeof value === "string" && value !== "")
|
|
14
|
+
return value;
|
|
15
|
+
problems.push(`${where}.${field} is missing`);
|
|
16
|
+
return undefined;
|
|
17
|
+
}
|
|
18
|
+
/** A required integer field with a floor. */
|
|
19
|
+
function requiredLegacyInteger(value, field, floor, where, problems) {
|
|
20
|
+
if (typeof value === "number" &&
|
|
21
|
+
Number.isInteger(value) &&
|
|
22
|
+
value >= floor) {
|
|
23
|
+
return value;
|
|
24
|
+
}
|
|
25
|
+
problems.push(`${where}.${field} must be an integer >= ${floor}`);
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
/** Translate one frozen schedule slot. */
|
|
29
|
+
function legacySlotOf(entry, index) {
|
|
30
|
+
const where = `legacy schedule slot[${index}]`;
|
|
31
|
+
if (!isRecord(entry)) {
|
|
32
|
+
return { ok: false, problems: [`${where} is not an object`] };
|
|
33
|
+
}
|
|
34
|
+
const problems = [];
|
|
35
|
+
const slot = requiredLegacyInteger(entry.slot, "slot", 1, where, problems);
|
|
36
|
+
const repetition = requiredLegacyInteger(entry.repetition, "repetition", 0, where, problems);
|
|
37
|
+
const pairId = requiredLegacyText(entry.pairId, "pairId", where, problems);
|
|
38
|
+
const scenario = requiredLegacyText(entry.scenario, "scenario", where, problems);
|
|
39
|
+
const arm = requiredLegacyText(entry.arm, "arm", where, problems);
|
|
40
|
+
const stamp = requiredLegacyText(entry.stamp, "stamp", where, problems);
|
|
41
|
+
if (problems.length > 0 ||
|
|
42
|
+
slot === undefined ||
|
|
43
|
+
repetition === undefined ||
|
|
44
|
+
pairId === undefined ||
|
|
45
|
+
scenario === undefined ||
|
|
46
|
+
arm === undefined ||
|
|
47
|
+
stamp === undefined) {
|
|
48
|
+
return { ok: false, problems };
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
ok: true,
|
|
52
|
+
slot: {
|
|
53
|
+
slot: {
|
|
54
|
+
slotId: stamp,
|
|
55
|
+
caseId: scenario,
|
|
56
|
+
armId: arm,
|
|
57
|
+
repetitionId: `r${repetition}`,
|
|
58
|
+
},
|
|
59
|
+
legacySlot: slot,
|
|
60
|
+
pairId,
|
|
61
|
+
legacyScenario: scenario,
|
|
62
|
+
legacyStamp: stamp,
|
|
63
|
+
legacyArm: arm,
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** Translate slots, refusing a duplicate historical stamp. */
|
|
68
|
+
function uniqueLegacySlots(value) {
|
|
69
|
+
const slots = [];
|
|
70
|
+
const problems = [];
|
|
71
|
+
const stamps = new Set();
|
|
72
|
+
for (const [index, entry] of value.entries()) {
|
|
73
|
+
const parsed = legacySlotOf(entry, index);
|
|
74
|
+
if (!parsed.ok) {
|
|
75
|
+
problems.push(...parsed.problems);
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
if (stamps.has(parsed.slot.legacyStamp)) {
|
|
79
|
+
problems.push(`legacy schedule slot[${index}].stamp duplicates ${parsed.slot.legacyStamp}`);
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
stamps.add(parsed.slot.legacyStamp);
|
|
83
|
+
slots.push(parsed.slot);
|
|
84
|
+
}
|
|
85
|
+
return { slots, problems };
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Translate one frozen prod014 `schedule.json` value.
|
|
89
|
+
* @param value The parsed JSON value.
|
|
90
|
+
* @returns The translated schedule, or explicit problems.
|
|
91
|
+
*/
|
|
92
|
+
export function readLegacySchedule(value) {
|
|
93
|
+
if (!isRecord(value)) {
|
|
94
|
+
return { ok: false, problems: ["legacy schedule must be an object"] };
|
|
95
|
+
}
|
|
96
|
+
if (typeof value.campaign !== "string" || value.campaign === "") {
|
|
97
|
+
return { ok: false, problems: ["legacy schedule has no campaign id"] };
|
|
98
|
+
}
|
|
99
|
+
if (!Array.isArray(value.slots) || value.slots.length === 0) {
|
|
100
|
+
return { ok: false, problems: ["legacy schedule has no slots"] };
|
|
101
|
+
}
|
|
102
|
+
const { slots, problems } = uniqueLegacySlots(value.slots);
|
|
103
|
+
if (problems.length > 0)
|
|
104
|
+
return { ok: false, problems };
|
|
105
|
+
return {
|
|
106
|
+
ok: true,
|
|
107
|
+
value: {
|
|
108
|
+
kind: "prod014-schedule",
|
|
109
|
+
legacyCampaignId: value.campaign,
|
|
110
|
+
generator: typeof value.generator === "string" ? value.generator : null,
|
|
111
|
+
slots,
|
|
112
|
+
},
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/** Translate the manifest's arms. */
|
|
116
|
+
function legacyArmsOf(value) {
|
|
117
|
+
if (value === undefined)
|
|
118
|
+
return { ok: true, value: [] };
|
|
119
|
+
if (!Array.isArray(value)) {
|
|
120
|
+
return {
|
|
121
|
+
ok: false,
|
|
122
|
+
problems: ["legacy manifest arms must be an array"],
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
const arms = [];
|
|
126
|
+
for (const [index, entry] of value.entries()) {
|
|
127
|
+
if (!isRecord(entry) || typeof entry.id !== "string") {
|
|
128
|
+
return {
|
|
129
|
+
ok: false,
|
|
130
|
+
problems: [`legacy manifest arm[${index}] is malformed`],
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
arms.push({
|
|
134
|
+
armId: entry.id,
|
|
135
|
+
label: typeof entry.label === "string" ? entry.label : "",
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
return { ok: true, value: arms };
|
|
139
|
+
}
|
|
140
|
+
/** Translate the manifest's scenarios, preserving unknown blocks. */
|
|
141
|
+
function legacyScenariosOf(value) {
|
|
142
|
+
if (value === undefined)
|
|
143
|
+
return { ok: true, value: [] };
|
|
144
|
+
if (!Array.isArray(value)) {
|
|
145
|
+
return {
|
|
146
|
+
ok: false,
|
|
147
|
+
problems: ["legacy manifest scenarios must be an array"],
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
const scenarios = [];
|
|
151
|
+
for (const [index, entry] of value.entries()) {
|
|
152
|
+
if (!isRecord(entry) ||
|
|
153
|
+
typeof entry.id !== "string" ||
|
|
154
|
+
typeof entry.kind !== "string") {
|
|
155
|
+
return {
|
|
156
|
+
ok: false,
|
|
157
|
+
problems: [`legacy manifest scenario[${index}] is malformed`],
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
const { id, kind, ...legacy } = entry;
|
|
161
|
+
scenarios.push({ caseId: id, kind, legacy });
|
|
162
|
+
}
|
|
163
|
+
return { ok: true, value: scenarios };
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Translate one frozen prod014 `manifest.json` value. Unknown fields remain
|
|
167
|
+
* in `legacy` so an old fact is never silently discarded.
|
|
168
|
+
* @param value The parsed JSON value.
|
|
169
|
+
* @returns The translated manifest, or explicit problems.
|
|
170
|
+
*/
|
|
171
|
+
export function readLegacyManifest(value) {
|
|
172
|
+
if (!isRecord(value)) {
|
|
173
|
+
return { ok: false, problems: ["legacy manifest must be an object"] };
|
|
174
|
+
}
|
|
175
|
+
if (value.schemaVersion !== 1) {
|
|
176
|
+
return {
|
|
177
|
+
ok: false,
|
|
178
|
+
problems: ["legacy manifest schemaVersion must be 1"],
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
if (!isRecord(value.campaign)) {
|
|
182
|
+
return {
|
|
183
|
+
ok: false,
|
|
184
|
+
problems: ["legacy manifest has no campaign block"],
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
const campaignId = value.campaign.id;
|
|
188
|
+
if (typeof campaignId !== "string" || campaignId === "") {
|
|
189
|
+
return { ok: false, problems: ["legacy manifest has no campaign id"] };
|
|
190
|
+
}
|
|
191
|
+
const arms = legacyArmsOf(value.arms);
|
|
192
|
+
if (!arms.ok)
|
|
193
|
+
return arms;
|
|
194
|
+
const scenarios = legacyScenariosOf(value.scenarios);
|
|
195
|
+
if (!scenarios.ok)
|
|
196
|
+
return scenarios;
|
|
197
|
+
return {
|
|
198
|
+
ok: true,
|
|
199
|
+
value: {
|
|
200
|
+
kind: "prod014-manifest",
|
|
201
|
+
legacyCampaignId: campaignId,
|
|
202
|
+
question: typeof value.campaign.question === "string"
|
|
203
|
+
? value.campaign.question
|
|
204
|
+
: null,
|
|
205
|
+
arms: arms.value,
|
|
206
|
+
scenarios: scenarios.value,
|
|
207
|
+
},
|
|
208
|
+
};
|
|
209
|
+
}
|