@skill-harness/core 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/types.d.ts +30 -0
- package/dist/adjudication.d.ts +9 -1
- package/dist/adjudication.js +41 -33
- package/dist/arms.d.ts +51 -0
- package/dist/arms.js +255 -0
- package/dist/capture-trace-types.d.ts +2 -0
- package/dist/command-cost.d.ts +9 -0
- package/dist/command-cost.js +23 -0
- package/dist/comparison.d.ts +1 -0
- package/dist/comparison.js +9 -3
- package/dist/execution-trace.js +9 -1
- package/dist/grade.d.ts +2 -0
- package/dist/grade.js +2 -1
- package/dist/index.d.ts +14 -0
- package/dist/index.js +14 -0
- package/dist/journal.d.ts +2 -1
- package/dist/judge-agreement.d.ts +19 -0
- package/dist/judge-agreement.js +31 -0
- package/dist/judge-policy.js +5 -3
- package/dist/lift.js +1 -1
- package/dist/lint.js +1 -1
- package/dist/prompt-normalization.d.ts +6 -0
- package/dist/prompt-normalization.js +14 -0
- package/dist/provider-failure.d.ts +56 -0
- package/dist/provider-failure.js +98 -0
- package/dist/qualification-capture.d.ts +44 -0
- package/dist/qualification-capture.js +163 -0
- package/dist/qualification-config.d.ts +233 -0
- package/dist/qualification-config.js +694 -0
- package/dist/qualification-lock.d.ts +15 -0
- package/dist/qualification-lock.js +87 -0
- package/dist/qualification-oauth-directory.d.ts +79 -0
- package/dist/qualification-oauth-directory.js +444 -0
- package/dist/qualification-panel-store.d.ts +14 -0
- package/dist/qualification-panel-store.js +188 -0
- package/dist/qualification-panels.d.ts +99 -0
- package/dist/qualification-panels.js +91 -0
- package/dist/qualification-process.d.ts +13 -0
- package/dist/qualification-process.js +188 -0
- package/dist/qualification-runner.d.ts +203 -0
- package/dist/qualification-runner.js +2037 -0
- package/dist/qualification-store.d.ts +187 -0
- package/dist/qualification-store.js +833 -0
- package/dist/qualification-validation.d.ts +12 -0
- package/dist/qualification-validation.js +8 -0
- package/dist/regate.js +49 -10
- package/dist/regrade.d.ts +3 -1
- package/dist/regrade.js +62 -6
- package/dist/reps.d.ts +4 -2
- package/dist/reps.js +13 -4
- package/dist/rescore.js +9 -1
- package/dist/results.d.ts +99 -7
- package/dist/results.js +228 -10
- package/dist/run.d.ts +14 -0
- package/dist/run.js +260 -57
- package/dist/score.d.ts +4 -3
- package/dist/score.js +13 -4
- package/dist/screen.d.ts +31 -0
- package/dist/screen.js +105 -0
- package/dist/seeded.d.ts +12 -1
- package/dist/seeded.js +9 -2
- package/dist/sources.d.ts +7 -1
- package/dist/sources.js +13 -5
- package/dist/spec.d.ts +1 -1
- package/dist/spec.js +1 -1
- package/dist/stability.js +1 -1
- package/dist/trajectory-gates.d.ts +14 -2
- package/dist/trajectory-gates.js +26 -6
- package/dist/vote-panel.d.ts +33 -0
- package/dist/vote-panel.js +41 -0
- package/dist/workspace.js +34 -3
- package/package.json +3 -3
- package/schemas/qualification-cell-result-v1.schema.json +32 -0
- package/schemas/qualification-config-v1.schema.json +244 -0
- package/schemas/qualification-invocation-request-v1.schema.json +59 -0
- package/schemas/qualification-judge-panel-result-v1.schema.json +42 -0
- package/schemas/results-v2.schema.json +19 -21
- package/schemas/results-v3.schema.json +654 -0
- package/schemas/specification-v1.schema.json +1 -1
- package/schemas/trajectory-event-v1.schema.json +20 -1
package/dist/execution-trace.js
CHANGED
|
@@ -157,12 +157,19 @@ export function parseTrace(lines, meta) {
|
|
|
157
157
|
}
|
|
158
158
|
}
|
|
159
159
|
const toolCalls = [...calls.values()].sort((a, b) => a.issueIndex - b.issueIndex);
|
|
160
|
+
const subscription = meta.subject.provider === "openai-codex" || meta.subject.provider === "claude-code";
|
|
161
|
+
const costSource = subscription
|
|
162
|
+
? "subscription"
|
|
163
|
+
: cost !== null
|
|
164
|
+
? "provider-reported"
|
|
165
|
+
: "unreported";
|
|
160
166
|
const metrics = {
|
|
161
167
|
input_tokens: inputTokens,
|
|
162
168
|
output_tokens: outputTokens,
|
|
163
169
|
cache_read_tokens: cacheReadTokens,
|
|
164
170
|
cache_write_tokens: cacheWriteTokens,
|
|
165
171
|
cost_usd: cost ?? 0,
|
|
172
|
+
cost_source: costSource,
|
|
166
173
|
tool_calls: toolCalls.length,
|
|
167
174
|
delegated_children: toolCalls
|
|
168
175
|
.filter((call) => call.name === "Agent")
|
|
@@ -348,12 +355,13 @@ export function mergeTraces(traces) {
|
|
|
348
355
|
cache_read_tokens: sum.cache_read_tokens + trace.metrics.cache_read_tokens,
|
|
349
356
|
cache_write_tokens: sum.cache_write_tokens + trace.metrics.cache_write_tokens,
|
|
350
357
|
cost_usd: sum.cost_usd + trace.metrics.cost_usd,
|
|
358
|
+
cost_source: sum.cost_source === trace.metrics.cost_source ? sum.cost_source : "unreported",
|
|
351
359
|
tool_calls: sum.tool_calls + trace.metrics.tool_calls,
|
|
352
360
|
delegated_children: sum.delegated_children + trace.metrics.delegated_children,
|
|
353
361
|
max_concurrency: Math.max(sum.max_concurrency, trace.metrics.max_concurrency),
|
|
354
362
|
}), {
|
|
355
363
|
input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0,
|
|
356
|
-
cost_usd: 0, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
|
|
364
|
+
cost_usd: 0, cost_source: traces[0].metrics.cost_source, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
|
|
357
365
|
})
|
|
358
366
|
: undefined;
|
|
359
367
|
const last = traces[traces.length - 1];
|
package/dist/grade.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Scenario } from "./spec.js";
|
|
2
2
|
import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
|
|
3
3
|
import type { Verdict } from "./score.js";
|
|
4
|
+
import { type CriterionVote } from "./results.js";
|
|
4
5
|
export interface JudgePromptInput {
|
|
5
6
|
skill: string;
|
|
6
7
|
persona: string;
|
|
@@ -36,6 +37,7 @@ export declare function parseVerdict(out: string): ParsedVerdict;
|
|
|
36
37
|
export declare function judgeResemblesSubject(judge: ModelRef, subject: ModelRef): boolean;
|
|
37
38
|
export interface GradeResult extends ParsedVerdict {
|
|
38
39
|
raw: string;
|
|
40
|
+
criteria: CriterionVote[];
|
|
39
41
|
/** Judge misfire: the overall verdict disagrees with AND(per-item grades). Recorded, never auto-passed; blocks SHIP until re-judged or overridden. */
|
|
40
42
|
suspect: boolean;
|
|
41
43
|
}
|
package/dist/grade.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createWorkspace } from "./workspace.js";
|
|
2
|
+
import { parseCriterionVotes } from "./results.js";
|
|
2
3
|
/** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
|
|
3
4
|
export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
|
|
4
5
|
/**
|
|
@@ -133,7 +134,7 @@ export async function gradeTranscript(adapter, judge, prompt, cwd) {
|
|
|
133
134
|
parsed.reason = `judge unparseable: ${snippet}`;
|
|
134
135
|
}
|
|
135
136
|
const suspect = detectMisfire(raw, parsed.verdict);
|
|
136
|
-
return { ...parsed, raw, suspect };
|
|
137
|
+
return { ...parsed, raw, suspect, criteria: parseCriterionVotes(raw) };
|
|
137
138
|
}
|
|
138
139
|
/**
|
|
139
140
|
* Grade a transcript in a fresh, isolated, throwaway workspace — never the
|
package/dist/index.d.ts
CHANGED
|
@@ -23,8 +23,12 @@ export * from "./util/env.js";
|
|
|
23
23
|
export * from "./scaffold.js";
|
|
24
24
|
export * from "./version.js";
|
|
25
25
|
export * from "./defaults.js";
|
|
26
|
+
export * from "./command-cost.js";
|
|
26
27
|
export * from "./judge-policy.js";
|
|
28
|
+
export * from "./judge-agreement.js";
|
|
27
29
|
export * from "./regate.js";
|
|
30
|
+
export * from "./screen.js";
|
|
31
|
+
export * from "./prompt-normalization.js";
|
|
28
32
|
export * from "./downgrade.js";
|
|
29
33
|
export * from "./canary.js";
|
|
30
34
|
export * from "./stability.js";
|
|
@@ -37,5 +41,15 @@ export * from "./trajectory-gates.js";
|
|
|
37
41
|
export * from "./instruction-coverage.js";
|
|
38
42
|
export * from "./affected.js";
|
|
39
43
|
export * from "./adjudication.js";
|
|
44
|
+
export * from "./vote-panel.js";
|
|
45
|
+
export * from "./qualification-panels.js";
|
|
46
|
+
export * from "./qualification-panel-store.js";
|
|
47
|
+
export * from "./qualification-validation.js";
|
|
40
48
|
export * from "./comparison.js";
|
|
41
49
|
export * from "./sandbox.js";
|
|
50
|
+
export * from "./provider-failure.js";
|
|
51
|
+
export * from "./arms.js";
|
|
52
|
+
export * from "./qualification-config.js";
|
|
53
|
+
export * from "./qualification-oauth-directory.js";
|
|
54
|
+
export * from "./qualification-store.js";
|
|
55
|
+
export * from "./qualification-runner.js";
|
package/dist/index.js
CHANGED
|
@@ -23,8 +23,12 @@ export * from "./util/env.js";
|
|
|
23
23
|
export * from "./scaffold.js";
|
|
24
24
|
export * from "./version.js";
|
|
25
25
|
export * from "./defaults.js";
|
|
26
|
+
export * from "./command-cost.js";
|
|
26
27
|
export * from "./judge-policy.js";
|
|
28
|
+
export * from "./judge-agreement.js";
|
|
27
29
|
export * from "./regate.js";
|
|
30
|
+
export * from "./screen.js";
|
|
31
|
+
export * from "./prompt-normalization.js";
|
|
28
32
|
export * from "./downgrade.js";
|
|
29
33
|
export * from "./canary.js";
|
|
30
34
|
export * from "./stability.js";
|
|
@@ -37,6 +41,16 @@ export * from "./trajectory-gates.js";
|
|
|
37
41
|
export * from "./instruction-coverage.js";
|
|
38
42
|
export * from "./affected.js";
|
|
39
43
|
export * from "./adjudication.js";
|
|
44
|
+
export * from "./vote-panel.js";
|
|
45
|
+
export * from "./qualification-panels.js";
|
|
46
|
+
export * from "./qualification-panel-store.js";
|
|
47
|
+
export * from "./qualification-validation.js";
|
|
40
48
|
export * from "./comparison.js";
|
|
41
49
|
export * from "./sandbox.js";
|
|
50
|
+
export * from "./provider-failure.js";
|
|
51
|
+
export * from "./arms.js";
|
|
52
|
+
export * from "./qualification-config.js";
|
|
53
|
+
export * from "./qualification-oauth-directory.js";
|
|
54
|
+
export * from "./qualification-store.js";
|
|
55
|
+
export * from "./qualification-runner.js";
|
|
42
56
|
//# sourceMappingURL=index.js.map
|
package/dist/journal.d.ts
CHANGED
|
@@ -90,6 +90,7 @@ export type JournalEvent = {
|
|
|
90
90
|
ts: string;
|
|
91
91
|
id: string;
|
|
92
92
|
attempt: number;
|
|
93
|
+
reason?: string;
|
|
93
94
|
rep?: number;
|
|
94
95
|
} | {
|
|
95
96
|
event: "rescore";
|
|
@@ -102,7 +103,7 @@ export type JournalEvent = {
|
|
|
102
103
|
}
|
|
103
104
|
/**
|
|
104
105
|
* A regate: needle gates re-evaluated against the saved staged diffs. `judge_calls`
|
|
105
|
-
* is on the record because regate
|
|
106
|
+
* is on the record because regate may spend only on the reps whose
|
|
106
107
|
* gate verdict flipped — a claim the journal should be able to settle. `skipped`
|
|
107
108
|
* names scenarios it could not regate (vitest/post_test, or missing diff artifacts).
|
|
108
109
|
*/
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { ResultsFile } from "./results.js";
|
|
2
|
+
import type { Verdict } from "./score.js";
|
|
3
|
+
export interface JudgeAgreementCell {
|
|
4
|
+
scenario: string;
|
|
5
|
+
status: "agree" | "disagree" | "error";
|
|
6
|
+
judges: string[];
|
|
7
|
+
verdicts: Verdict[];
|
|
8
|
+
detail: string;
|
|
9
|
+
}
|
|
10
|
+
export interface JudgeAgreementReport {
|
|
11
|
+
cells: JudgeAgreementCell[];
|
|
12
|
+
agree: number;
|
|
13
|
+
disagree: number;
|
|
14
|
+
error: number;
|
|
15
|
+
comparable: number;
|
|
16
|
+
rate: number | null;
|
|
17
|
+
}
|
|
18
|
+
/** Pure/offline projection of judgments already persisted in results.yaml. */
|
|
19
|
+
export declare function judgeAgreement(results: ResultsFile): JudgeAgreementReport;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** Pure/offline projection of judgments already persisted in results.yaml. */
|
|
2
|
+
export function judgeAgreement(results) {
|
|
3
|
+
const cells = results.scenarios.map((scenario) => {
|
|
4
|
+
const judgments = scenario.judge_history ?? scenario.adjudication?.judgments ?? [];
|
|
5
|
+
const distinct = new Map();
|
|
6
|
+
for (const judgment of judgments) {
|
|
7
|
+
const key = `${judgment.judge.provider}:${judgment.judge.model}`;
|
|
8
|
+
distinct.delete(key); // insertion order now reflects each judge's latest persisted vote
|
|
9
|
+
distinct.set(key, judgment);
|
|
10
|
+
}
|
|
11
|
+
const pair = [...distinct.entries()].slice(-2);
|
|
12
|
+
if (pair.length < 2 || pair.some(([, judgment]) => judgment.suspect || judgment.verdict === "ERROR" || judgment.verdict === "NOT-MEASURED" || judgment.verdict === "JUDGE-AMBIGUOUS")) {
|
|
13
|
+
return {
|
|
14
|
+
scenario: scenario.id, status: "error", judges: pair.map(([judge]) => judge),
|
|
15
|
+
verdicts: pair.map(([, judgment]) => judgment.verdict),
|
|
16
|
+
detail: pair.length < 2 ? "fewer than two distinct persisted judges" : "one or more judgments is not a clean vote",
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
const status = pair[0][1].verdict === pair[1][1].verdict ? "agree" : "disagree";
|
|
20
|
+
return {
|
|
21
|
+
scenario: scenario.id, status, judges: pair.map(([judge]) => judge),
|
|
22
|
+
verdicts: pair.map(([, judgment]) => judgment.verdict), detail: `${pair[0][1].verdict} vs ${pair[1][1].verdict}`,
|
|
23
|
+
};
|
|
24
|
+
});
|
|
25
|
+
const agree = cells.filter((cell) => cell.status === "agree").length;
|
|
26
|
+
const disagree = cells.filter((cell) => cell.status === "disagree").length;
|
|
27
|
+
const error = cells.filter((cell) => cell.status === "error").length;
|
|
28
|
+
const comparable = agree + disagree;
|
|
29
|
+
return { cells, agree, disagree, error, comparable, rate: comparable ? agree / comparable : null };
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=judge-agreement.js.map
|
package/dist/judge-policy.js
CHANGED
|
@@ -4,15 +4,17 @@ import { BAKED_DEFAULT_JUDGE } from "./defaults.js";
|
|
|
4
4
|
* Judge providers that cannot bill a per-token API.
|
|
5
5
|
*
|
|
6
6
|
* `claude-code` shells out to the `claude` CLI, authenticated with the user's
|
|
7
|
-
* Claude subscription (OAuth). `
|
|
8
|
-
*
|
|
7
|
+
* Claude subscription (OAuth). `openai-codex` stays in Pi and authenticates with
|
|
8
|
+
* the user's ChatGPT subscription (OAuth). `ollama`/`lmstudio`/`llamacpp`/`local`
|
|
9
|
+
* are local runtimes. Everything else — including direct `openai` — is assumed
|
|
10
|
+
* to charge.
|
|
9
11
|
*
|
|
10
12
|
* An **allow-list**, deliberately, not a deny-list of known-paid providers: a
|
|
11
13
|
* provider nobody has classified yet should be treated as able to bill, because
|
|
12
14
|
* being wrong in that direction produces a surprise invoice while being wrong in
|
|
13
15
|
* this direction produces one extra flag.
|
|
14
16
|
*/
|
|
15
|
-
const FREE_JUDGE_PROVIDERS = new Set(["claude-code", "ollama", "lmstudio", "llamacpp", "local"]);
|
|
17
|
+
const FREE_JUDGE_PROVIDERS = new Set(["claude-code", "openai-codex", "ollama", "lmstudio", "llamacpp", "local"]);
|
|
16
18
|
/** Whether judging with this ref can charge a per-token API. */
|
|
17
19
|
export function isMeteredJudge(judge) {
|
|
18
20
|
return !FREE_JUDGE_PROVIDERS.has(judge.provider);
|
package/dist/lift.js
CHANGED
|
@@ -29,7 +29,7 @@ function conclusive(verdict, suspect) {
|
|
|
29
29
|
// misfire are judge failures. Treating any of them as a FAIL would make
|
|
30
30
|
// "red FAIL -> green PASS" fire on infrastructure noise, and lift would
|
|
31
31
|
// measure flakiness while looking like skill value.
|
|
32
|
-
return !suspect && verdict !== "ERROR" && verdict !== "JUDGE-AMBIGUOUS";
|
|
32
|
+
return !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS";
|
|
33
33
|
}
|
|
34
34
|
function classify(red, green) {
|
|
35
35
|
if (!conclusive(red.verdict, red.suspect) || !conclusive(green.verdict, green.suspect))
|
package/dist/lint.js
CHANGED
|
@@ -279,7 +279,7 @@ export function lintSkill(skillDir) {
|
|
|
279
279
|
else if (current !== recorded) {
|
|
280
280
|
// The remedy is per key kind, and it is the whole point of the split: a
|
|
281
281
|
// rubric edit is re-gradeable from saved transcripts, a policy edit is a
|
|
282
|
-
// free rescore, a
|
|
282
|
+
// free rescore, while a gate edit avoids a subject re-run via regate. Only stimulus drift costs
|
|
283
283
|
// model spend. One message saying "re-run" for all four is what made
|
|
284
284
|
// correcting a known-bad rubric expensive enough to skip.
|
|
285
285
|
findings.push({ skill, scenario, code: "stale", message: `${what} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; ${remedyForKey(key)}` });
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export declare const PROMPT_NORMALIZATION_RULE: "cwd-line-v1";
|
|
2
|
+
export declare const PROMPT_NORMALIZATION_PATTERN = "^(Current working directory:)[^\\r\\n]*(\\r?)$";
|
|
3
|
+
export declare const PROMPT_NORMALIZATION_FLAGS = "gm";
|
|
4
|
+
export declare const PROMPT_NORMALIZATION_REPLACEMENT = "$1<normalized>$2";
|
|
5
|
+
export declare const PROMPT_NORMALIZATION_SOURCE_KEY = "observation:prompt-normalization";
|
|
6
|
+
export declare const PROMPT_NORMALIZATION_SOURCE_DIGEST: string;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export const PROMPT_NORMALIZATION_RULE = "cwd-line-v1";
|
|
3
|
+
export const PROMPT_NORMALIZATION_PATTERN = "^(Current working directory:)[^\\r\\n]*(\\r?)$";
|
|
4
|
+
export const PROMPT_NORMALIZATION_FLAGS = "gm";
|
|
5
|
+
export const PROMPT_NORMALIZATION_REPLACEMENT = "$1<normalized>$2";
|
|
6
|
+
export const PROMPT_NORMALIZATION_SOURCE_KEY = "observation:prompt-normalization";
|
|
7
|
+
export const PROMPT_NORMALIZATION_SOURCE_DIGEST = createHash("sha256").update(JSON.stringify([
|
|
8
|
+
"prompt-normalization-registry",
|
|
9
|
+
PROMPT_NORMALIZATION_RULE,
|
|
10
|
+
PROMPT_NORMALIZATION_PATTERN,
|
|
11
|
+
PROMPT_NORMALIZATION_FLAGS,
|
|
12
|
+
PROMPT_NORMALIZATION_REPLACEMENT,
|
|
13
|
+
])).digest("hex");
|
|
14
|
+
//# sourceMappingURL=prompt-normalization.js.map
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider-side failure detection.
|
|
3
|
+
*
|
|
4
|
+
* A provider outage is INFRASTRUCTURE, never a model verdict. Measured 2026-08-22:
|
|
5
|
+
* an invalidated `openai-codex` OAuth token makes pi report
|
|
6
|
+
* `stopReason: "error"` with a `provider_transport_failure` diagnostic and **exit 0**
|
|
7
|
+
* in `--mode json`, and exit 1 with `Encountered invalidated oauth token for user,
|
|
8
|
+
* failing request` on stderr in text mode. Neither path was classified, so a wave
|
|
9
|
+
* against a dead token produced model FAILs shaped exactly like findings.
|
|
10
|
+
*
|
|
11
|
+
* Two entry points because the two run paths surface it differently, and the
|
|
12
|
+
* structured one is the WEAKER signal — it hides the real message behind a generic
|
|
13
|
+
* transport error and exits 0. Do not assume `--structured` improves diagnosis.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Written into a transcript by the adapter when pi failed provider-side, so the
|
|
17
|
+
* text path can carry a machine-readable signal through a `string` return without
|
|
18
|
+
* widening `HarnessAdapter.run`. Read back by `providerFailureFromTranscript`.
|
|
19
|
+
*
|
|
20
|
+
* The marker must not be forgeable by a model that types the same words into its
|
|
21
|
+
* answer — and a verdict is what hangs on it, since a forged marker converts a
|
|
22
|
+
* would-be FAIL into ERROR and (via `judgeOneRep`'s short-circuit) permanently
|
|
23
|
+
* suppresses the judge on every later re-grade. Line-anchoring alone does not do
|
|
24
|
+
* that: nothing stops a model emitting a newline and then the marker.
|
|
25
|
+
*
|
|
26
|
+
* So position, not spelling, is the guarantee. The marker is written ONLY into the
|
|
27
|
+
* transcript PREAMBLE — everything before the first `>>> ` turn header — which the
|
|
28
|
+
* adapter composes before any model output exists. There is no ordering by which a
|
|
29
|
+
* model's text can land there. See `withProviderFailure` (write) and
|
|
30
|
+
* `providerFailureFromTranscript` (read); both live here so the two halves of the
|
|
31
|
+
* convention cannot drift apart.
|
|
32
|
+
*/
|
|
33
|
+
export declare const PROVIDER_FAILURE_MARKER = "[skill-harness] provider failure:";
|
|
34
|
+
/**
|
|
35
|
+
* Prefix `transcript` with the provider-failure marker, or return it unchanged.
|
|
36
|
+
*
|
|
37
|
+
* The one supported way to write the marker: it must precede the first turn header
|
|
38
|
+
* to be readable back, and putting the rule next to the reader keeps a future
|
|
39
|
+
* adapter from re-introducing the forgeable placement.
|
|
40
|
+
*/
|
|
41
|
+
export declare function withProviderFailure(transcript: string, failure: string | null): string;
|
|
42
|
+
/**
|
|
43
|
+
* A provider failure named by one `pi --mode json` line, or null.
|
|
44
|
+
*
|
|
45
|
+
* Fail-closed on the diagnostic, fail-open on the parse: an unreadable line is not
|
|
46
|
+
* evidence of a failure, and throwing here would abort a wave over one bad line.
|
|
47
|
+
*/
|
|
48
|
+
export declare function providerFailureFromJsonLine(line: string): string | null;
|
|
49
|
+
/**
|
|
50
|
+
* The failure the adapter recorded in a transcript's preamble, or null.
|
|
51
|
+
*
|
|
52
|
+
* Scanning STOPS at the first turn header. Everything from there on is model
|
|
53
|
+
* output or interleaved with it, so a marker found there is not evidence the
|
|
54
|
+
* provider failed — it is evidence the model typed the words.
|
|
55
|
+
*/
|
|
56
|
+
export declare function providerFailureFromTranscript(transcript: string): string | null;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider-side failure detection.
|
|
3
|
+
*
|
|
4
|
+
* A provider outage is INFRASTRUCTURE, never a model verdict. Measured 2026-08-22:
|
|
5
|
+
* an invalidated `openai-codex` OAuth token makes pi report
|
|
6
|
+
* `stopReason: "error"` with a `provider_transport_failure` diagnostic and **exit 0**
|
|
7
|
+
* in `--mode json`, and exit 1 with `Encountered invalidated oauth token for user,
|
|
8
|
+
* failing request` on stderr in text mode. Neither path was classified, so a wave
|
|
9
|
+
* against a dead token produced model FAILs shaped exactly like findings.
|
|
10
|
+
*
|
|
11
|
+
* Two entry points because the two run paths surface it differently, and the
|
|
12
|
+
* structured one is the WEAKER signal — it hides the real message behind a generic
|
|
13
|
+
* transport error and exits 0. Do not assume `--structured` improves diagnosis.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Written into a transcript by the adapter when pi failed provider-side, so the
|
|
17
|
+
* text path can carry a machine-readable signal through a `string` return without
|
|
18
|
+
* widening `HarnessAdapter.run`. Read back by `providerFailureFromTranscript`.
|
|
19
|
+
*
|
|
20
|
+
* The marker must not be forgeable by a model that types the same words into its
|
|
21
|
+
* answer — and a verdict is what hangs on it, since a forged marker converts a
|
|
22
|
+
* would-be FAIL into ERROR and (via `judgeOneRep`'s short-circuit) permanently
|
|
23
|
+
* suppresses the judge on every later re-grade. Line-anchoring alone does not do
|
|
24
|
+
* that: nothing stops a model emitting a newline and then the marker.
|
|
25
|
+
*
|
|
26
|
+
* So position, not spelling, is the guarantee. The marker is written ONLY into the
|
|
27
|
+
* transcript PREAMBLE — everything before the first `>>> ` turn header — which the
|
|
28
|
+
* adapter composes before any model output exists. There is no ordering by which a
|
|
29
|
+
* model's text can land there. See `withProviderFailure` (write) and
|
|
30
|
+
* `providerFailureFromTranscript` (read); both live here so the two halves of the
|
|
31
|
+
* convention cannot drift apart.
|
|
32
|
+
*/
|
|
33
|
+
export const PROVIDER_FAILURE_MARKER = "[skill-harness] provider failure:";
|
|
34
|
+
/**
|
|
35
|
+
* The adapters' shared turn-header prefix (`>>> USER:` / `>>> USER (turn n/m):`),
|
|
36
|
+
* which is where the preamble ends and model-influenced content begins.
|
|
37
|
+
*/
|
|
38
|
+
const TURN_HEADER_PREFIX = ">>> ";
|
|
39
|
+
/**
|
|
40
|
+
* Prefix `transcript` with the provider-failure marker, or return it unchanged.
|
|
41
|
+
*
|
|
42
|
+
* The one supported way to write the marker: it must precede the first turn header
|
|
43
|
+
* to be readable back, and putting the rule next to the reader keeps a future
|
|
44
|
+
* adapter from re-introducing the forgeable placement.
|
|
45
|
+
*/
|
|
46
|
+
export function withProviderFailure(transcript, failure) {
|
|
47
|
+
return failure ? `${PROVIDER_FAILURE_MARKER} ${failure}\n\n${transcript}` : transcript;
|
|
48
|
+
}
|
|
49
|
+
/** Diagnostic types that mean the provider never ran the request. */
|
|
50
|
+
const FAILURE_DIAGNOSTICS = new Set(["provider_transport_failure"]);
|
|
51
|
+
/**
|
|
52
|
+
* A provider failure named by one `pi --mode json` line, or null.
|
|
53
|
+
*
|
|
54
|
+
* Fail-closed on the diagnostic, fail-open on the parse: an unreadable line is not
|
|
55
|
+
* evidence of a failure, and throwing here would abort a wave over one bad line.
|
|
56
|
+
*/
|
|
57
|
+
export function providerFailureFromJsonLine(line) {
|
|
58
|
+
let parsed;
|
|
59
|
+
try {
|
|
60
|
+
parsed = JSON.parse(line);
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
const message = parsed?.message;
|
|
66
|
+
if (!message || typeof message !== "object")
|
|
67
|
+
return null;
|
|
68
|
+
const diagnostics = message.diagnostics;
|
|
69
|
+
if (!Array.isArray(diagnostics))
|
|
70
|
+
return null;
|
|
71
|
+
for (const raw of diagnostics) {
|
|
72
|
+
if (typeof raw?.type !== "string" || !FAILURE_DIAGNOSTICS.has(raw.type))
|
|
73
|
+
continue;
|
|
74
|
+
const provider = typeof message.provider === "string"
|
|
75
|
+
? message.provider
|
|
76
|
+
: "unknown provider";
|
|
77
|
+
const detail = typeof raw.error?.message === "string" ? raw.error.message : raw.type;
|
|
78
|
+
return `${provider}: ${detail}`;
|
|
79
|
+
}
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* The failure the adapter recorded in a transcript's preamble, or null.
|
|
84
|
+
*
|
|
85
|
+
* Scanning STOPS at the first turn header. Everything from there on is model
|
|
86
|
+
* output or interleaved with it, so a marker found there is not evidence the
|
|
87
|
+
* provider failed — it is evidence the model typed the words.
|
|
88
|
+
*/
|
|
89
|
+
export function providerFailureFromTranscript(transcript) {
|
|
90
|
+
for (const line of transcript.split("\n")) {
|
|
91
|
+
if (line.startsWith(TURN_HEADER_PREFIX))
|
|
92
|
+
return null;
|
|
93
|
+
if (line.startsWith(PROVIDER_FAILURE_MARKER))
|
|
94
|
+
return line.slice(PROVIDER_FAILURE_MARKER.length).trim();
|
|
95
|
+
}
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=provider-failure.js.map
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export interface QualificationOutputReceipt {
|
|
2
|
+
path: string;
|
|
3
|
+
total_bytes: number;
|
|
4
|
+
captured_bytes: number;
|
|
5
|
+
truncated: boolean;
|
|
6
|
+
sha256: string;
|
|
7
|
+
}
|
|
8
|
+
export interface QualificationBoundedCapture {
|
|
9
|
+
path: string;
|
|
10
|
+
total_bytes: number;
|
|
11
|
+
captured_bytes: number;
|
|
12
|
+
truncated: boolean;
|
|
13
|
+
write(chunk: Buffer): void;
|
|
14
|
+
close(): void;
|
|
15
|
+
}
|
|
16
|
+
export interface QualificationArtifactAttestation {
|
|
17
|
+
ok: boolean;
|
|
18
|
+
actual: {
|
|
19
|
+
provider: string;
|
|
20
|
+
model: string;
|
|
21
|
+
} | null;
|
|
22
|
+
fallback: boolean;
|
|
23
|
+
refused: boolean;
|
|
24
|
+
error: string | null;
|
|
25
|
+
}
|
|
26
|
+
export declare function redactQualificationOutput(text: string, secretValues?: readonly string[]): string;
|
|
27
|
+
export declare function openQualificationBoundedCapture(path: string, limit: number, redact: (text: string) => string): QualificationBoundedCapture;
|
|
28
|
+
export declare function qualificationOutputReceipt(spoolRoot: string, path: string, capture: QualificationBoundedCapture): QualificationOutputReceipt;
|
|
29
|
+
export declare function verifyQualificationOutputReceipt(spoolRoot: string, receipt: QualificationOutputReceipt): void;
|
|
30
|
+
export declare function attestQualificationPiJsonl(text: string, requested: {
|
|
31
|
+
provider: string;
|
|
32
|
+
model: string;
|
|
33
|
+
}): QualificationArtifactAttestation;
|
|
34
|
+
export declare function spawnQualificationCapture(command: string, argv: string[], options: {
|
|
35
|
+
cwd: string;
|
|
36
|
+
env: NodeJS.ProcessEnv;
|
|
37
|
+
timeout_ms: number;
|
|
38
|
+
output_limit_bytes: number;
|
|
39
|
+
}): Promise<{
|
|
40
|
+
stdout: string;
|
|
41
|
+
stderr: string;
|
|
42
|
+
code: number | null;
|
|
43
|
+
timed_out: boolean;
|
|
44
|
+
}>;
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { closeSync, constants, fstatSync, fsyncSync, openSync, readFileSync, writeSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { StringDecoder } from "node:string_decoder";
|
|
5
|
+
import { redactText } from "./capture.js";
|
|
6
|
+
import { qualificationSha256 } from "./qualification-config.js";
|
|
7
|
+
export function redactQualificationOutput(text, secretValues = []) {
|
|
8
|
+
let output = text;
|
|
9
|
+
for (const secret of [...new Set(secretValues)].sort((left, right) => right.length - left.length)) {
|
|
10
|
+
if (secret.length >= 8)
|
|
11
|
+
output = output.split(secret).join("[REDACTED credential]");
|
|
12
|
+
}
|
|
13
|
+
output = output
|
|
14
|
+
.replace(/\b([A-Z][A-Z0-9_]*(?:API_KEY|ACCESS_KEY|SECRET_KEY|AUTH_TOKEN|BEARER_TOKEN|SESSION_TOKEN))\s*[:=]\s*[^\s,;"']+/gi, "$1=[REDACTED credential]")
|
|
15
|
+
.replace(/\bBearer\s+[A-Za-z0-9._~+\/-]{8,}/gi, "Bearer [REDACTED credential]")
|
|
16
|
+
.replace(/\b(?:sk|sess|key|token)-[A-Za-z0-9._-]{8,}/gi, "[REDACTED credential]")
|
|
17
|
+
.replace(/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, "[REDACTED credential]");
|
|
18
|
+
return redactText(output);
|
|
19
|
+
}
|
|
20
|
+
export function openQualificationBoundedCapture(path, limit, redact) {
|
|
21
|
+
const fd = openSync(path, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
|
|
22
|
+
const decoder = new StringDecoder("utf8");
|
|
23
|
+
let pending = "";
|
|
24
|
+
let acceptedRawBytes = 0;
|
|
25
|
+
let closed = false;
|
|
26
|
+
const persist = (text) => {
|
|
27
|
+
if (!text)
|
|
28
|
+
return;
|
|
29
|
+
const sanitized = Buffer.from(redact(text), "utf8");
|
|
30
|
+
const remaining = Math.max(0, limit - state.captured_bytes);
|
|
31
|
+
const part = sanitized.subarray(0, remaining);
|
|
32
|
+
if (part.length > 0) {
|
|
33
|
+
writeSync(fd, part);
|
|
34
|
+
state.captured_bytes += part.length;
|
|
35
|
+
fsyncSync(fd);
|
|
36
|
+
}
|
|
37
|
+
if (sanitized.length > remaining)
|
|
38
|
+
state.truncated = true;
|
|
39
|
+
};
|
|
40
|
+
const flushLines = () => {
|
|
41
|
+
let newline = pending.indexOf("\n");
|
|
42
|
+
while (newline >= 0) {
|
|
43
|
+
persist(pending.slice(0, newline + 1));
|
|
44
|
+
pending = pending.slice(newline + 1);
|
|
45
|
+
newline = pending.indexOf("\n");
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
const state = {
|
|
49
|
+
path, total_bytes: 0, captured_bytes: 0, truncated: false,
|
|
50
|
+
write(chunk) {
|
|
51
|
+
state.total_bytes += chunk.length;
|
|
52
|
+
const remainingRaw = Math.max(0, limit - acceptedRawBytes);
|
|
53
|
+
const accepted = chunk.subarray(0, remainingRaw);
|
|
54
|
+
acceptedRawBytes += accepted.length;
|
|
55
|
+
if (accepted.length > 0) {
|
|
56
|
+
pending += decoder.write(accepted);
|
|
57
|
+
flushLines();
|
|
58
|
+
}
|
|
59
|
+
if (chunk.length > remainingRaw)
|
|
60
|
+
state.truncated = true;
|
|
61
|
+
},
|
|
62
|
+
close() {
|
|
63
|
+
if (closed)
|
|
64
|
+
return;
|
|
65
|
+
pending += decoder.end();
|
|
66
|
+
// If the raw limit split a line, discard that incomplete tail rather than
|
|
67
|
+
// persisting a credential prefix that cannot be matched as a whole value.
|
|
68
|
+
if (!state.truncated)
|
|
69
|
+
persist(pending);
|
|
70
|
+
pending = "";
|
|
71
|
+
fsyncSync(fd);
|
|
72
|
+
closeSync(fd);
|
|
73
|
+
closed = true;
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
return state;
|
|
77
|
+
}
|
|
78
|
+
export function qualificationOutputReceipt(spoolRoot, path, capture) {
|
|
79
|
+
const bytes = readFileSync(path);
|
|
80
|
+
return {
|
|
81
|
+
path: path.slice(spoolRoot.length + 1),
|
|
82
|
+
total_bytes: capture.total_bytes,
|
|
83
|
+
captured_bytes: bytes.length,
|
|
84
|
+
truncated: capture.truncated,
|
|
85
|
+
sha256: qualificationSha256(bytes),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
export function verifyQualificationOutputReceipt(spoolRoot, receipt) {
|
|
89
|
+
const path = join(spoolRoot, receipt.path);
|
|
90
|
+
let fd;
|
|
91
|
+
try {
|
|
92
|
+
fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
|
93
|
+
const stat = fstatSync(fd);
|
|
94
|
+
if (!stat.isFile())
|
|
95
|
+
throw new Error(`qualification output ${receipt.path} must be a regular non-symlink file`);
|
|
96
|
+
const bytes = readFileSync(fd);
|
|
97
|
+
if (bytes.length !== receipt.captured_bytes || qualificationSha256(bytes) !== receipt.sha256) {
|
|
98
|
+
throw new Error(`qualification output ${receipt.path} digest/size mismatch`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
finally {
|
|
102
|
+
if (fd !== undefined)
|
|
103
|
+
closeSync(fd);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
export function attestQualificationPiJsonl(text, requested) {
|
|
107
|
+
const records = [];
|
|
108
|
+
try {
|
|
109
|
+
for (const line of text.split("\n")) {
|
|
110
|
+
if (!line.trim())
|
|
111
|
+
continue;
|
|
112
|
+
const value = JSON.parse(line);
|
|
113
|
+
if (!plainObject(value))
|
|
114
|
+
throw new Error("event is not an object");
|
|
115
|
+
records.push(value);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
return { ok: false, actual: null, fallback: false, refused: false, error: "artifact is not valid JSONL" };
|
|
120
|
+
}
|
|
121
|
+
const fallback = records.some((record) => record.type === "provider_fallback" || record.type === "model_fallback" || Object.hasOwn(record, "fallback_provider") || Object.hasOwn(record, "fallback_model"));
|
|
122
|
+
const messages = records
|
|
123
|
+
.filter((record) => record.type === "message_end" && plainObject(record.message) && record.message.role === "assistant")
|
|
124
|
+
.map((record) => record.message);
|
|
125
|
+
if (messages.length === 0)
|
|
126
|
+
return { ok: false, actual: null, fallback, refused: false, error: "artifact has no authoritative assistant provider/model identity" };
|
|
127
|
+
const identities = messages.map((message) => ({ provider: stringValue(message.provider), model: stringValue(message.model) }));
|
|
128
|
+
if (identities.some((identity) => !identity.provider || !identity.model))
|
|
129
|
+
return { ok: false, actual: null, fallback, refused: false, error: "artifact assistant identity is missing provider or model" };
|
|
130
|
+
const actual = { provider: identities.at(-1).provider, model: identities.at(-1).model };
|
|
131
|
+
if (fallback)
|
|
132
|
+
return { ok: false, actual, fallback: true, refused: false, error: "artifact reports provider/model fallback" };
|
|
133
|
+
if (identities.some((identity) => identity.provider !== requested.provider))
|
|
134
|
+
return { ok: false, actual, fallback: false, refused: false, error: `provider substitution: requested ${requested.provider}, observed ${actual.provider}` };
|
|
135
|
+
if (identities.some((identity) => identity.model !== requested.model))
|
|
136
|
+
return { ok: false, actual, fallback: false, refused: false, error: `model substitution: requested ${requested.model}, observed ${actual.model}` };
|
|
137
|
+
const terminal = messages.at(-1);
|
|
138
|
+
const errorMessage = stringValue(terminal.errorMessage) ?? "";
|
|
139
|
+
const refused = terminal.stopReason === "error" && /(?:refus|not supported|not available|subscription|usage limit|unauthor)/i.test(errorMessage);
|
|
140
|
+
if (terminal.stopReason === "error" && !refused)
|
|
141
|
+
return { ok: false, actual, fallback: false, refused: false, error: errorMessage || "assistant ended with an error" };
|
|
142
|
+
return { ok: true, actual, fallback: false, refused, error: refused ? errorMessage : null };
|
|
143
|
+
}
|
|
144
|
+
export async function spawnQualificationCapture(command, argv, options) {
|
|
145
|
+
return new Promise((resolve, reject) => {
|
|
146
|
+
const child = spawn(command, argv, { cwd: options.cwd, env: options.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
147
|
+
let stdout = Buffer.alloc(0), stderr = Buffer.alloc(0), timedOut = false;
|
|
148
|
+
const append = (current, chunk) => Buffer.concat([current, chunk]).subarray(0, options.output_limit_bytes);
|
|
149
|
+
child.stdout.on("data", (chunk) => { stdout = append(stdout, chunk); });
|
|
150
|
+
child.stderr.on("data", (chunk) => { stderr = append(stderr, chunk); });
|
|
151
|
+
const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, options.timeout_ms);
|
|
152
|
+
child.once("error", (error) => { clearTimeout(timer); reject(error); });
|
|
153
|
+
child.once("close", (code) => { clearTimeout(timer); resolve({ stdout: stdout.toString("utf8"), stderr: stderr.toString("utf8"), code, timed_out: timedOut }); });
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
function plainObject(value) {
|
|
157
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
158
|
+
return false;
|
|
159
|
+
const prototype = Object.getPrototypeOf(value);
|
|
160
|
+
return prototype === Object.prototype || prototype === null;
|
|
161
|
+
}
|
|
162
|
+
function stringValue(value) { return typeof value === "string" ? value : undefined; }
|
|
163
|
+
//# sourceMappingURL=qualification-capture.js.map
|