@skill-harness/core 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +2 -1
  2. package/dist/adapters/types.d.ts +30 -0
  3. package/dist/adjudication.d.ts +9 -1
  4. package/dist/adjudication.js +41 -33
  5. package/dist/arms.d.ts +51 -0
  6. package/dist/arms.js +255 -0
  7. package/dist/capture-trace-types.d.ts +2 -0
  8. package/dist/command-cost.d.ts +9 -0
  9. package/dist/command-cost.js +23 -0
  10. package/dist/comparison.d.ts +1 -0
  11. package/dist/comparison.js +9 -3
  12. package/dist/execution-trace.js +9 -1
  13. package/dist/grade.d.ts +2 -0
  14. package/dist/grade.js +2 -1
  15. package/dist/index.d.ts +14 -0
  16. package/dist/index.js +14 -0
  17. package/dist/journal.d.ts +2 -1
  18. package/dist/judge-agreement.d.ts +19 -0
  19. package/dist/judge-agreement.js +31 -0
  20. package/dist/judge-policy.js +5 -3
  21. package/dist/lift.js +1 -1
  22. package/dist/lint.js +1 -1
  23. package/dist/prompt-normalization.d.ts +6 -0
  24. package/dist/prompt-normalization.js +14 -0
  25. package/dist/provider-failure.d.ts +56 -0
  26. package/dist/provider-failure.js +98 -0
  27. package/dist/qualification-capture.d.ts +44 -0
  28. package/dist/qualification-capture.js +163 -0
  29. package/dist/qualification-config.d.ts +233 -0
  30. package/dist/qualification-config.js +694 -0
  31. package/dist/qualification-lock.d.ts +15 -0
  32. package/dist/qualification-lock.js +87 -0
  33. package/dist/qualification-oauth-directory.d.ts +79 -0
  34. package/dist/qualification-oauth-directory.js +444 -0
  35. package/dist/qualification-panel-store.d.ts +14 -0
  36. package/dist/qualification-panel-store.js +188 -0
  37. package/dist/qualification-panels.d.ts +99 -0
  38. package/dist/qualification-panels.js +91 -0
  39. package/dist/qualification-process.d.ts +13 -0
  40. package/dist/qualification-process.js +188 -0
  41. package/dist/qualification-runner.d.ts +203 -0
  42. package/dist/qualification-runner.js +2037 -0
  43. package/dist/qualification-store.d.ts +187 -0
  44. package/dist/qualification-store.js +833 -0
  45. package/dist/qualification-validation.d.ts +12 -0
  46. package/dist/qualification-validation.js +8 -0
  47. package/dist/regate.js +49 -10
  48. package/dist/regrade.d.ts +3 -1
  49. package/dist/regrade.js +62 -6
  50. package/dist/reps.d.ts +4 -2
  51. package/dist/reps.js +13 -4
  52. package/dist/rescore.js +9 -1
  53. package/dist/results.d.ts +99 -7
  54. package/dist/results.js +228 -10
  55. package/dist/run.d.ts +14 -0
  56. package/dist/run.js +260 -57
  57. package/dist/score.d.ts +4 -3
  58. package/dist/score.js +13 -4
  59. package/dist/screen.d.ts +31 -0
  60. package/dist/screen.js +105 -0
  61. package/dist/seeded.d.ts +12 -1
  62. package/dist/seeded.js +9 -2
  63. package/dist/sources.d.ts +7 -1
  64. package/dist/sources.js +13 -5
  65. package/dist/spec.d.ts +1 -1
  66. package/dist/spec.js +1 -1
  67. package/dist/stability.js +1 -1
  68. package/dist/trajectory-gates.d.ts +14 -2
  69. package/dist/trajectory-gates.js +26 -6
  70. package/dist/vote-panel.d.ts +33 -0
  71. package/dist/vote-panel.js +41 -0
  72. package/dist/workspace.js +34 -3
  73. package/package.json +3 -3
  74. package/schemas/qualification-cell-result-v1.schema.json +32 -0
  75. package/schemas/qualification-config-v1.schema.json +244 -0
  76. package/schemas/qualification-invocation-request-v1.schema.json +59 -0
  77. package/schemas/qualification-judge-panel-result-v1.schema.json +42 -0
  78. package/schemas/results-v2.schema.json +19 -21
  79. package/schemas/results-v3.schema.json +654 -0
  80. package/schemas/specification-v1.schema.json +1 -1
  81. package/schemas/trajectory-event-v1.schema.json +20 -1
@@ -157,12 +157,19 @@ export function parseTrace(lines, meta) {
157
157
  }
158
158
  }
159
159
  const toolCalls = [...calls.values()].sort((a, b) => a.issueIndex - b.issueIndex);
160
+ const subscription = meta.subject.provider === "openai-codex" || meta.subject.provider === "claude-code";
161
+ const costSource = subscription
162
+ ? "subscription"
163
+ : cost !== null
164
+ ? "provider-reported"
165
+ : "unreported";
160
166
  const metrics = {
161
167
  input_tokens: inputTokens,
162
168
  output_tokens: outputTokens,
163
169
  cache_read_tokens: cacheReadTokens,
164
170
  cache_write_tokens: cacheWriteTokens,
165
171
  cost_usd: cost ?? 0,
172
+ cost_source: costSource,
166
173
  tool_calls: toolCalls.length,
167
174
  delegated_children: toolCalls
168
175
  .filter((call) => call.name === "Agent")
@@ -348,12 +355,13 @@ export function mergeTraces(traces) {
348
355
  cache_read_tokens: sum.cache_read_tokens + trace.metrics.cache_read_tokens,
349
356
  cache_write_tokens: sum.cache_write_tokens + trace.metrics.cache_write_tokens,
350
357
  cost_usd: sum.cost_usd + trace.metrics.cost_usd,
358
+ cost_source: sum.cost_source === trace.metrics.cost_source ? sum.cost_source : "unreported",
351
359
  tool_calls: sum.tool_calls + trace.metrics.tool_calls,
352
360
  delegated_children: sum.delegated_children + trace.metrics.delegated_children,
353
361
  max_concurrency: Math.max(sum.max_concurrency, trace.metrics.max_concurrency),
354
362
  }), {
355
363
  input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0,
356
- cost_usd: 0, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
364
+ cost_usd: 0, cost_source: traces[0].metrics.cost_source, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
357
365
  })
358
366
  : undefined;
359
367
  const last = traces[traces.length - 1];
package/dist/grade.d.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import type { Scenario } from "./spec.js";
2
2
  import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
3
3
  import type { Verdict } from "./score.js";
4
+ import { type CriterionVote } from "./results.js";
4
5
  export interface JudgePromptInput {
5
6
  skill: string;
6
7
  persona: string;
@@ -36,6 +37,7 @@ export declare function parseVerdict(out: string): ParsedVerdict;
36
37
  export declare function judgeResemblesSubject(judge: ModelRef, subject: ModelRef): boolean;
37
38
  export interface GradeResult extends ParsedVerdict {
38
39
  raw: string;
40
+ criteria: CriterionVote[];
39
41
  /** Judge misfire: the overall verdict disagrees with AND(per-item grades). Recorded, never auto-passed; blocks SHIP until re-judged or overridden. */
40
42
  suspect: boolean;
41
43
  }
package/dist/grade.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { createWorkspace } from "./workspace.js";
2
+ import { parseCriterionVotes } from "./results.js";
2
3
  /** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
3
4
  export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
4
5
  /**
@@ -133,7 +134,7 @@ export async function gradeTranscript(adapter, judge, prompt, cwd) {
133
134
  parsed.reason = `judge unparseable: ${snippet}`;
134
135
  }
135
136
  const suspect = detectMisfire(raw, parsed.verdict);
136
- return { ...parsed, raw, suspect };
137
+ return { ...parsed, raw, suspect, criteria: parseCriterionVotes(raw) };
137
138
  }
138
139
  /**
139
140
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the
package/dist/index.d.ts CHANGED
@@ -23,8 +23,12 @@ export * from "./util/env.js";
23
23
  export * from "./scaffold.js";
24
24
  export * from "./version.js";
25
25
  export * from "./defaults.js";
26
+ export * from "./command-cost.js";
26
27
  export * from "./judge-policy.js";
28
+ export * from "./judge-agreement.js";
27
29
  export * from "./regate.js";
30
+ export * from "./screen.js";
31
+ export * from "./prompt-normalization.js";
28
32
  export * from "./downgrade.js";
29
33
  export * from "./canary.js";
30
34
  export * from "./stability.js";
@@ -37,5 +41,15 @@ export * from "./trajectory-gates.js";
37
41
  export * from "./instruction-coverage.js";
38
42
  export * from "./affected.js";
39
43
  export * from "./adjudication.js";
44
+ export * from "./vote-panel.js";
45
+ export * from "./qualification-panels.js";
46
+ export * from "./qualification-panel-store.js";
47
+ export * from "./qualification-validation.js";
40
48
  export * from "./comparison.js";
41
49
  export * from "./sandbox.js";
50
+ export * from "./provider-failure.js";
51
+ export * from "./arms.js";
52
+ export * from "./qualification-config.js";
53
+ export * from "./qualification-oauth-directory.js";
54
+ export * from "./qualification-store.js";
55
+ export * from "./qualification-runner.js";
package/dist/index.js CHANGED
@@ -23,8 +23,12 @@ export * from "./util/env.js";
23
23
  export * from "./scaffold.js";
24
24
  export * from "./version.js";
25
25
  export * from "./defaults.js";
26
+ export * from "./command-cost.js";
26
27
  export * from "./judge-policy.js";
28
+ export * from "./judge-agreement.js";
27
29
  export * from "./regate.js";
30
+ export * from "./screen.js";
31
+ export * from "./prompt-normalization.js";
28
32
  export * from "./downgrade.js";
29
33
  export * from "./canary.js";
30
34
  export * from "./stability.js";
@@ -37,6 +41,16 @@ export * from "./trajectory-gates.js";
37
41
  export * from "./instruction-coverage.js";
38
42
  export * from "./affected.js";
39
43
  export * from "./adjudication.js";
44
+ export * from "./vote-panel.js";
45
+ export * from "./qualification-panels.js";
46
+ export * from "./qualification-panel-store.js";
47
+ export * from "./qualification-validation.js";
40
48
  export * from "./comparison.js";
41
49
  export * from "./sandbox.js";
50
+ export * from "./provider-failure.js";
51
+ export * from "./arms.js";
52
+ export * from "./qualification-config.js";
53
+ export * from "./qualification-oauth-directory.js";
54
+ export * from "./qualification-store.js";
55
+ export * from "./qualification-runner.js";
42
56
  //# sourceMappingURL=index.js.map
package/dist/journal.d.ts CHANGED
@@ -90,6 +90,7 @@ export type JournalEvent = {
90
90
  ts: string;
91
91
  id: string;
92
92
  attempt: number;
93
+ reason?: string;
93
94
  rep?: number;
94
95
  } | {
95
96
  event: "rescore";
@@ -102,7 +103,7 @@ export type JournalEvent = {
102
103
  }
103
104
  /**
104
105
  * A regate: needle gates re-evaluated against the saved staged diffs. `judge_calls`
105
- * is on the record because regate is advertised as free apart from the reps whose
106
+ * is on the record because regate may spend only on the reps whose
106
107
  * gate verdict flipped — a claim the journal should be able to settle. `skipped`
107
108
  * names scenarios it could not regate (vitest/post_test, or missing diff artifacts).
108
109
  */
@@ -0,0 +1,19 @@
1
+ import type { ResultsFile } from "./results.js";
2
+ import type { Verdict } from "./score.js";
3
+ export interface JudgeAgreementCell {
4
+ scenario: string;
5
+ status: "agree" | "disagree" | "error";
6
+ judges: string[];
7
+ verdicts: Verdict[];
8
+ detail: string;
9
+ }
10
+ export interface JudgeAgreementReport {
11
+ cells: JudgeAgreementCell[];
12
+ agree: number;
13
+ disagree: number;
14
+ error: number;
15
+ comparable: number;
16
+ rate: number | null;
17
+ }
18
+ /** Pure/offline projection of judgments already persisted in results.yaml. */
19
+ export declare function judgeAgreement(results: ResultsFile): JudgeAgreementReport;
@@ -0,0 +1,31 @@
1
+ /** Pure/offline projection of judgments already persisted in results.yaml. */
2
+ export function judgeAgreement(results) {
3
+ const cells = results.scenarios.map((scenario) => {
4
+ const judgments = scenario.judge_history ?? scenario.adjudication?.judgments ?? [];
5
+ const distinct = new Map();
6
+ for (const judgment of judgments) {
7
+ const key = `${judgment.judge.provider}:${judgment.judge.model}`;
8
+ distinct.delete(key); // insertion order now reflects each judge's latest persisted vote
9
+ distinct.set(key, judgment);
10
+ }
11
+ const pair = [...distinct.entries()].slice(-2);
12
+ if (pair.length < 2 || pair.some(([, judgment]) => judgment.suspect || judgment.verdict === "ERROR" || judgment.verdict === "NOT-MEASURED" || judgment.verdict === "JUDGE-AMBIGUOUS")) {
13
+ return {
14
+ scenario: scenario.id, status: "error", judges: pair.map(([judge]) => judge),
15
+ verdicts: pair.map(([, judgment]) => judgment.verdict),
16
+ detail: pair.length < 2 ? "fewer than two distinct persisted judges" : "one or more judgments is not a clean vote",
17
+ };
18
+ }
19
+ const status = pair[0][1].verdict === pair[1][1].verdict ? "agree" : "disagree";
20
+ return {
21
+ scenario: scenario.id, status, judges: pair.map(([judge]) => judge),
22
+ verdicts: pair.map(([, judgment]) => judgment.verdict), detail: `${pair[0][1].verdict} vs ${pair[1][1].verdict}`,
23
+ };
24
+ });
25
+ const agree = cells.filter((cell) => cell.status === "agree").length;
26
+ const disagree = cells.filter((cell) => cell.status === "disagree").length;
27
+ const error = cells.filter((cell) => cell.status === "error").length;
28
+ const comparable = agree + disagree;
29
+ return { cells, agree, disagree, error, comparable, rate: comparable ? agree / comparable : null };
30
+ }
31
+ //# sourceMappingURL=judge-agreement.js.map
@@ -4,15 +4,17 @@ import { BAKED_DEFAULT_JUDGE } from "./defaults.js";
4
4
  * Judge providers that cannot bill a per-token API.
5
5
  *
6
6
  * `claude-code` shells out to the `claude` CLI, authenticated with the user's
7
- * Claude subscription (OAuth). `ollama`/`lmstudio`/`llamacpp`/`local` are local
8
- * runtimes. Everything else is assumed to charge.
7
+ * Claude subscription (OAuth). `openai-codex` stays in Pi and authenticates with
8
+ * the user's ChatGPT subscription (OAuth). `ollama`/`lmstudio`/`llamacpp`/`local`
9
+ * are local runtimes. Everything else — including direct `openai` — is assumed
10
+ * to charge.
9
11
  *
10
12
  * An **allow-list**, deliberately, not a deny-list of known-paid providers: a
11
13
  * provider nobody has classified yet should be treated as able to bill, because
12
14
  * being wrong in that direction produces a surprise invoice while being wrong in
13
15
  * this direction produces one extra flag.
14
16
  */
15
- const FREE_JUDGE_PROVIDERS = new Set(["claude-code", "ollama", "lmstudio", "llamacpp", "local"]);
17
+ const FREE_JUDGE_PROVIDERS = new Set(["claude-code", "openai-codex", "ollama", "lmstudio", "llamacpp", "local"]);
16
18
  /** Whether judging with this ref can charge a per-token API. */
17
19
  export function isMeteredJudge(judge) {
18
20
  return !FREE_JUDGE_PROVIDERS.has(judge.provider);
package/dist/lift.js CHANGED
@@ -29,7 +29,7 @@ function conclusive(verdict, suspect) {
29
29
  // misfire are judge failures. Treating any of them as a FAIL would make
30
30
  // "red FAIL -> green PASS" fire on infrastructure noise, and lift would
31
31
  // measure flakiness while looking like skill value.
32
- return !suspect && verdict !== "ERROR" && verdict !== "JUDGE-AMBIGUOUS";
32
+ return !suspect && verdict !== "ERROR" && verdict !== "NOT-MEASURED" && verdict !== "JUDGE-AMBIGUOUS";
33
33
  }
34
34
  function classify(red, green) {
35
35
  if (!conclusive(red.verdict, red.suspect) || !conclusive(green.verdict, green.suspect))
package/dist/lint.js CHANGED
@@ -279,7 +279,7 @@ export function lintSkill(skillDir) {
279
279
  else if (current !== recorded) {
280
280
  // The remedy is per key kind, and it is the whole point of the split: a
281
281
  // rubric edit is re-gradeable from saved transcripts, a policy edit is a
282
- // free rescore, a needle edit is a free regate. Only stimulus drift costs
282
+ // free rescore, while a gate edit avoids a subject re-run via regate. Only stimulus drift costs
283
283
  // model spend. One message saying "re-run" for all four is what made
284
284
  // correcting a known-bad rubric expensive enough to skip.
285
285
  findings.push({ skill, scenario, code: "stale", message: `${what} changed since the newest ${basename(tagDir)} run (${newest}) — results are stale; ${remedyForKey(key)}` });
@@ -0,0 +1,6 @@
1
+ export declare const PROMPT_NORMALIZATION_RULE: "cwd-line-v1";
2
+ export declare const PROMPT_NORMALIZATION_PATTERN = "^(Current working directory:)[^\\r\\n]*(\\r?)$";
3
+ export declare const PROMPT_NORMALIZATION_FLAGS = "gm";
4
+ export declare const PROMPT_NORMALIZATION_REPLACEMENT = "$1<normalized>$2";
5
+ export declare const PROMPT_NORMALIZATION_SOURCE_KEY = "observation:prompt-normalization";
6
+ export declare const PROMPT_NORMALIZATION_SOURCE_DIGEST: string;
@@ -0,0 +1,14 @@
1
+ import { createHash } from "node:crypto";
2
+ export const PROMPT_NORMALIZATION_RULE = "cwd-line-v1";
3
+ export const PROMPT_NORMALIZATION_PATTERN = "^(Current working directory:)[^\\r\\n]*(\\r?)$";
4
+ export const PROMPT_NORMALIZATION_FLAGS = "gm";
5
+ export const PROMPT_NORMALIZATION_REPLACEMENT = "$1<normalized>$2";
6
+ export const PROMPT_NORMALIZATION_SOURCE_KEY = "observation:prompt-normalization";
7
+ export const PROMPT_NORMALIZATION_SOURCE_DIGEST = createHash("sha256").update(JSON.stringify([
8
+ "prompt-normalization-registry",
9
+ PROMPT_NORMALIZATION_RULE,
10
+ PROMPT_NORMALIZATION_PATTERN,
11
+ PROMPT_NORMALIZATION_FLAGS,
12
+ PROMPT_NORMALIZATION_REPLACEMENT,
13
+ ])).digest("hex");
14
+ //# sourceMappingURL=prompt-normalization.js.map
@@ -0,0 +1,56 @@
1
+ /**
2
+ * Provider-side failure detection.
3
+ *
4
+ * A provider outage is INFRASTRUCTURE, never a model verdict. Measured 2026-08-22:
5
+ * an invalidated `openai-codex` OAuth token makes pi report
6
+ * `stopReason: "error"` with a `provider_transport_failure` diagnostic and **exit 0**
7
+ * in `--mode json`, and exit 1 with `Encountered invalidated oauth token for user,
8
+ * failing request` on stderr in text mode. Neither path was classified, so a wave
9
+ * against a dead token produced model FAILs shaped exactly like findings.
10
+ *
11
+ * Two entry points because the two run paths surface it differently, and the
12
+ * structured one is the WEAKER signal — it hides the real message behind a generic
13
+ * transport error and exits 0. Do not assume `--structured` improves diagnosis.
14
+ */
15
+ /**
16
+ * Written into a transcript by the adapter when pi failed provider-side, so the
17
+ * text path can carry a machine-readable signal through a `string` return without
18
+ * widening `HarnessAdapter.run`. Read back by `providerFailureFromTranscript`.
19
+ *
20
+ * The marker must not be forgeable by a model that types the same words into its
21
+ * answer — and a verdict is what hangs on it, since a forged marker converts a
22
+ * would-be FAIL into ERROR and (via `judgeOneRep`'s short-circuit) permanently
23
+ * suppresses the judge on every later re-grade. Line-anchoring alone does not do
24
+ * that: nothing stops a model emitting a newline and then the marker.
25
+ *
26
+ * So position, not spelling, is the guarantee. The marker is written ONLY into the
27
+ * transcript PREAMBLE — everything before the first `>>> ` turn header — which the
28
+ * adapter composes before any model output exists. There is no ordering by which a
29
+ * model's text can land there. See `withProviderFailure` (write) and
30
+ * `providerFailureFromTranscript` (read); both live here so the two halves of the
31
+ * convention cannot drift apart.
32
+ */
33
+ export declare const PROVIDER_FAILURE_MARKER = "[skill-harness] provider failure:";
34
+ /**
35
+ * Prefix `transcript` with the provider-failure marker, or return it unchanged.
36
+ *
37
+ * The one supported way to write the marker: it must precede the first turn header
38
+ * to be readable back, and putting the rule next to the reader keeps a future
39
+ * adapter from re-introducing the forgeable placement.
40
+ */
41
+ export declare function withProviderFailure(transcript: string, failure: string | null): string;
42
+ /**
43
+ * A provider failure named by one `pi --mode json` line, or null.
44
+ *
45
+ * Fail-closed on the diagnostic, fail-open on the parse: an unreadable line is not
46
+ * evidence of a failure, and throwing here would abort a wave over one bad line.
47
+ */
48
+ export declare function providerFailureFromJsonLine(line: string): string | null;
49
+ /**
50
+ * The failure the adapter recorded in a transcript's preamble, or null.
51
+ *
52
+ * Scanning STOPS at the first turn header. Everything from there on is model
53
+ * output or interleaved with it, so a marker found there is not evidence the
54
+ * provider failed — it is evidence the model typed the words.
55
+ */
56
+ export declare function providerFailureFromTranscript(transcript: string): string | null;
@@ -0,0 +1,98 @@
1
+ /**
2
+ * Provider-side failure detection.
3
+ *
4
+ * A provider outage is INFRASTRUCTURE, never a model verdict. Measured 2026-08-22:
5
+ * an invalidated `openai-codex` OAuth token makes pi report
6
+ * `stopReason: "error"` with a `provider_transport_failure` diagnostic and **exit 0**
7
+ * in `--mode json`, and exit 1 with `Encountered invalidated oauth token for user,
8
+ * failing request` on stderr in text mode. Neither path was classified, so a wave
9
+ * against a dead token produced model FAILs shaped exactly like findings.
10
+ *
11
+ * Two entry points because the two run paths surface it differently, and the
12
+ * structured one is the WEAKER signal — it hides the real message behind a generic
13
+ * transport error and exits 0. Do not assume `--structured` improves diagnosis.
14
+ */
15
+ /**
16
+ * Written into a transcript by the adapter when pi failed provider-side, so the
17
+ * text path can carry a machine-readable signal through a `string` return without
18
+ * widening `HarnessAdapter.run`. Read back by `providerFailureFromTranscript`.
19
+ *
20
+ * The marker must not be forgeable by a model that types the same words into its
21
+ * answer — and a verdict is what hangs on it, since a forged marker converts a
22
+ * would-be FAIL into ERROR and (via `judgeOneRep`'s short-circuit) permanently
23
+ * suppresses the judge on every later re-grade. Line-anchoring alone does not do
24
+ * that: nothing stops a model emitting a newline and then the marker.
25
+ *
26
+ * So position, not spelling, is the guarantee. The marker is written ONLY into the
27
+ * transcript PREAMBLE — everything before the first `>>> ` turn header — which the
28
+ * adapter composes before any model output exists. There is no ordering by which a
29
+ * model's text can land there. See `withProviderFailure` (write) and
30
+ * `providerFailureFromTranscript` (read); both live here so the two halves of the
31
+ * convention cannot drift apart.
32
+ */
33
+ export const PROVIDER_FAILURE_MARKER = "[skill-harness] provider failure:";
34
+ /**
35
+ * The adapters' shared turn-header prefix (`>>> USER:` / `>>> USER (turn n/m):`),
36
+ * which is where the preamble ends and model-influenced content begins.
37
+ */
38
+ const TURN_HEADER_PREFIX = ">>> ";
39
+ /**
40
+ * Prefix `transcript` with the provider-failure marker, or return it unchanged.
41
+ *
42
+ * The one supported way to write the marker: it must precede the first turn header
43
+ * to be readable back, and putting the rule next to the reader keeps a future
44
+ * adapter from re-introducing the forgeable placement.
45
+ */
46
+ export function withProviderFailure(transcript, failure) {
47
+ return failure ? `${PROVIDER_FAILURE_MARKER} ${failure}\n\n${transcript}` : transcript;
48
+ }
49
+ /** Diagnostic types that mean the provider never ran the request. */
50
+ const FAILURE_DIAGNOSTICS = new Set(["provider_transport_failure"]);
51
+ /**
52
+ * A provider failure named by one `pi --mode json` line, or null.
53
+ *
54
+ * Fail-closed on the diagnostic, fail-open on the parse: an unreadable line is not
55
+ * evidence of a failure, and throwing here would abort a wave over one bad line.
56
+ */
57
+ export function providerFailureFromJsonLine(line) {
58
+ let parsed;
59
+ try {
60
+ parsed = JSON.parse(line);
61
+ }
62
+ catch {
63
+ return null;
64
+ }
65
+ const message = parsed?.message;
66
+ if (!message || typeof message !== "object")
67
+ return null;
68
+ const diagnostics = message.diagnostics;
69
+ if (!Array.isArray(diagnostics))
70
+ return null;
71
+ for (const raw of diagnostics) {
72
+ if (typeof raw?.type !== "string" || !FAILURE_DIAGNOSTICS.has(raw.type))
73
+ continue;
74
+ const provider = typeof message.provider === "string"
75
+ ? message.provider
76
+ : "unknown provider";
77
+ const detail = typeof raw.error?.message === "string" ? raw.error.message : raw.type;
78
+ return `${provider}: ${detail}`;
79
+ }
80
+ return null;
81
+ }
82
+ /**
83
+ * The failure the adapter recorded in a transcript's preamble, or null.
84
+ *
85
+ * Scanning STOPS at the first turn header. Everything from there on is model
86
+ * output or interleaved with it, so a marker found there is not evidence the
87
+ * provider failed — it is evidence the model typed the words.
88
+ */
89
+ export function providerFailureFromTranscript(transcript) {
90
+ for (const line of transcript.split("\n")) {
91
+ if (line.startsWith(TURN_HEADER_PREFIX))
92
+ return null;
93
+ if (line.startsWith(PROVIDER_FAILURE_MARKER))
94
+ return line.slice(PROVIDER_FAILURE_MARKER.length).trim();
95
+ }
96
+ return null;
97
+ }
98
+ //# sourceMappingURL=provider-failure.js.map
@@ -0,0 +1,44 @@
1
+ export interface QualificationOutputReceipt {
2
+ path: string;
3
+ total_bytes: number;
4
+ captured_bytes: number;
5
+ truncated: boolean;
6
+ sha256: string;
7
+ }
8
+ export interface QualificationBoundedCapture {
9
+ path: string;
10
+ total_bytes: number;
11
+ captured_bytes: number;
12
+ truncated: boolean;
13
+ write(chunk: Buffer): void;
14
+ close(): void;
15
+ }
16
+ export interface QualificationArtifactAttestation {
17
+ ok: boolean;
18
+ actual: {
19
+ provider: string;
20
+ model: string;
21
+ } | null;
22
+ fallback: boolean;
23
+ refused: boolean;
24
+ error: string | null;
25
+ }
26
+ export declare function redactQualificationOutput(text: string, secretValues?: readonly string[]): string;
27
+ export declare function openQualificationBoundedCapture(path: string, limit: number, redact: (text: string) => string): QualificationBoundedCapture;
28
+ export declare function qualificationOutputReceipt(spoolRoot: string, path: string, capture: QualificationBoundedCapture): QualificationOutputReceipt;
29
+ export declare function verifyQualificationOutputReceipt(spoolRoot: string, receipt: QualificationOutputReceipt): void;
30
+ export declare function attestQualificationPiJsonl(text: string, requested: {
31
+ provider: string;
32
+ model: string;
33
+ }): QualificationArtifactAttestation;
34
+ export declare function spawnQualificationCapture(command: string, argv: string[], options: {
35
+ cwd: string;
36
+ env: NodeJS.ProcessEnv;
37
+ timeout_ms: number;
38
+ output_limit_bytes: number;
39
+ }): Promise<{
40
+ stdout: string;
41
+ stderr: string;
42
+ code: number | null;
43
+ timed_out: boolean;
44
+ }>;
@@ -0,0 +1,163 @@
1
+ import { spawn } from "node:child_process";
2
+ import { closeSync, constants, fstatSync, fsyncSync, openSync, readFileSync, writeSync } from "node:fs";
3
+ import { join } from "node:path";
4
+ import { StringDecoder } from "node:string_decoder";
5
+ import { redactText } from "./capture.js";
6
+ import { qualificationSha256 } from "./qualification-config.js";
7
+ export function redactQualificationOutput(text, secretValues = []) {
8
+ let output = text;
9
+ for (const secret of [...new Set(secretValues)].sort((left, right) => right.length - left.length)) {
10
+ if (secret.length >= 8)
11
+ output = output.split(secret).join("[REDACTED credential]");
12
+ }
13
+ output = output
14
+ .replace(/\b([A-Z][A-Z0-9_]*(?:API_KEY|ACCESS_KEY|SECRET_KEY|AUTH_TOKEN|BEARER_TOKEN|SESSION_TOKEN))\s*[:=]\s*[^\s,;"']+/gi, "$1=[REDACTED credential]")
15
+ .replace(/\bBearer\s+[A-Za-z0-9._~+\/-]{8,}/gi, "Bearer [REDACTED credential]")
16
+ .replace(/\b(?:sk|sess|key|token)-[A-Za-z0-9._-]{8,}/gi, "[REDACTED credential]")
17
+ .replace(/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, "[REDACTED credential]");
18
+ return redactText(output);
19
+ }
20
+ export function openQualificationBoundedCapture(path, limit, redact) {
21
+ const fd = openSync(path, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
22
+ const decoder = new StringDecoder("utf8");
23
+ let pending = "";
24
+ let acceptedRawBytes = 0;
25
+ let closed = false;
26
+ const persist = (text) => {
27
+ if (!text)
28
+ return;
29
+ const sanitized = Buffer.from(redact(text), "utf8");
30
+ const remaining = Math.max(0, limit - state.captured_bytes);
31
+ const part = sanitized.subarray(0, remaining);
32
+ if (part.length > 0) {
33
+ writeSync(fd, part);
34
+ state.captured_bytes += part.length;
35
+ fsyncSync(fd);
36
+ }
37
+ if (sanitized.length > remaining)
38
+ state.truncated = true;
39
+ };
40
+ const flushLines = () => {
41
+ let newline = pending.indexOf("\n");
42
+ while (newline >= 0) {
43
+ persist(pending.slice(0, newline + 1));
44
+ pending = pending.slice(newline + 1);
45
+ newline = pending.indexOf("\n");
46
+ }
47
+ };
48
+ const state = {
49
+ path, total_bytes: 0, captured_bytes: 0, truncated: false,
50
+ write(chunk) {
51
+ state.total_bytes += chunk.length;
52
+ const remainingRaw = Math.max(0, limit - acceptedRawBytes);
53
+ const accepted = chunk.subarray(0, remainingRaw);
54
+ acceptedRawBytes += accepted.length;
55
+ if (accepted.length > 0) {
56
+ pending += decoder.write(accepted);
57
+ flushLines();
58
+ }
59
+ if (chunk.length > remainingRaw)
60
+ state.truncated = true;
61
+ },
62
+ close() {
63
+ if (closed)
64
+ return;
65
+ pending += decoder.end();
66
+ // If the raw limit split a line, discard that incomplete tail rather than
67
+ // persisting a credential prefix that cannot be matched as a whole value.
68
+ if (!state.truncated)
69
+ persist(pending);
70
+ pending = "";
71
+ fsyncSync(fd);
72
+ closeSync(fd);
73
+ closed = true;
74
+ },
75
+ };
76
+ return state;
77
+ }
78
+ export function qualificationOutputReceipt(spoolRoot, path, capture) {
79
+ const bytes = readFileSync(path);
80
+ return {
81
+ path: path.slice(spoolRoot.length + 1),
82
+ total_bytes: capture.total_bytes,
83
+ captured_bytes: bytes.length,
84
+ truncated: capture.truncated,
85
+ sha256: qualificationSha256(bytes),
86
+ };
87
+ }
88
+ export function verifyQualificationOutputReceipt(spoolRoot, receipt) {
89
+ const path = join(spoolRoot, receipt.path);
90
+ let fd;
91
+ try {
92
+ fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
93
+ const stat = fstatSync(fd);
94
+ if (!stat.isFile())
95
+ throw new Error(`qualification output ${receipt.path} must be a regular non-symlink file`);
96
+ const bytes = readFileSync(fd);
97
+ if (bytes.length !== receipt.captured_bytes || qualificationSha256(bytes) !== receipt.sha256) {
98
+ throw new Error(`qualification output ${receipt.path} digest/size mismatch`);
99
+ }
100
+ }
101
+ finally {
102
+ if (fd !== undefined)
103
+ closeSync(fd);
104
+ }
105
+ }
106
+ export function attestQualificationPiJsonl(text, requested) {
107
+ const records = [];
108
+ try {
109
+ for (const line of text.split("\n")) {
110
+ if (!line.trim())
111
+ continue;
112
+ const value = JSON.parse(line);
113
+ if (!plainObject(value))
114
+ throw new Error("event is not an object");
115
+ records.push(value);
116
+ }
117
+ }
118
+ catch {
119
+ return { ok: false, actual: null, fallback: false, refused: false, error: "artifact is not valid JSONL" };
120
+ }
121
+ const fallback = records.some((record) => record.type === "provider_fallback" || record.type === "model_fallback" || Object.hasOwn(record, "fallback_provider") || Object.hasOwn(record, "fallback_model"));
122
+ const messages = records
123
+ .filter((record) => record.type === "message_end" && plainObject(record.message) && record.message.role === "assistant")
124
+ .map((record) => record.message);
125
+ if (messages.length === 0)
126
+ return { ok: false, actual: null, fallback, refused: false, error: "artifact has no authoritative assistant provider/model identity" };
127
+ const identities = messages.map((message) => ({ provider: stringValue(message.provider), model: stringValue(message.model) }));
128
+ if (identities.some((identity) => !identity.provider || !identity.model))
129
+ return { ok: false, actual: null, fallback, refused: false, error: "artifact assistant identity is missing provider or model" };
130
+ const actual = { provider: identities.at(-1).provider, model: identities.at(-1).model };
131
+ if (fallback)
132
+ return { ok: false, actual, fallback: true, refused: false, error: "artifact reports provider/model fallback" };
133
+ if (identities.some((identity) => identity.provider !== requested.provider))
134
+ return { ok: false, actual, fallback: false, refused: false, error: `provider substitution: requested ${requested.provider}, observed ${actual.provider}` };
135
+ if (identities.some((identity) => identity.model !== requested.model))
136
+ return { ok: false, actual, fallback: false, refused: false, error: `model substitution: requested ${requested.model}, observed ${actual.model}` };
137
+ const terminal = messages.at(-1);
138
+ const errorMessage = stringValue(terminal.errorMessage) ?? "";
139
+ const refused = terminal.stopReason === "error" && /(?:refus|not supported|not available|subscription|usage limit|unauthor)/i.test(errorMessage);
140
+ if (terminal.stopReason === "error" && !refused)
141
+ return { ok: false, actual, fallback: false, refused: false, error: errorMessage || "assistant ended with an error" };
142
+ return { ok: true, actual, fallback: false, refused, error: refused ? errorMessage : null };
143
+ }
144
+ export async function spawnQualificationCapture(command, argv, options) {
145
+ return new Promise((resolve, reject) => {
146
+ const child = spawn(command, argv, { cwd: options.cwd, env: options.env, stdio: ["ignore", "pipe", "pipe"] });
147
+ let stdout = Buffer.alloc(0), stderr = Buffer.alloc(0), timedOut = false;
148
+ const append = (current, chunk) => Buffer.concat([current, chunk]).subarray(0, options.output_limit_bytes);
149
+ child.stdout.on("data", (chunk) => { stdout = append(stdout, chunk); });
150
+ child.stderr.on("data", (chunk) => { stderr = append(stderr, chunk); });
151
+ const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, options.timeout_ms);
152
+ child.once("error", (error) => { clearTimeout(timer); reject(error); });
153
+ child.once("close", (code) => { clearTimeout(timer); resolve({ stdout: stdout.toString("utf8"), stderr: stderr.toString("utf8"), code, timed_out: timedOut }); });
154
+ });
155
+ }
156
+ function plainObject(value) {
157
+ if (!value || typeof value !== "object" || Array.isArray(value))
158
+ return false;
159
+ const prototype = Object.getPrototypeOf(value);
160
+ return prototype === Object.prototype || prototype === null;
161
+ }
162
+ function stringValue(value) { return typeof value === "string" ? value : undefined; }
163
+ //# sourceMappingURL=qualification-capture.js.map