@skill-harness/core 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adoption.d.ts +128 -0
- package/dist/adoption.js +102 -0
- package/dist/command-cost.d.ts +1 -1
- package/dist/command-cost.js +1 -1
- package/dist/factory-calibration.d.ts +63 -0
- package/dist/factory-calibration.js +115 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +9 -0
- package/dist/intervention-results.d.ts +6 -0
- package/dist/intervention-results.js +52 -0
- package/dist/intervention.d.ts +123 -0
- package/dist/intervention.js +218 -0
- package/dist/investigation.d.ts +94 -0
- package/dist/investigation.js +148 -0
- package/dist/qualification-process.js +5 -3
- package/dist/trajectory-gates.d.ts +0 -12
- package/dist/trajectory-gates.js +0 -36
- package/dist/trusted-host-supervision.d.ts +25 -0
- package/dist/trusted-host-supervision.js +94 -0
- package/dist/work-capture-schema.d.ts +38 -0
- package/dist/work-capture-schema.js +33 -0
- package/dist/work-capture.d.ts +43 -0
- package/dist/work-capture.js +81 -0
- package/dist/work-signals.d.ts +55 -0
- package/dist/work-signals.js +71 -0
- package/package.json +1 -1
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
export interface AdoptionInput {
|
|
2
|
+
hypothesisDigest: string;
|
|
3
|
+
experimentDigest: string;
|
|
4
|
+
candidateDigest: string;
|
|
5
|
+
scopeDigest: string;
|
|
6
|
+
assessmentPolicyDigest: string;
|
|
7
|
+
rollbackCandidateDigest: string;
|
|
8
|
+
activationBoundary: "next-orders";
|
|
9
|
+
expiresAt: number;
|
|
10
|
+
}
|
|
11
|
+
export interface AdoptionBinding extends AdoptionInput {
|
|
12
|
+
version: "adoption-binding-v1";
|
|
13
|
+
id: string;
|
|
14
|
+
}
|
|
15
|
+
export interface AdoptionAuthority {
|
|
16
|
+
id: string;
|
|
17
|
+
adoptions: readonly string[];
|
|
18
|
+
rollbacks: readonly string[];
|
|
19
|
+
}
|
|
20
|
+
export interface AdoptionFacts {
|
|
21
|
+
experimentDigest: string;
|
|
22
|
+
candidateDigest: string;
|
|
23
|
+
scopeDigest: string;
|
|
24
|
+
assessmentPolicyDigest: string;
|
|
25
|
+
eligible: boolean;
|
|
26
|
+
}
|
|
27
|
+
export declare function buildAdoptionBinding(input: AdoptionInput): AdoptionBinding;
|
|
28
|
+
/** Independent host declarations and independently verified eligible facts are required again at use. */
|
|
29
|
+
export declare function authorizeAdoption(binding: AdoptionBinding, authority: AdoptionAuthority | null, facts: AdoptionFacts, now: number): Readonly<{
|
|
30
|
+
id: string;
|
|
31
|
+
version: "adoption-receipt-v1";
|
|
32
|
+
bindingId: string;
|
|
33
|
+
authorityId: string;
|
|
34
|
+
hypothesisDigest: string;
|
|
35
|
+
experimentDigest: string;
|
|
36
|
+
candidateDigest: string;
|
|
37
|
+
scopeDigest: string;
|
|
38
|
+
assessmentPolicyDigest: string;
|
|
39
|
+
rollbackCandidateDigest: string;
|
|
40
|
+
activationBoundary: "next-orders";
|
|
41
|
+
grantExpansion: false;
|
|
42
|
+
expiresAt: number;
|
|
43
|
+
}>;
|
|
44
|
+
export type AdoptionReceipt = ReturnType<typeof authorizeAdoption>;
|
|
45
|
+
export declare function validateAdoptionReceipt(receipt: AdoptionReceipt, binding: AdoptionBinding, authority: AdoptionAuthority | null, facts: AdoptionFacts, now: number): void;
|
|
46
|
+
export interface ProductionObservation {
|
|
47
|
+
id: string;
|
|
48
|
+
adoptionId: string;
|
|
49
|
+
candidateDigest: string;
|
|
50
|
+
scopeDigest: string;
|
|
51
|
+
originalRequirementDigest: string;
|
|
52
|
+
currentRequirementDigest: string;
|
|
53
|
+
/** Only an independent host may supply confirmed outcomes; observer/model claims are not confirmation. */
|
|
54
|
+
outcome: "confirmed-defect" | "success" | "unknown";
|
|
55
|
+
acceptanceDigest: string | null;
|
|
56
|
+
acceptedArtifactDigest: string | null;
|
|
57
|
+
observedArtifactDigest: string;
|
|
58
|
+
evidence: string[];
|
|
59
|
+
}
|
|
60
|
+
export declare function classifyProductionObservation(adoption: AdoptionReceipt, input: ProductionObservation): Readonly<{
|
|
61
|
+
state: "unknown" | "success" | "out-of-scope" | "changed-requirements" | "changed-artifact" | "caught-defect" | "escape";
|
|
62
|
+
adoptionId: string;
|
|
63
|
+
hypothesisDigest: string;
|
|
64
|
+
experimentDigest: string;
|
|
65
|
+
observationId: string;
|
|
66
|
+
acceptanceDigest: string | null;
|
|
67
|
+
acceptedArtifactDigest: string | null;
|
|
68
|
+
observedArtifactDigest: string;
|
|
69
|
+
automaticRollback: false;
|
|
70
|
+
}>;
|
|
71
|
+
export interface RollbackRequest {
|
|
72
|
+
version: "rollback-request-v1";
|
|
73
|
+
id: string;
|
|
74
|
+
adoptionId: string;
|
|
75
|
+
scopeDigest: string;
|
|
76
|
+
restoreCandidateDigest: string;
|
|
77
|
+
reason: "operator-request" | "confirmed-escape";
|
|
78
|
+
evidence: readonly string[];
|
|
79
|
+
expiresAt: number;
|
|
80
|
+
}
|
|
81
|
+
export declare function buildRollbackRequest(adoption: AdoptionReceipt, reason: RollbackRequest["reason"], evidence: string[], expiresAt?: number): RollbackRequest;
|
|
82
|
+
/** This validates a recorded choice, not an automatic rollback action or a caller-created authority flag. */
|
|
83
|
+
export declare function authorizeRollback(adoption: AdoptionReceipt, request: RollbackRequest, authority: AdoptionAuthority | null, now: number, current: {
|
|
84
|
+
adoptionId: string;
|
|
85
|
+
candidateDigest: string;
|
|
86
|
+
scopeDigest: string;
|
|
87
|
+
}): Readonly<{
|
|
88
|
+
requestId: string;
|
|
89
|
+
adoptionId: string;
|
|
90
|
+
restoreCandidateDigest: string;
|
|
91
|
+
scopeDigest: string;
|
|
92
|
+
grantExpansion: false;
|
|
93
|
+
application: "not-performed";
|
|
94
|
+
}>;
|
|
95
|
+
export interface LeanOutcome {
|
|
96
|
+
population: string;
|
|
97
|
+
complete: boolean;
|
|
98
|
+
accepted: boolean | null;
|
|
99
|
+
repairAttempts: number | null;
|
|
100
|
+
escape: "confirmed" | "none-observed" | "unknown";
|
|
101
|
+
requirementsChanged: boolean;
|
|
102
|
+
decisionWaitMs: number | null;
|
|
103
|
+
questions: number | null;
|
|
104
|
+
}
|
|
105
|
+
export declare function summarizeLeanOutcomes(population: string, rows: readonly LeanOutcome[]): {
|
|
106
|
+
population: string;
|
|
107
|
+
firstPassAccepted: number;
|
|
108
|
+
firstPassDenominator: number;
|
|
109
|
+
incomplete: number;
|
|
110
|
+
confirmedEscapes: number;
|
|
111
|
+
changedRequirements: number;
|
|
112
|
+
recovery: {
|
|
113
|
+
known: number;
|
|
114
|
+
observed: number;
|
|
115
|
+
total: number;
|
|
116
|
+
};
|
|
117
|
+
decisionWait: {
|
|
118
|
+
known: number;
|
|
119
|
+
observed: number;
|
|
120
|
+
total: number;
|
|
121
|
+
};
|
|
122
|
+
questions: {
|
|
123
|
+
known: number;
|
|
124
|
+
observed: number;
|
|
125
|
+
total: number;
|
|
126
|
+
};
|
|
127
|
+
universalScore: null;
|
|
128
|
+
};
|
package/dist/adoption.js
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
const SHA = /^[a-f0-9]{64}$/;
|
|
3
|
+
const text = (x) => typeof x === "string" && x.length > 0 && x.length <= 512 && !/[\u0000-\u001f\u007f]/.test(x);
|
|
4
|
+
const closed = (value, names) => {
|
|
5
|
+
if (!value || ![Object.prototype, null].includes(Object.getPrototypeOf(value)))
|
|
6
|
+
return false;
|
|
7
|
+
const fields = Object.getOwnPropertyDescriptors(value);
|
|
8
|
+
return Reflect.ownKeys(fields).length === names.length && Reflect.ownKeys(fields).every(key => typeof key === "string" && names.includes(key) && fields[key].enumerable && Object.hasOwn(fields[key], "value"));
|
|
9
|
+
};
|
|
10
|
+
const hash = (value) => createHash("sha256").update(JSON.stringify(Object.keys(value).sort().map(k => [k, value[k]]))).digest("hex");
|
|
11
|
+
export function buildAdoptionBinding(input) {
|
|
12
|
+
if (!input || !closed(input, ["hypothesisDigest", "experimentDigest", "candidateDigest", "scopeDigest", "assessmentPolicyDigest", "rollbackCandidateDigest", "activationBoundary", "expiresAt"])
|
|
13
|
+
|| ![input.hypothesisDigest, input.experimentDigest, input.candidateDigest, input.scopeDigest, input.assessmentPolicyDigest, input.rollbackCandidateDigest].every(s => typeof s === "string" && SHA.test(s))
|
|
14
|
+
|| input.activationBoundary !== "next-orders" || !Number.isSafeInteger(input.expiresAt) || input.expiresAt < 0)
|
|
15
|
+
throw new Error("invalid scoped adoption binding");
|
|
16
|
+
const body = { version: "adoption-binding-v1", ...input };
|
|
17
|
+
return Object.freeze({ ...body, id: hash(body) });
|
|
18
|
+
}
|
|
19
|
+
function bindingValid(binding) {
|
|
20
|
+
const { version, id, ...input } = binding;
|
|
21
|
+
if (version !== "adoption-binding-v1" || buildAdoptionBinding(input).id !== id)
|
|
22
|
+
throw new Error("adoption binding changed");
|
|
23
|
+
}
|
|
24
|
+
function authorityValid(authority) {
|
|
25
|
+
return !!authority && text(authority.id) && Array.isArray(authority.adoptions) && Array.isArray(authority.rollbacks)
|
|
26
|
+
&& authority.adoptions.length <= 256 && authority.rollbacks.length <= 256 && [...authority.adoptions, ...authority.rollbacks].every(id => typeof id === "string" && SHA.test(id));
|
|
27
|
+
}
|
|
28
|
+
/** Independent host declarations and independently verified eligible facts are required again at use. */
|
|
29
|
+
export function authorizeAdoption(binding, authority, facts, now) {
|
|
30
|
+
bindingValid(binding);
|
|
31
|
+
if (!authorityValid(authority) || !authority.adoptions.includes(binding.id))
|
|
32
|
+
throw new Error("independent adoption authority required");
|
|
33
|
+
if (!Number.isSafeInteger(now) || now < 0 || now >= binding.expiresAt || !facts || !closed(facts, ["experimentDigest", "candidateDigest", "scopeDigest", "assessmentPolicyDigest", "eligible"]) || facts.eligible !== true
|
|
34
|
+
|| facts.experimentDigest !== binding.experimentDigest || facts.candidateDigest !== binding.candidateDigest
|
|
35
|
+
|| facts.scopeDigest !== binding.scopeDigest || facts.assessmentPolicyDigest !== binding.assessmentPolicyDigest)
|
|
36
|
+
throw new Error("adoption expired, ineligible or outside verified scope");
|
|
37
|
+
const body = { version: "adoption-receipt-v1", bindingId: binding.id, authorityId: authority.id, hypothesisDigest: binding.hypothesisDigest,
|
|
38
|
+
experimentDigest: binding.experimentDigest, candidateDigest: binding.candidateDigest, scopeDigest: binding.scopeDigest,
|
|
39
|
+
assessmentPolicyDigest: binding.assessmentPolicyDigest, rollbackCandidateDigest: binding.rollbackCandidateDigest,
|
|
40
|
+
activationBoundary: "next-orders", grantExpansion: false, expiresAt: binding.expiresAt };
|
|
41
|
+
return Object.freeze({ ...body, id: hash(body) });
|
|
42
|
+
}
|
|
43
|
+
export function validateAdoptionReceipt(receipt, binding, authority, facts, now) {
|
|
44
|
+
if (authorizeAdoption(binding, authority, facts, now).id !== receipt.id)
|
|
45
|
+
throw new Error("adoption receipt does not match current independent authority/facts");
|
|
46
|
+
receiptValid(receipt);
|
|
47
|
+
}
|
|
48
|
+
function receiptValid(receipt) {
|
|
49
|
+
const { id, ...body } = receipt;
|
|
50
|
+
if (!closed(receipt, ["version", "bindingId", "authorityId", "hypothesisDigest", "experimentDigest", "candidateDigest", "scopeDigest", "assessmentPolicyDigest", "rollbackCandidateDigest", "activationBoundary", "grantExpansion", "expiresAt", "id"])
|
|
51
|
+
|| body.version !== "adoption-receipt-v1" || !text(body.authorityId) || body.activationBoundary !== "next-orders" || body.grantExpansion !== false || hash(body) !== id)
|
|
52
|
+
throw new Error("adoption receipt changed");
|
|
53
|
+
const binding = buildAdoptionBinding({ hypothesisDigest: body.hypothesisDigest, experimentDigest: body.experimentDigest, candidateDigest: body.candidateDigest,
|
|
54
|
+
scopeDigest: body.scopeDigest, assessmentPolicyDigest: body.assessmentPolicyDigest, rollbackCandidateDigest: body.rollbackCandidateDigest, activationBoundary: body.activationBoundary, expiresAt: body.expiresAt });
|
|
55
|
+
if (binding.id !== body.bindingId)
|
|
56
|
+
throw new Error("adoption receipt binding mismatch");
|
|
57
|
+
}
|
|
58
|
+
export function classifyProductionObservation(adoption, input) {
|
|
59
|
+
receiptValid(adoption);
|
|
60
|
+
if (!input || !text(input.id) || input.evidence.length > 256 || input.evidence.some(h => !SHA.test(h))
|
|
61
|
+
|| ![input.originalRequirementDigest, input.currentRequirementDigest, input.candidateDigest, input.scopeDigest, input.observedArtifactDigest].every(h => typeof h === "string" && SHA.test(h))
|
|
62
|
+
|| ![input.acceptanceDigest, input.acceptedArtifactDigest].every(h => h === null || (typeof h === "string" && SHA.test(h)))
|
|
63
|
+
|| (input.acceptanceDigest === null) !== (input.acceptedArtifactDigest === null)
|
|
64
|
+
|| !["confirmed-defect", "success", "unknown"].includes(input.outcome))
|
|
65
|
+
throw new Error("invalid production observation");
|
|
66
|
+
const state = input.adoptionId !== adoption.id || input.candidateDigest !== adoption.candidateDigest || input.scopeDigest !== adoption.scopeDigest ? "out-of-scope"
|
|
67
|
+
: input.originalRequirementDigest !== input.currentRequirementDigest ? "changed-requirements"
|
|
68
|
+
: input.acceptedArtifactDigest !== null && input.acceptedArtifactDigest !== input.observedArtifactDigest ? "changed-artifact"
|
|
69
|
+
: input.outcome === "confirmed-defect" && input.evidence.length ? input.acceptanceDigest === null ? "caught-defect" : "escape" : input.outcome === "success" ? "success" : "unknown";
|
|
70
|
+
return Object.freeze({ state, adoptionId: adoption.id, hypothesisDigest: adoption.hypothesisDigest, experimentDigest: adoption.experimentDigest, observationId: input.id, acceptanceDigest: input.acceptanceDigest, acceptedArtifactDigest: input.acceptedArtifactDigest, observedArtifactDigest: input.observedArtifactDigest, automaticRollback: false });
|
|
71
|
+
}
|
|
72
|
+
export function buildRollbackRequest(adoption, reason, evidence, expiresAt = adoption.expiresAt) {
|
|
73
|
+
receiptValid(adoption);
|
|
74
|
+
if (!Number.isSafeInteger(expiresAt) || expiresAt < 0 || !["operator-request", "confirmed-escape"].includes(reason) || !evidence.length || evidence.length > 256 || evidence.some(h => !SHA.test(h)))
|
|
75
|
+
throw new Error("invalid rollback request");
|
|
76
|
+
const body = { version: "rollback-request-v1", adoptionId: adoption.id, scopeDigest: adoption.scopeDigest, restoreCandidateDigest: adoption.rollbackCandidateDigest, reason, expiresAt, evidence: Object.freeze([...new Set(evidence)].sort()) };
|
|
77
|
+
return Object.freeze({ ...body, id: hash(body) });
|
|
78
|
+
}
|
|
79
|
+
/** This validates a recorded choice, not an automatic rollback action or a caller-created authority flag. */
|
|
80
|
+
export function authorizeRollback(adoption, request, authority, now, current) {
|
|
81
|
+
receiptValid(adoption);
|
|
82
|
+
const { id, ...body } = request;
|
|
83
|
+
if (!authorityValid(authority) || authority.id !== adoption.authorityId || !authority.rollbacks.includes(id))
|
|
84
|
+
throw new Error("independent rollback authority required");
|
|
85
|
+
if (hash(body) !== id || request.version !== "rollback-request-v1" || request.adoptionId !== adoption.id || request.scopeDigest !== adoption.scopeDigest
|
|
86
|
+
|| request.restoreCandidateDigest !== adoption.rollbackCandidateDigest || !Number.isSafeInteger(now) || now < 0 || !Number.isSafeInteger(request.expiresAt) || now >= request.expiresAt
|
|
87
|
+
|| !current || current.adoptionId !== adoption.id || current.candidateDigest !== adoption.candidateDigest || current.scopeDigest !== adoption.scopeDigest)
|
|
88
|
+
throw new Error("rollback binding changed or expired");
|
|
89
|
+
return Object.freeze({ requestId: id, adoptionId: adoption.id, restoreCandidateDigest: request.restoreCandidateDigest, scopeDigest: adoption.scopeDigest, grantExpansion: false, application: "not-performed" });
|
|
90
|
+
}
|
|
91
|
+
export function summarizeLeanOutcomes(population, rows) {
|
|
92
|
+
if (!text(population) || rows.length > 10000 || rows.some(r => r.population !== population || typeof r.complete !== "boolean" || typeof r.requirementsChanged !== "boolean"
|
|
93
|
+
|| ![true, false, null].includes(r.accepted) || !["confirmed", "none-observed", "unknown"].includes(r.escape)
|
|
94
|
+
|| [r.repairAttempts, r.decisionWaitMs, r.questions].some(n => n !== null && (!Number.isSafeInteger(n) || n < 0))))
|
|
95
|
+
throw new Error("invalid or incomparable lean population");
|
|
96
|
+
const complete = rows.filter(r => r.complete && r.accepted !== null && r.repairAttempts !== null);
|
|
97
|
+
const observed = (field) => ({ known: rows.reduce((n, r) => n + (r[field] ?? 0), 0), observed: rows.filter(r => r[field] !== null).length, total: rows.length });
|
|
98
|
+
return { population, firstPassAccepted: complete.filter(r => r.accepted && r.repairAttempts === 0).length, firstPassDenominator: complete.length,
|
|
99
|
+
incomplete: rows.length - complete.length, confirmedEscapes: rows.filter(r => r.escape === "confirmed" && !r.requirementsChanged).length,
|
|
100
|
+
changedRequirements: rows.filter(r => r.requirementsChanged).length, recovery: observed("repairAttempts"), decisionWait: observed("decisionWaitMs"), questions: observed("questions"), universalScore: null };
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=adoption.js.map
|
package/dist/command-cost.d.ts
CHANGED
|
@@ -4,6 +4,6 @@
|
|
|
4
4
|
* Keep this list narrow: commands that can conditionally spend (notably `regate`)
|
|
5
5
|
* do not belong here even when their common path only replays saved artifacts.
|
|
6
6
|
*/
|
|
7
|
-
export declare const FREE_OFFLINE_COMMANDS: readonly ["affected", "coverage", "init", "judge-agreement", "lint", "list", "
|
|
7
|
+
export declare const FREE_OFFLINE_COMMANDS: readonly ["affected", "archive", "coverage", "init", "judge-agreement", "lint", "list", "rescore", "restamp", "screen", "stability"];
|
|
8
8
|
export type FreeOfflineCommand = typeof FREE_OFFLINE_COMMANDS[number];
|
|
9
9
|
export declare function isFreeOfflineCommand(command: string): command is FreeOfflineCommand;
|
package/dist/command-cost.js
CHANGED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
export interface CalibrationComponent {
|
|
2
|
+
kind: "detector" | "adjudicator";
|
|
3
|
+
id: string;
|
|
4
|
+
version: string;
|
|
5
|
+
population: string;
|
|
6
|
+
}
|
|
7
|
+
export type CalibrationKind = "positive" | "approval" | "rejection";
|
|
8
|
+
export type CalibrationSplit = "calibration" | "heldout" | "tuning";
|
|
9
|
+
export interface CalibrationPrediction {
|
|
10
|
+
id: string;
|
|
11
|
+
incidentId: string;
|
|
12
|
+
component: CalibrationComponent;
|
|
13
|
+
kind: CalibrationKind;
|
|
14
|
+
split: CalibrationSplit;
|
|
15
|
+
}
|
|
16
|
+
/** Independently established host context, not fields accepted from predictions or panel agreement. */
|
|
17
|
+
export interface CalibrationOutcome extends CalibrationPrediction {
|
|
18
|
+
correct: boolean;
|
|
19
|
+
}
|
|
20
|
+
export interface CalibrationReport {
|
|
21
|
+
component: CalibrationComponent;
|
|
22
|
+
kind: CalibrationKind;
|
|
23
|
+
split: Exclude<CalibrationSplit, "tuning">;
|
|
24
|
+
correct: number;
|
|
25
|
+
incorrect: number;
|
|
26
|
+
resolved: number;
|
|
27
|
+
unresolved: number;
|
|
28
|
+
conflicted: number;
|
|
29
|
+
totalIncidents: number;
|
|
30
|
+
precision: number | null;
|
|
31
|
+
interval: {
|
|
32
|
+
lower: number;
|
|
33
|
+
upper: number;
|
|
34
|
+
} | null;
|
|
35
|
+
advisoryOnly: true;
|
|
36
|
+
}
|
|
37
|
+
/** Version/population/conditioning-specific statistics; no policy, grants, routing or adoption is changed. */
|
|
38
|
+
export declare function calibratePredictions(predictions: readonly CalibrationPrediction[], context?: {
|
|
39
|
+
outcomes: readonly CalibrationOutcome[];
|
|
40
|
+
}): {
|
|
41
|
+
version: "factory-calibration-v1";
|
|
42
|
+
reports: CalibrationReport[];
|
|
43
|
+
excludedTuning: number;
|
|
44
|
+
referenceBasis: "independently-supplied-host-context";
|
|
45
|
+
advisoryOnly: true;
|
|
46
|
+
};
|
|
47
|
+
export interface ExposurePolicy {
|
|
48
|
+
id: string;
|
|
49
|
+
component: CalibrationComponent;
|
|
50
|
+
kind: CalibrationKind;
|
|
51
|
+
split: "calibration" | "heldout";
|
|
52
|
+
minimumResolved: number;
|
|
53
|
+
minimumLowerBound: number;
|
|
54
|
+
attentionRemaining: number;
|
|
55
|
+
expiresAt: number;
|
|
56
|
+
retireBelowUpperBound?: number;
|
|
57
|
+
}
|
|
58
|
+
/** A selected pre-established host policy may produce advice; this never applies it or expands authority. */
|
|
59
|
+
export declare function recommendExposure(report: CalibrationReport, policy: ExposurePolicy | null, now: number): {
|
|
60
|
+
mode: "silent" | "ask" | "retire";
|
|
61
|
+
reason: string;
|
|
62
|
+
advisoryOnly: true;
|
|
63
|
+
};
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
const computedReports = new WeakSet();
|
|
2
|
+
const text = (x) => typeof x === "string" && x.length > 0 && x.length <= 512 && !/[\u0000-\u001f\u007f]/.test(x);
|
|
3
|
+
function closed(x, fields) {
|
|
4
|
+
return x !== null && typeof x === "object" && !Array.isArray(x) && Object.keys(x).length === fields.length && Object.keys(x).every(k => fields.includes(k));
|
|
5
|
+
}
|
|
6
|
+
function valid(x, outcome = false) {
|
|
7
|
+
return closed(x, outcome ? ["id", "incidentId", "component", "kind", "split", "correct"] : ["id", "incidentId", "component", "kind", "split"])
|
|
8
|
+
&& text(x.id) && text(x.incidentId) && closed(x.component, ["kind", "id", "version", "population"])
|
|
9
|
+
&& [x.component.id, x.component.version, x.component.population].every(text)
|
|
10
|
+
&& ["calibration", "heldout", "tuning"].includes(x.split)
|
|
11
|
+
&& (x.component.kind === "detector" ? x.kind === "positive" : x.component.kind === "adjudicator" && ["approval", "rejection"].includes(x.kind))
|
|
12
|
+
&& (!outcome || typeof x.correct === "boolean");
|
|
13
|
+
}
|
|
14
|
+
const key = (x) => JSON.stringify([x.component.kind, x.component.id, x.component.version, x.component.population, x.kind, x.split]);
|
|
15
|
+
function interval(correct, count) {
|
|
16
|
+
const z = 1.96, z2 = z * z, p = correct / count, denominator = 1 + z2 / count;
|
|
17
|
+
const center = (p + z2 / (2 * count)) / denominator;
|
|
18
|
+
const half = z * Math.sqrt(p * (1 - p) / count + z2 / (4 * count * count)) / denominator;
|
|
19
|
+
return { lower: Math.max(0, center - half), upper: Math.min(1, center + half) };
|
|
20
|
+
}
|
|
21
|
+
/** Version/population/conditioning-specific statistics; no policy, grants, routing or adoption is changed. */
|
|
22
|
+
export function calibratePredictions(predictions, context = { outcomes: [] }) {
|
|
23
|
+
if (predictions.length > 10000 || context.outcomes.length > 10000 || predictions.some(p => !valid(p)) || context.outcomes.some(r => !valid(r, true)))
|
|
24
|
+
throw new Error("invalid calibration input");
|
|
25
|
+
const byId = new Map();
|
|
26
|
+
for (const p of predictions) {
|
|
27
|
+
const previous = byId.get(p.id);
|
|
28
|
+
if (previous && (key(previous) !== key(p) || previous.incidentId !== p.incidentId))
|
|
29
|
+
throw new Error("prediction identity conflict");
|
|
30
|
+
byId.set(p.id, p);
|
|
31
|
+
}
|
|
32
|
+
const groups = new Map();
|
|
33
|
+
let excludedTuning = 0;
|
|
34
|
+
for (const p of byId.values()) {
|
|
35
|
+
if (p.split === "tuning") {
|
|
36
|
+
excludedTuning++;
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
const k = key(p);
|
|
40
|
+
groups.set(k, [...(groups.get(k) ?? []), p]);
|
|
41
|
+
}
|
|
42
|
+
const referenceIds = new Map();
|
|
43
|
+
const conflictingIds = new Set();
|
|
44
|
+
for (const r of context.outcomes) {
|
|
45
|
+
const signature = JSON.stringify([key(r), r.incidentId, r.correct]);
|
|
46
|
+
if (referenceIds.has(r.id) && referenceIds.get(r.id) !== signature)
|
|
47
|
+
conflictingIds.add(r.id);
|
|
48
|
+
referenceIds.set(r.id, signature);
|
|
49
|
+
}
|
|
50
|
+
const labels = new Map();
|
|
51
|
+
const conflictedIncidents = new Set();
|
|
52
|
+
for (const r of context.outcomes) {
|
|
53
|
+
const k = JSON.stringify([key(r), r.incidentId]);
|
|
54
|
+
if (conflictingIds.has(r.id)) {
|
|
55
|
+
conflictedIncidents.add(k);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const values = labels.get(k) ?? new Set();
|
|
59
|
+
values.add(r.correct);
|
|
60
|
+
labels.set(k, values);
|
|
61
|
+
}
|
|
62
|
+
const reports = [...groups].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([groupKey, group]) => {
|
|
63
|
+
const incidents = new Set(group.map(p => p.incidentId));
|
|
64
|
+
let correct = 0, incorrect = 0, unresolved = 0, conflicted = 0;
|
|
65
|
+
for (const incident of incidents) {
|
|
66
|
+
const incidentKey = JSON.stringify([groupKey, incident]);
|
|
67
|
+
const outcomes = labels.get(incidentKey) ?? new Set();
|
|
68
|
+
if (outcomes.size !== 1 || conflictedIncidents.has(incidentKey)) {
|
|
69
|
+
unresolved++;
|
|
70
|
+
if (outcomes.size > 1 || conflictedIncidents.has(incidentKey))
|
|
71
|
+
conflicted++;
|
|
72
|
+
}
|
|
73
|
+
else if (outcomes.has(true))
|
|
74
|
+
correct++;
|
|
75
|
+
else
|
|
76
|
+
incorrect++;
|
|
77
|
+
}
|
|
78
|
+
const resolved = correct + incorrect;
|
|
79
|
+
return { component: { ...group[0].component }, kind: group[0].kind, split: group[0].split,
|
|
80
|
+
correct, incorrect, resolved, unresolved, conflicted, totalIncidents: incidents.size,
|
|
81
|
+
precision: resolved ? correct / resolved : null, interval: resolved ? interval(correct, resolved) : null, advisoryOnly: true };
|
|
82
|
+
});
|
|
83
|
+
for (const report of reports) {
|
|
84
|
+
Object.freeze(report.component);
|
|
85
|
+
if (report.interval)
|
|
86
|
+
Object.freeze(report.interval);
|
|
87
|
+
Object.freeze(report);
|
|
88
|
+
computedReports.add(report);
|
|
89
|
+
}
|
|
90
|
+
return { version: "factory-calibration-v1", reports, excludedTuning, referenceBasis: "independently-supplied-host-context", advisoryOnly: true };
|
|
91
|
+
}
|
|
92
|
+
/** A selected pre-established host policy may produce advice; this never applies it or expands authority. */
|
|
93
|
+
export function recommendExposure(report, policy, now) {
|
|
94
|
+
if (!computedReports.has(report))
|
|
95
|
+
throw new Error("recompute calibration from predictions and independent reference context before evaluating exposure");
|
|
96
|
+
const result = (mode, reason) => ({ mode, reason, advisoryOnly: true });
|
|
97
|
+
if (!policy)
|
|
98
|
+
return result("silent", "policy-unavailable");
|
|
99
|
+
if (!text(policy.id) || !Number.isSafeInteger(policy.minimumResolved) || policy.minimumResolved < 1
|
|
100
|
+
|| !Number.isSafeInteger(policy.attentionRemaining) || policy.attentionRemaining < 0
|
|
101
|
+
|| !Number.isFinite(policy.minimumLowerBound) || policy.minimumLowerBound < 0 || policy.minimumLowerBound > 1
|
|
102
|
+
|| !Number.isFinite(policy.expiresAt) || !Number.isFinite(now)
|
|
103
|
+
|| (policy.retireBelowUpperBound !== undefined && (!Number.isFinite(policy.retireBelowUpperBound) || policy.retireBelowUpperBound < 0 || policy.retireBelowUpperBound > 1)))
|
|
104
|
+
throw new Error("invalid exposure policy");
|
|
105
|
+
if (key(policy) !== key(report) || now >= policy.expiresAt)
|
|
106
|
+
return result("silent", "policy-inapplicable-or-expired");
|
|
107
|
+
if (!report.interval || report.resolved < policy.minimumResolved)
|
|
108
|
+
return result("silent", "insufficient-independent-evidence");
|
|
109
|
+
if (policy.retireBelowUpperBound !== undefined && report.interval.upper < policy.retireBelowUpperBound)
|
|
110
|
+
return result("retire", "below-selected-reliability-policy");
|
|
111
|
+
if (report.interval.lower < policy.minimumLowerBound || policy.attentionRemaining === 0)
|
|
112
|
+
return result("silent", "confidence-or-attention-bound");
|
|
113
|
+
return result("ask", "selected-evidence-and-attention-policy-met");
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=factory-calibration.js.map
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export * from "./trusted-host-supervision.js";
|
|
1
2
|
export * from "./spec.js";
|
|
2
3
|
export * from "./discover.js";
|
|
3
4
|
export * from "./run.js";
|
|
@@ -33,6 +34,14 @@ export * from "./downgrade.js";
|
|
|
33
34
|
export * from "./canary.js";
|
|
34
35
|
export * from "./stability.js";
|
|
35
36
|
export * from "./capture-trace-types.js";
|
|
37
|
+
export * from "./work-capture.js";
|
|
38
|
+
export * from "./work-signals.js";
|
|
39
|
+
export * from "./work-capture-schema.js";
|
|
40
|
+
export * from "./factory-calibration.js";
|
|
41
|
+
export * from "./adoption.js";
|
|
42
|
+
export * from "./investigation.js";
|
|
43
|
+
export * from "./intervention.js";
|
|
44
|
+
export * from "./intervention-results.js";
|
|
36
45
|
export * from "./spec-write.js";
|
|
37
46
|
export * from "./capture.js";
|
|
38
47
|
export * from "./execution-trace.js";
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export * from "./trusted-host-supervision.js";
|
|
1
2
|
export * from "./spec.js";
|
|
2
3
|
export * from "./discover.js";
|
|
3
4
|
export * from "./run.js";
|
|
@@ -33,6 +34,14 @@ export * from "./downgrade.js";
|
|
|
33
34
|
export * from "./canary.js";
|
|
34
35
|
export * from "./stability.js";
|
|
35
36
|
export * from "./capture-trace-types.js";
|
|
37
|
+
export * from "./work-capture.js";
|
|
38
|
+
export * from "./work-signals.js";
|
|
39
|
+
export * from "./work-capture-schema.js";
|
|
40
|
+
export * from "./factory-calibration.js";
|
|
41
|
+
export * from "./adoption.js";
|
|
42
|
+
export * from "./investigation.js";
|
|
43
|
+
export * from "./intervention.js";
|
|
44
|
+
export * from "./intervention-results.js";
|
|
36
45
|
export * from "./spec-write.js";
|
|
37
46
|
export * from "./capture.js";
|
|
38
47
|
export * from "./execution-trace.js";
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { InterventionManifest, InterventionEvidence } from "./intervention.js";
|
|
2
|
+
/** Normalize existing validated schema3 records, retaining the complete source votes alongside the summary. */
|
|
3
|
+
export declare function interventionEvidenceFromResults(manifest: InterventionManifest, armId: string, raw: unknown, outputBytes: ReadonlyMap<string, Uint8Array>): {
|
|
4
|
+
evidence: InterventionEvidence;
|
|
5
|
+
sourceResults: import("./results.js").ResultsFile;
|
|
6
|
+
};
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { validateResults } from "./results.js";
|
|
3
|
+
import { aggregateMetrics } from "./comparison.js";
|
|
4
|
+
/** Normalize existing validated schema3 records, retaining the complete source votes alongside the summary. */
|
|
5
|
+
export function interventionEvidenceFromResults(manifest, armId, raw, outputBytes) {
|
|
6
|
+
const sourceResults = validateResults(raw), arm = manifest.arms.find(a => a.id === armId);
|
|
7
|
+
if (!arm || sourceResults.model !== arm.configuration.model)
|
|
8
|
+
throw new Error("intervention result model mismatch");
|
|
9
|
+
if (sourceResults.schema !== 3 || sourceResults.partial || sourceResults.mode !== manifest.common.mode
|
|
10
|
+
|| `${sourceResults.judge.provider}:${sourceResults.judge.model}` !== manifest.judge)
|
|
11
|
+
throw new Error("incomplete or incompatible intervention results");
|
|
12
|
+
if (sourceResults.source_hashes?.["intervention:manifest"] !== manifest.id || sourceResults.source_hashes?.["intervention:inputs"] !== manifest.inputDigest)
|
|
13
|
+
throw new Error("recorded frozen intervention binding missing; never stamp historical results");
|
|
14
|
+
if (sourceResults.scenarios.length !== manifest.cases.length || sourceResults.scenarios.some(s => !manifest.cases.some(c => c.id === s.id)))
|
|
15
|
+
throw new Error("intervention scenario plan mismatch");
|
|
16
|
+
const cells = [], artifacts = new Set();
|
|
17
|
+
for (const spec of manifest.cases) {
|
|
18
|
+
const scenario = sourceResults.scenarios.find(s => s.id === spec.id);
|
|
19
|
+
if (scenario.criterion_count !== spec.criteria || (scenario.reps ?? 1) !== spec.reps || (scenario.pass_threshold ?? 1) !== spec.threshold)
|
|
20
|
+
throw new Error("intervention repetition/criterion policy mismatch");
|
|
21
|
+
for (const panel of scenario.rep_judgments ?? []) {
|
|
22
|
+
const delivery = sourceResults.subject_invocations.find(o => o.scenario_id === spec.id && o.repetition === panel.repetition)?.prompt?.status;
|
|
23
|
+
if (!delivery || !panel.objective)
|
|
24
|
+
throw new Error("intervention objective/delivery evidence missing");
|
|
25
|
+
const objective = panel.objective.status;
|
|
26
|
+
const bytes = outputBytes.get(`${spec.id}:${panel.repetition}`);
|
|
27
|
+
if (bytes !== undefined && (!(bytes instanceof Uint8Array) || bytes.byteLength > 8 * 1024 * 1024))
|
|
28
|
+
throw new Error("intervention output exceeds bounds");
|
|
29
|
+
const artifactSha256 = bytes ? createHash("sha256").update(bytes).digest("hex") : null;
|
|
30
|
+
if (artifactSha256)
|
|
31
|
+
artifacts.add(artifactSha256);
|
|
32
|
+
let criteria = [], suspect = false;
|
|
33
|
+
if (delivery === "PASS" && objective === "PASS") {
|
|
34
|
+
if (panel.judgments.length !== 1)
|
|
35
|
+
throw new Error("multi-judge normalization not yet supported; retain original panel without selecting a convenient vote");
|
|
36
|
+
const judgment = panel.judgments[0];
|
|
37
|
+
if (`${judgment.judge.provider}:${judgment.judge.model}` !== manifest.judge)
|
|
38
|
+
throw new Error("intervention cell judge mismatch");
|
|
39
|
+
criteria = [...(judgment.criteria ?? [])].sort((a, b) => a.index - b.index).map(c => c.verdict === "ERROR" ? "UNKNOWN" : c.verdict);
|
|
40
|
+
suspect = judgment.suspect || scenario.suspect;
|
|
41
|
+
}
|
|
42
|
+
cells.push({ caseId: spec.id, repetition: panel.repetition, delivery, objective, criteria, suspect, artifactSha256 });
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
const metrics = aggregateMetrics(sourceResults.scenarios);
|
|
46
|
+
const cost = manifest.resourceMetric === "wall_ms" ? (sourceResults.scenarios.every(s => s.metrics) ? metrics.wall_time_ms : null)
|
|
47
|
+
: manifest.resourceMetric === "tool_calls" ? metrics.tool_calls
|
|
48
|
+
: metrics.cost_source === "provider-reported" && metrics.subject_metrics_reps === metrics.total_reps ? metrics.subject_cost_usd : null;
|
|
49
|
+
const evidence = { armId, inputDigest: manifest.inputDigest, artifactDigests: [...artifacts].sort(), cells, cost, costUnit: manifest.resourceMetric };
|
|
50
|
+
return { evidence, sourceResults };
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=intervention-results.js.map
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/** Plain, bounded JSON encoding for retained intervention inputs; not semantic validation. */
|
|
2
|
+
export declare function interventionCanonicalJson(value: unknown): string;
|
|
3
|
+
export type InterventionAxis = "model" | "effort" | "skill" | "prompt" | "configuration";
|
|
4
|
+
export interface InterventionDraft {
|
|
5
|
+
family: "intervention";
|
|
6
|
+
investigationSha256: string;
|
|
7
|
+
resourceMetric: "usd" | "wall_ms" | "tool_calls";
|
|
8
|
+
axes: InterventionAxis[];
|
|
9
|
+
common: {
|
|
10
|
+
mode: "green" | "force";
|
|
11
|
+
scenarioSha256: string;
|
|
12
|
+
rubricSha256: string;
|
|
13
|
+
fixtureSha256: string;
|
|
14
|
+
heldoutSha256: string;
|
|
15
|
+
harnessSha256: string;
|
|
16
|
+
judgePolicySha256: string;
|
|
17
|
+
};
|
|
18
|
+
proposer: string;
|
|
19
|
+
judge: string;
|
|
20
|
+
cases: Array<{
|
|
21
|
+
id: string;
|
|
22
|
+
criteria: number;
|
|
23
|
+
reps: number;
|
|
24
|
+
threshold: number;
|
|
25
|
+
critical: boolean;
|
|
26
|
+
}>;
|
|
27
|
+
arms: Array<{
|
|
28
|
+
id: string;
|
|
29
|
+
configuration: Record<InterventionAxis, string>;
|
|
30
|
+
}>;
|
|
31
|
+
}
|
|
32
|
+
export interface InterventionManifest extends InterventionDraft {
|
|
33
|
+
version: "intervention-comparison-v1";
|
|
34
|
+
id: string;
|
|
35
|
+
inputDigest: string;
|
|
36
|
+
changedAxes: InterventionAxis[];
|
|
37
|
+
deterministicSampling: false;
|
|
38
|
+
}
|
|
39
|
+
export declare function freezeIntervention(input: InterventionDraft): InterventionManifest;
|
|
40
|
+
export interface InterventionCell {
|
|
41
|
+
caseId: string;
|
|
42
|
+
repetition: number;
|
|
43
|
+
delivery: "PASS" | "NOT-MEASURED" | "ERROR";
|
|
44
|
+
objective: "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED";
|
|
45
|
+
criteria: Array<"PASS" | "FAIL" | "UNKNOWN">;
|
|
46
|
+
suspect: boolean;
|
|
47
|
+
artifactSha256: string | null;
|
|
48
|
+
}
|
|
49
|
+
export interface InterventionEvidence {
|
|
50
|
+
armId: string;
|
|
51
|
+
inputDigest: string;
|
|
52
|
+
artifactDigests: string[];
|
|
53
|
+
cells: InterventionCell[];
|
|
54
|
+
cost: number | null;
|
|
55
|
+
costUnit: string;
|
|
56
|
+
}
|
|
57
|
+
export declare const interventionEvidenceDigest: (value: InterventionEvidence) => string;
|
|
58
|
+
export interface InterventionQualification {
|
|
59
|
+
manifestId: string;
|
|
60
|
+
proposer: {
|
|
61
|
+
requested: string;
|
|
62
|
+
canonical: string;
|
|
63
|
+
};
|
|
64
|
+
judge: {
|
|
65
|
+
requested: string;
|
|
66
|
+
canonical: string;
|
|
67
|
+
};
|
|
68
|
+
subjects: Record<string, {
|
|
69
|
+
requested: string;
|
|
70
|
+
canonical: string;
|
|
71
|
+
}>;
|
|
72
|
+
evidenceDigests: Record<string, string>;
|
|
73
|
+
/** Actual independently retained output bytes, not an availability flag supplied by an arm. */
|
|
74
|
+
artifacts: ReadonlyMap<string, Uint8Array>;
|
|
75
|
+
}
|
|
76
|
+
export interface InterventionAssessment {
|
|
77
|
+
manifestId: string;
|
|
78
|
+
complete: boolean;
|
|
79
|
+
arms: Array<{
|
|
80
|
+
armId: string;
|
|
81
|
+
state: "MEASURED" | "FAILED" | "MISSING" | "ERROR" | "NOT-MEASURED" | "UNRESOLVED";
|
|
82
|
+
eligible: boolean;
|
|
83
|
+
artifactDigests: string[];
|
|
84
|
+
cost: number | null;
|
|
85
|
+
costUnit: string;
|
|
86
|
+
}>;
|
|
87
|
+
cheapestEligible: string | null;
|
|
88
|
+
routingDefault: null;
|
|
89
|
+
adoptionAuthorized: false;
|
|
90
|
+
}
|
|
91
|
+
/** Pre-spend role gate: consumes independently resolved host facts, never guesses aliases or calls a provider. */
|
|
92
|
+
export declare function assertInterventionRoles(manifest: InterventionManifest, qualification: InterventionQualification | null): asserts qualification is InterventionQualification;
|
|
93
|
+
/** Inspects separately host-qualified evidence; does not obtain qualification or execute a model. */
|
|
94
|
+
export declare function assessIntervention(manifest: InterventionManifest, evidence: readonly InterventionEvidence[], qualification: InterventionQualification | null): InterventionAssessment;
|
|
95
|
+
export type BlindQualityChoice = {
|
|
96
|
+
kind: "one" | "tie" | "none" | "insufficient";
|
|
97
|
+
labels: string[];
|
|
98
|
+
};
|
|
99
|
+
/** Only this view omits identity/cost; arbitrary artifact content can still leak clues and must be reviewed. */
|
|
100
|
+
export declare function createBlindComparison(manifest: InterventionManifest, assessment: InterventionAssessment, fixtureOrPersistedSeed?: string): Readonly<{
|
|
101
|
+
view: () => {
|
|
102
|
+
version: string;
|
|
103
|
+
cards: {
|
|
104
|
+
label: string;
|
|
105
|
+
artifactDigests: string[];
|
|
106
|
+
}[];
|
|
107
|
+
limitations: string[];
|
|
108
|
+
};
|
|
109
|
+
readArtifact(label: string, hash: string): Buffer<ArrayBuffer>;
|
|
110
|
+
choose(input: BlindQualityChoice): BlindQualityChoice;
|
|
111
|
+
reveal(): {
|
|
112
|
+
manifestId: string;
|
|
113
|
+
choice: BlindQualityChoice;
|
|
114
|
+
arms: {
|
|
115
|
+
label: string;
|
|
116
|
+
armId: string;
|
|
117
|
+
configuration: Record<InterventionAxis, string>;
|
|
118
|
+
cost: number | null;
|
|
119
|
+
costUnit: string;
|
|
120
|
+
}[];
|
|
121
|
+
routingDefault: null;
|
|
122
|
+
};
|
|
123
|
+
}>;
|