@skill-harness/core 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/types.d.ts +19 -0
- package/dist/adjudication.d.ts +9 -1
- package/dist/adjudication.js +41 -33
- package/dist/arms.d.ts +1 -1
- package/dist/capture-trace-types.d.ts +2 -0
- package/dist/command-cost.d.ts +9 -0
- package/dist/command-cost.js +23 -0
- package/dist/comparison.d.ts +1 -0
- package/dist/comparison.js +9 -3
- package/dist/execution-trace.js +9 -1
- package/dist/grade.d.ts +2 -0
- package/dist/grade.js +2 -1
- package/dist/index.d.ts +12 -0
- package/dist/index.js +12 -0
- package/dist/journal.d.ts +1 -1
- package/dist/judge-agreement.d.ts +19 -0
- package/dist/judge-agreement.js +31 -0
- package/dist/judge-policy.js +5 -3
- package/dist/lift.js +1 -1
- package/dist/lint.js +1 -1
- package/dist/prompt-normalization.d.ts +6 -0
- package/dist/prompt-normalization.js +14 -0
- package/dist/qualification-capture.d.ts +44 -0
- package/dist/qualification-capture.js +163 -0
- package/dist/qualification-config.d.ts +233 -0
- package/dist/qualification-config.js +694 -0
- package/dist/qualification-lock.d.ts +15 -0
- package/dist/qualification-lock.js +87 -0
- package/dist/qualification-oauth-directory.d.ts +79 -0
- package/dist/qualification-oauth-directory.js +444 -0
- package/dist/qualification-panel-store.d.ts +14 -0
- package/dist/qualification-panel-store.js +188 -0
- package/dist/qualification-panels.d.ts +99 -0
- package/dist/qualification-panels.js +91 -0
- package/dist/qualification-process.d.ts +13 -0
- package/dist/qualification-process.js +188 -0
- package/dist/qualification-runner.d.ts +203 -0
- package/dist/qualification-runner.js +2037 -0
- package/dist/qualification-store.d.ts +187 -0
- package/dist/qualification-store.js +833 -0
- package/dist/qualification-validation.d.ts +12 -0
- package/dist/qualification-validation.js +8 -0
- package/dist/regate.js +42 -10
- package/dist/regrade.d.ts +3 -1
- package/dist/regrade.js +37 -5
- package/dist/reps.d.ts +4 -2
- package/dist/reps.js +13 -4
- package/dist/rescore.js +2 -1
- package/dist/results.d.ts +53 -7
- package/dist/results.js +210 -7
- package/dist/run.js +66 -17
- package/dist/score.d.ts +4 -3
- package/dist/score.js +13 -4
- package/dist/screen.d.ts +31 -0
- package/dist/screen.js +105 -0
- package/dist/seeded.d.ts +2 -1
- package/dist/seeded.js +1 -0
- package/dist/sources.d.ts +7 -1
- package/dist/sources.js +13 -5
- package/dist/spec.d.ts +1 -1
- package/dist/spec.js +1 -1
- package/dist/stability.js +1 -1
- package/dist/trajectory-gates.d.ts +14 -2
- package/dist/trajectory-gates.js +26 -6
- package/dist/vote-panel.d.ts +33 -0
- package/dist/vote-panel.js +41 -0
- package/dist/workspace.js +4 -1
- package/package.json +3 -3
- package/schemas/qualification-cell-result-v1.schema.json +32 -0
- package/schemas/qualification-config-v1.schema.json +244 -0
- package/schemas/qualification-invocation-request-v1.schema.json +59 -0
- package/schemas/qualification-judge-panel-result-v1.schema.json +42 -0
- package/schemas/results-v2.schema.json +19 -21
- package/schemas/results-v3.schema.json +654 -0
- package/schemas/specification-v1.schema.json +1 -1
- package/schemas/trajectory-event-v1.schema.json +20 -1
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { QUALIFICATION_OAUTH_DIRECTORY_POLICY_V2, QUALIFICATION_TERMINAL_RECEIPT_VERSION_V3, type QualificationArmV1, type QualificationConfigV1, type QualificationInvocationRequestV1, type QualificationRole, type QualificationAccountingPolicy, verifyQualificationExecutable } from "./qualification-config.js";
|
|
2
|
+
export declare const QUALIFICATION_INVOCATION_VERSION: "qualification-invocation-v1";
|
|
3
|
+
export declare const QUALIFICATION_INVOCATION_VERSION_V2: "qualification-invocation-v2";
|
|
4
|
+
export declare const QUALIFICATION_INVOCATION_VERSION_V3: "qualification-invocation-v3";
|
|
5
|
+
export declare const QUALIFICATION_INVOCATION_VERSION_V4: "qualification-invocation-v4";
|
|
6
|
+
export declare const QUALIFICATION_INVOCATION_INPUT_BINDING_VERSION: "qualification-invocation-input-binding-v1";
|
|
7
|
+
export declare const QUALIFICATION_LIFECYCLE_VERSION: "qualification-lifecycle-v1";
|
|
8
|
+
export declare const QUALIFICATION_ACCOUNTING_VERSION: "qualification-accounting-v1";
|
|
9
|
+
export type QualificationLifecyclePhase = "prepared" | "launch-claimed" | "running" | "terminal";
|
|
10
|
+
export type QualificationTerminalStatus = "completed" | "failed" | "timed-out" | "aborted" | "refused" | "invalid-artifact";
|
|
11
|
+
export interface QualificationInvocationInputBindingV1 {
|
|
12
|
+
schema_version: typeof QUALIFICATION_INVOCATION_INPUT_BINDING_VERSION;
|
|
13
|
+
content_type: "application/json";
|
|
14
|
+
bytes: number;
|
|
15
|
+
sha256: string;
|
|
16
|
+
}
|
|
17
|
+
export interface QualificationInvocationInputOccurrenceV1 {
|
|
18
|
+
path: string;
|
|
19
|
+
realpath: string;
|
|
20
|
+
device: number;
|
|
21
|
+
inode: number;
|
|
22
|
+
mode: number;
|
|
23
|
+
uid: number;
|
|
24
|
+
gid: number;
|
|
25
|
+
link_count: number;
|
|
26
|
+
bytes: number;
|
|
27
|
+
sha256: string;
|
|
28
|
+
}
|
|
29
|
+
export interface QualificationInvocationV1 {
|
|
30
|
+
schema_version: typeof QUALIFICATION_INVOCATION_VERSION | typeof QUALIFICATION_INVOCATION_VERSION_V2 | typeof QUALIFICATION_INVOCATION_VERSION_V3 | typeof QUALIFICATION_INVOCATION_VERSION_V4;
|
|
31
|
+
/** Present only in v2/v3 records; omission retains the historical digest. */
|
|
32
|
+
oauth_directory_policy?: typeof QUALIFICATION_OAUTH_DIRECTORY_POLICY_V2;
|
|
33
|
+
/** Present only in v3 records; omission cannot be interpreted as v3. */
|
|
34
|
+
terminal_receipt_version?: typeof QUALIFICATION_TERMINAL_RECEIPT_VERSION_V3;
|
|
35
|
+
invocation_input?: QualificationInvocationInputBindingV1;
|
|
36
|
+
invocation_input_occurrence?: QualificationInvocationInputOccurrenceV1;
|
|
37
|
+
prepared_attempt?: 0;
|
|
38
|
+
accounting_event_sha256?: null;
|
|
39
|
+
invocation_id: string;
|
|
40
|
+
measurement_identity_sha256: string;
|
|
41
|
+
continuation_authority_sha256: string;
|
|
42
|
+
continuation_authority_expires_at: string;
|
|
43
|
+
scenario: QualificationInvocationRequestV1["scenario"];
|
|
44
|
+
role: QualificationRole;
|
|
45
|
+
counts_as_measurement: boolean;
|
|
46
|
+
arms: {
|
|
47
|
+
subject: string;
|
|
48
|
+
judge: string;
|
|
49
|
+
selected: string;
|
|
50
|
+
};
|
|
51
|
+
repetition: number;
|
|
52
|
+
/** Present only in v4 judge-panel member invocations. */
|
|
53
|
+
panel?: NonNullable<QualificationInvocationRequestV1["panel"]>;
|
|
54
|
+
requested: {
|
|
55
|
+
provider: string;
|
|
56
|
+
model: string;
|
|
57
|
+
};
|
|
58
|
+
authentication: QualificationArmV1["authentication"];
|
|
59
|
+
pins: {
|
|
60
|
+
product: QualificationConfigV1["product"];
|
|
61
|
+
engine: QualificationConfigV1["engine"];
|
|
62
|
+
producer: QualificationConfigV1["producer"];
|
|
63
|
+
runner: QualificationConfigV1["runner"];
|
|
64
|
+
};
|
|
65
|
+
configuration_sha256: string;
|
|
66
|
+
created_at: string;
|
|
67
|
+
ceiling_policy: QualificationAccountingPolicy;
|
|
68
|
+
execution: {
|
|
69
|
+
executable: QualificationArmV1["executable"];
|
|
70
|
+
executable_identity: ReturnType<typeof verifyQualificationExecutable>;
|
|
71
|
+
resources: QualificationArmV1["resources"];
|
|
72
|
+
working_directory_identity: {
|
|
73
|
+
realpath: string;
|
|
74
|
+
device: number;
|
|
75
|
+
inode: number;
|
|
76
|
+
};
|
|
77
|
+
arguments: string[];
|
|
78
|
+
allowed_environment_names: string[];
|
|
79
|
+
timeout_ms: number;
|
|
80
|
+
output_limit_bytes: number;
|
|
81
|
+
fallback: false;
|
|
82
|
+
metered_override: false;
|
|
83
|
+
};
|
|
84
|
+
expected_artifact: {
|
|
85
|
+
path: string;
|
|
86
|
+
type: "pi-jsonl";
|
|
87
|
+
};
|
|
88
|
+
/** Self-digest over every preceding field; detects accidental mutation. */
|
|
89
|
+
invocation_sha256: string;
|
|
90
|
+
}
|
|
91
|
+
export interface QualificationLifecycleEvent {
|
|
92
|
+
seq: number;
|
|
93
|
+
from: QualificationLifecyclePhase | null;
|
|
94
|
+
to: QualificationLifecyclePhase;
|
|
95
|
+
terminal_status: QualificationTerminalStatus | null;
|
|
96
|
+
at: string;
|
|
97
|
+
detail: Record<string, unknown>;
|
|
98
|
+
previous_sha256: string | null;
|
|
99
|
+
event_sha256: string;
|
|
100
|
+
}
|
|
101
|
+
export interface QualificationLifecycleV1 {
|
|
102
|
+
schema_version: typeof QUALIFICATION_LIFECYCLE_VERSION;
|
|
103
|
+
invocation_id: string;
|
|
104
|
+
phase: QualificationLifecyclePhase;
|
|
105
|
+
terminal_status: QualificationTerminalStatus | null;
|
|
106
|
+
events: QualificationLifecycleEvent[];
|
|
107
|
+
chain_head: string;
|
|
108
|
+
}
|
|
109
|
+
export interface QualificationAccountingEventInput {
|
|
110
|
+
invocation_id: string;
|
|
111
|
+
role: QualificationRole;
|
|
112
|
+
call_class: "subject" | "judge";
|
|
113
|
+
counts_as_measurement: boolean;
|
|
114
|
+
launched_at: string;
|
|
115
|
+
}
|
|
116
|
+
export interface QualificationAccountingEvent extends QualificationAccountingEventInput {
|
|
117
|
+
seq: number;
|
|
118
|
+
previous_sha256: string | null;
|
|
119
|
+
event_sha256: string;
|
|
120
|
+
}
|
|
121
|
+
export interface QualificationAccountingLedgerV1 {
|
|
122
|
+
schema_version: typeof QUALIFICATION_ACCOUNTING_VERSION;
|
|
123
|
+
policy: QualificationAccountingPolicy;
|
|
124
|
+
events: QualificationAccountingEvent[];
|
|
125
|
+
chain_head: string | null;
|
|
126
|
+
}
|
|
127
|
+
export interface QualificationAccountingReport {
|
|
128
|
+
counts: {
|
|
129
|
+
subject: number;
|
|
130
|
+
judge: number;
|
|
131
|
+
measurement: number;
|
|
132
|
+
total: number;
|
|
133
|
+
};
|
|
134
|
+
roles: Partial<Record<QualificationRole, number>>;
|
|
135
|
+
chain_head: string | null;
|
|
136
|
+
}
|
|
137
|
+
export interface QualificationSpoolValidation {
|
|
138
|
+
ok: true;
|
|
139
|
+
configuration_sha256: string;
|
|
140
|
+
invocations: number;
|
|
141
|
+
accounting: QualificationAccountingReport;
|
|
142
|
+
}
|
|
143
|
+
export declare function qualificationSpoolPaths(spoolDir: string, invocationId?: string): {
|
|
144
|
+
root: string;
|
|
145
|
+
configuration: string;
|
|
146
|
+
accounting: string;
|
|
147
|
+
lock: string;
|
|
148
|
+
invocations: string;
|
|
149
|
+
artifacts: string;
|
|
150
|
+
invocation: string | undefined;
|
|
151
|
+
invocationRecord: string | undefined;
|
|
152
|
+
lifecycle: string | undefined;
|
|
153
|
+
invocationLock: string | undefined;
|
|
154
|
+
terminal: string | undefined;
|
|
155
|
+
};
|
|
156
|
+
export declare function prepareQualificationInvocation(options: {
|
|
157
|
+
spool_dir: string;
|
|
158
|
+
config_path: string;
|
|
159
|
+
request_path: string;
|
|
160
|
+
expected_configuration_sha256?: string;
|
|
161
|
+
now?: () => string;
|
|
162
|
+
/** Deterministic source-race injection for offline tests only. */
|
|
163
|
+
test_hooks?: {
|
|
164
|
+
after_initial_input_validation?: () => void;
|
|
165
|
+
};
|
|
166
|
+
}): QualificationInvocationV1;
|
|
167
|
+
export declare function readQualificationInvocation(spoolDir: string, invocationId: string): QualificationInvocationV1;
|
|
168
|
+
export declare function readQualificationLifecycle(spoolDir: string, invocationId: string): QualificationLifecycleV1;
|
|
169
|
+
export declare function writeQualificationLifecycle(spoolDir: string, lifecycle: QualificationLifecycleV1): void;
|
|
170
|
+
export declare function transitionQualificationLifecycle(lifecycle: QualificationLifecycleV1, to: QualificationLifecyclePhase, options: {
|
|
171
|
+
at: string;
|
|
172
|
+
terminal_status?: QualificationTerminalStatus | null;
|
|
173
|
+
detail?: Record<string, unknown>;
|
|
174
|
+
}): QualificationLifecycleV1;
|
|
175
|
+
export declare function createQualificationAccountingLedger(policy?: QualificationAccountingPolicy): QualificationAccountingLedgerV1;
|
|
176
|
+
export declare function appendQualificationAccountingEvent(inputLedger: QualificationAccountingLedgerV1, input: QualificationAccountingEventInput): QualificationAccountingLedgerV1;
|
|
177
|
+
export declare function validateQualificationAccountingLedger(value: unknown): QualificationAccountingReport;
|
|
178
|
+
export declare function readQualificationAccounting(spoolDir: string): QualificationAccountingLedgerV1;
|
|
179
|
+
export declare function writeQualificationAccounting(spoolDir: string, ledger: QualificationAccountingLedgerV1): void;
|
|
180
|
+
export declare function validateQualificationSpool(spoolDir: string): QualificationSpoolValidation;
|
|
181
|
+
export declare function readQualificationSpoolConfig(spoolDir: string): QualificationConfigV1;
|
|
182
|
+
export declare function withQualificationFileLock<T>(lockPath: string, action: () => T): T;
|
|
183
|
+
export declare function atomicWriteCanonical(path: string, value: unknown, exclusive: boolean): void;
|
|
184
|
+
export declare function atomicWriteBytes(path: string, bytes: Buffer, exclusive: boolean): void;
|
|
185
|
+
export declare function readCanonicalJson(path: string, ctx: string): unknown;
|
|
186
|
+
export declare function verifyQualificationInvocationInput(invocation: QualificationInvocationV1): QualificationInvocationInputBindingV1;
|
|
187
|
+
export declare function fsyncQualificationDirectory(path: string): void;
|