@skill-harness/core 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/dist/adapters/types.d.ts +0 -12
- package/dist/capture-trace-types.d.ts +0 -96
- package/dist/capture-trace-types.js +0 -4
- package/dist/command-cost.d.ts +1 -1
- package/dist/command-cost.js +0 -5
- package/dist/execution-trace.js +1 -1
- package/dist/index.d.ts +3 -23
- package/dist/index.js +3 -23
- package/dist/instruction-coverage.d.ts +0 -7
- package/dist/instruction-coverage.js +5 -7
- package/dist/journal.d.ts +0 -8
- package/dist/metrics.d.ts +18 -0
- package/dist/metrics.js +28 -0
- package/dist/redaction.d.ts +6 -0
- package/dist/redaction.js +55 -0
- package/dist/regate.js +6 -71
- package/dist/regrade.d.ts +1 -3
- package/dist/regrade.js +1 -21
- package/dist/report.d.ts +1 -15
- package/dist/report.js +1 -13
- package/dist/reps.d.ts +1 -2
- package/dist/reps.js +0 -2
- package/dist/results.d.ts +0 -9
- package/dist/results.js +2 -19
- package/dist/run.js +11 -72
- package/dist/seeded.d.ts +1 -7
- package/dist/seeded.js +5 -11
- package/dist/sources.js +4 -9
- package/dist/spec.d.ts +0 -13
- package/dist/spec.js +6 -48
- package/dist/trajectory-events.d.ts +58 -0
- package/dist/trajectory-events.js +119 -0
- package/package.json +1 -1
- package/schemas/results-v2.schema.json +376 -75
- package/schemas/specification-v1.schema.json +128 -174
- package/dist/adjudication.d.ts +0 -219
- package/dist/adjudication.js +0 -422
- package/dist/adoption.d.ts +0 -128
- package/dist/adoption.js +0 -102
- package/dist/affected.d.ts +0 -88
- package/dist/affected.js +0 -222
- package/dist/capture.d.ts +0 -193
- package/dist/capture.js +0 -344
- package/dist/comparison.d.ts +0 -97
- package/dist/comparison.js +0 -189
- package/dist/factory-calibration.d.ts +0 -63
- package/dist/factory-calibration.js +0 -115
- package/dist/intervention-results.d.ts +0 -6
- package/dist/intervention-results.js +0 -52
- package/dist/intervention.d.ts +0 -123
- package/dist/intervention.js +0 -218
- package/dist/investigation.d.ts +0 -94
- package/dist/investigation.js +0 -148
- package/dist/judge-agreement.d.ts +0 -19
- package/dist/judge-agreement.js +0 -31
- package/dist/qualification-capture.d.ts +0 -44
- package/dist/qualification-capture.js +0 -163
- package/dist/qualification-config.d.ts +0 -233
- package/dist/qualification-config.js +0 -694
- package/dist/qualification-lock.d.ts +0 -15
- package/dist/qualification-lock.js +0 -87
- package/dist/qualification-oauth-directory.d.ts +0 -79
- package/dist/qualification-oauth-directory.js +0 -444
- package/dist/qualification-panel-store.d.ts +0 -14
- package/dist/qualification-panel-store.js +0 -188
- package/dist/qualification-panels.d.ts +0 -99
- package/dist/qualification-panels.js +0 -91
- package/dist/qualification-process.d.ts +0 -13
- package/dist/qualification-process.js +0 -190
- package/dist/qualification-runner.d.ts +0 -203
- package/dist/qualification-runner.js +0 -2037
- package/dist/qualification-store.d.ts +0 -187
- package/dist/qualification-store.js +0 -833
- package/dist/qualification-validation.d.ts +0 -12
- package/dist/qualification-validation.js +0 -8
- package/dist/screen.d.ts +0 -31
- package/dist/screen.js +0 -105
- package/dist/trajectory-gates.d.ts +0 -137
- package/dist/trajectory-gates.js +0 -576
- package/dist/trusted-host-supervision.d.ts +0 -25
- package/dist/trusted-host-supervision.js +0 -94
- package/dist/work-capture-schema.d.ts +0 -38
- package/dist/work-capture-schema.js +0 -33
- package/dist/work-capture.d.ts +0 -43
- package/dist/work-capture.js +0 -81
- package/dist/work-signals.d.ts +0 -55
- package/dist/work-signals.js +0 -71
- package/schemas/qualification-cell-result-v1.schema.json +0 -32
- package/schemas/qualification-config-v1.schema.json +0 -244
- package/schemas/qualification-invocation-request-v1.schema.json +0 -59
- package/schemas/qualification-judge-panel-result-v1.schema.json +0 -42
- package/schemas/trajectory-event-v1.schema.json +0 -80
package/dist/spec.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
2
|
import yaml from "js-yaml";
|
|
3
3
|
import { parseTraceAssert } from "./trace-gates.js";
|
|
4
|
-
import { parseTrajectoryAssert } from "./trajectory-gates.js";
|
|
5
4
|
/** Thrown on any validation failure. Message always carries the spec file path. */
|
|
6
5
|
export class SpecError extends Error {
|
|
7
6
|
constructor(message, file) {
|
|
@@ -89,46 +88,6 @@ function resolveExtensions(env, hasSystemPrompt, id, file) {
|
|
|
89
88
|
* with empty-git or a fixture — asking for one on a bare cwd is an authoring mistake,
|
|
90
89
|
* not something to silently ignore.
|
|
91
90
|
*/
|
|
92
|
-
function resolveEventSources(env, id, file) {
|
|
93
|
-
const raw = env && typeof env === "object" ? env.event_sources : undefined;
|
|
94
|
-
if (raw === undefined)
|
|
95
|
-
return undefined;
|
|
96
|
-
if (!Array.isArray(raw) || raw.length === 0) {
|
|
97
|
-
throw new SpecError(`scenario \`${id}\` env.event_sources must be a non-empty list`, file);
|
|
98
|
-
}
|
|
99
|
-
const allowedAdapters = new Set([
|
|
100
|
-
"normalized-v1", "principal-assurance-v1", "pi-daddy-v1", "pi-daddy-ledger-v3",
|
|
101
|
-
]);
|
|
102
|
-
return raw.map((entry, index) => {
|
|
103
|
-
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
|
104
|
-
throw new SpecError(`scenario \`${id}\` env.event_sources[${index}] must be a mapping`, file);
|
|
105
|
-
}
|
|
106
|
-
const source = entry;
|
|
107
|
-
for (const key of Object.keys(source)) {
|
|
108
|
-
if (!["adapter", "path", "required"].includes(key)) {
|
|
109
|
-
throw new SpecError(`scenario \`${id}\` env.event_sources[${index}] has unknown key \`${key}\``, file);
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
if (!allowedAdapters.has(source.adapter)) {
|
|
113
|
-
throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].adapter is unsupported`, file);
|
|
114
|
-
}
|
|
115
|
-
if (typeof source.path !== "string" || !source.path.trim()) {
|
|
116
|
-
throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].path must be non-empty`, file);
|
|
117
|
-
}
|
|
118
|
-
const path = source.path.trim();
|
|
119
|
-
if (path.startsWith("/") || /^[A-Za-z]:[\\/]/.test(path) || path.includes("\\") || path.split("/").some((part) => part === ".." || part === "." || part === "")) {
|
|
120
|
-
throw new SpecError(`scenario \`${id}\` env.event_sources paths must be workspace-relative without traversal (got \`${path}\`)`, file);
|
|
121
|
-
}
|
|
122
|
-
if (source.required !== undefined && typeof source.required !== "boolean") {
|
|
123
|
-
throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].required must be true or false`, file);
|
|
124
|
-
}
|
|
125
|
-
return {
|
|
126
|
-
adapter: source.adapter,
|
|
127
|
-
path,
|
|
128
|
-
required: source.required !== false,
|
|
129
|
-
};
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
91
|
function resolveRemote(env, workspace, id, file) {
|
|
133
92
|
const raw = env && typeof env === "object" ? env.remote : undefined;
|
|
134
93
|
if (raw === undefined)
|
|
@@ -220,12 +179,12 @@ export function parseSpec(text, file) {
|
|
|
220
179
|
// `assert.trace` is legal for inline AND seeded scenarios — it reads the
|
|
221
180
|
// execution trace, which every run produces, not a staged diff.
|
|
222
181
|
const rawAssert = s.assert;
|
|
182
|
+
if (rawAssert?.trajectory !== undefined) {
|
|
183
|
+
throw new SpecError(`scenario \`${id}\` uses removed \`assert.trajectory\`; delete it or replace it with an active objective gate`, file);
|
|
184
|
+
}
|
|
223
185
|
if (rawAssert?.trace !== undefined) {
|
|
224
186
|
scenario.traceAssert = parseTraceAssert(rawAssert.trace, `${file}: scenario \`${id}\``);
|
|
225
187
|
}
|
|
226
|
-
if (rawAssert?.trajectory !== undefined) {
|
|
227
|
-
scenario.trajectoryAssert = parseTrajectoryAssert(rawAssert.trajectory, `${file}: scenario \`${id}\``);
|
|
228
|
-
}
|
|
229
188
|
if (mode === "seeded") {
|
|
230
189
|
if (typeof s.fixture !== "string" || s.fixture.length === 0) {
|
|
231
190
|
throw new SpecError(`seeded scenario \`${id}\` requires a \`fixture\` path`, file);
|
|
@@ -276,12 +235,11 @@ export function parseSpec(text, file) {
|
|
|
276
235
|
scenario.assert = assertObj;
|
|
277
236
|
}
|
|
278
237
|
}
|
|
238
|
+
if (s.env && typeof s.env === "object" && Object.hasOwn(s.env, "event_sources")) {
|
|
239
|
+
throw new SpecError(`scenario \`${id}\` uses removed \`env.event_sources\`; it is no longer collected`, file);
|
|
240
|
+
}
|
|
279
241
|
scenario.workspace = resolveWorkspace(s.env, mode, scenario.fixture, id, file);
|
|
280
242
|
scenario.remote = resolveRemote(s.env, scenario.workspace, id, file);
|
|
281
|
-
scenario.eventSources = resolveEventSources(s.env, id, file);
|
|
282
|
-
if (scenario.eventSources && !scenario.trajectoryAssert) {
|
|
283
|
-
throw new SpecError(`scenario \`${id}\` declares env.event_sources without assert.trajectory`, file);
|
|
284
|
-
}
|
|
285
243
|
if (s.system_prompt_file !== undefined) {
|
|
286
244
|
if (typeof s.system_prompt_file !== "string" || !s.system_prompt_file.trim()) {
|
|
287
245
|
throw new SpecError(`scenario \`${id}\` \`system_prompt_file\` must be a non-empty string`, file);
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
export declare const LEGACY_TRAJECTORY_EVENT_VERSION: "1.0";
|
|
2
|
+
export declare const TRAJECTORY_EVENT_VERSION: "1.1";
|
|
3
|
+
export type TrajectoryEventVersion = typeof LEGACY_TRAJECTORY_EVENT_VERSION | typeof TRAJECTORY_EVENT_VERSION;
|
|
4
|
+
export interface TrajectoryEventSource {
|
|
5
|
+
adapter: "normalized-v1" | "principal-assurance-v1" | "pi-daddy-v1" | "pi-daddy-ledger-v3";
|
|
6
|
+
path: string;
|
|
7
|
+
required: boolean;
|
|
8
|
+
}
|
|
9
|
+
export interface TrajectoryApproval {
|
|
10
|
+
id?: string;
|
|
11
|
+
capability?: string;
|
|
12
|
+
subject?: string;
|
|
13
|
+
source?: string;
|
|
14
|
+
scope?: string;
|
|
15
|
+
approved_at?: string;
|
|
16
|
+
expires_at?: string;
|
|
17
|
+
used_at?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface TrajectoryDigests {
|
|
20
|
+
plan?: string;
|
|
21
|
+
task?: string;
|
|
22
|
+
definition?: string;
|
|
23
|
+
head?: string;
|
|
24
|
+
tree?: string;
|
|
25
|
+
[key: string]: string | undefined;
|
|
26
|
+
}
|
|
27
|
+
/** Normalized pi-daddy/principal event retained by adapter consumers. */
|
|
28
|
+
export interface TrajectoryEventV1 {
|
|
29
|
+
event_version: TrajectoryEventVersion;
|
|
30
|
+
seq: number;
|
|
31
|
+
type: string;
|
|
32
|
+
source: string;
|
|
33
|
+
at?: string;
|
|
34
|
+
run_id?: string;
|
|
35
|
+
task_id?: string;
|
|
36
|
+
workspace_id?: string;
|
|
37
|
+
context_id?: string;
|
|
38
|
+
finding_id?: string;
|
|
39
|
+
parent_id?: string;
|
|
40
|
+
child_id?: string;
|
|
41
|
+
execution_id?: string;
|
|
42
|
+
parent_execution_id?: string | null;
|
|
43
|
+
task_from_execution_id?: string;
|
|
44
|
+
workflow_fact_id?: string;
|
|
45
|
+
deadline_at?: string;
|
|
46
|
+
phase?: string;
|
|
47
|
+
tool?: string;
|
|
48
|
+
capability?: string;
|
|
49
|
+
requested_capabilities?: string[];
|
|
50
|
+
effective_capabilities?: string[];
|
|
51
|
+
refusal_code?: string;
|
|
52
|
+
exit_code?: number;
|
|
53
|
+
digests?: TrajectoryDigests;
|
|
54
|
+
approval?: TrajectoryApproval;
|
|
55
|
+
requirements?: string[];
|
|
56
|
+
attributes?: Record<string, unknown>;
|
|
57
|
+
}
|
|
58
|
+
export declare function deserializeTrajectoryEvents(text: string): TrajectoryEventV1[] | null;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
export const LEGACY_TRAJECTORY_EVENT_VERSION = "1.0";
|
|
2
|
+
export const TRAJECTORY_EVENT_VERSION = "1.1";
|
|
3
|
+
export function deserializeTrajectoryEvents(text) {
|
|
4
|
+
const out = [];
|
|
5
|
+
try {
|
|
6
|
+
for (const line of text.split("\n").filter((entry) => entry.trim())) {
|
|
7
|
+
const event = JSON.parse(line);
|
|
8
|
+
if (validateEvent(event) !== null)
|
|
9
|
+
return null;
|
|
10
|
+
out.push(event);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
if (!out.length)
|
|
17
|
+
return null;
|
|
18
|
+
const sequences = out.map((event) => event.seq).sort((a, b) => a - b);
|
|
19
|
+
if (new Set(sequences).size !== out.length || sequences.some((seq, index) => seq !== index + 1))
|
|
20
|
+
return null;
|
|
21
|
+
return out;
|
|
22
|
+
}
|
|
23
|
+
const V11_EVENT_KEYS = new Set(["execution_id", "parent_execution_id", "task_from_execution_id", "workflow_fact_id", "deadline_at"]);
|
|
24
|
+
const EVENT_KEYS = new Set([
|
|
25
|
+
"event_version", "seq", "type", "source", "at", "run_id", "task_id", "workspace_id", "context_id", "finding_id",
|
|
26
|
+
"parent_id", "child_id", ...V11_EVENT_KEYS,
|
|
27
|
+
"phase", "tool", "capability", "requested_capabilities", "effective_capabilities", "refusal_code",
|
|
28
|
+
"exit_code", "digests", "approval", "requirements", "attributes",
|
|
29
|
+
]);
|
|
30
|
+
const APPROVAL_KEYS = new Set(["id", "capability", "subject", "source", "scope", "approved_at", "expires_at", "used_at"]);
|
|
31
|
+
const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
|
32
|
+
const SHA256_RE = /^[a-fA-F0-9]{64}$/;
|
|
33
|
+
const GIT_SHA_RE = /^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/;
|
|
34
|
+
const REFUSAL_RE = /^[A-Z][A-Z0-9_]*$/;
|
|
35
|
+
function validateEvent(event) {
|
|
36
|
+
if (!event || typeof event !== "object" || Array.isArray(event))
|
|
37
|
+
return "event must be an object";
|
|
38
|
+
const object = event;
|
|
39
|
+
const unknown = Object.keys(object).find((key) => !EVENT_KEYS.has(key));
|
|
40
|
+
if (unknown)
|
|
41
|
+
return `unknown field ${unknown}`;
|
|
42
|
+
if (event.event_version !== LEGACY_TRAJECTORY_EVENT_VERSION && event.event_version !== TRAJECTORY_EVENT_VERSION)
|
|
43
|
+
return `unsupported event_version ${String(event.event_version)}`;
|
|
44
|
+
if (event.event_version === LEGACY_TRAJECTORY_EVENT_VERSION) {
|
|
45
|
+
const versionedField = Object.keys(object).find((key) => V11_EVENT_KEYS.has(key));
|
|
46
|
+
if (versionedField)
|
|
47
|
+
return `${versionedField} requires event_version ${TRAJECTORY_EVENT_VERSION}`;
|
|
48
|
+
}
|
|
49
|
+
if (!Number.isInteger(event.seq) || event.seq < 1)
|
|
50
|
+
return "seq must be a positive integer";
|
|
51
|
+
if (typeof event.type !== "string" || !event.type)
|
|
52
|
+
return "type must be a non-empty string";
|
|
53
|
+
if (typeof event.source !== "string" || !event.source)
|
|
54
|
+
return "source must be a non-empty string";
|
|
55
|
+
if (event.at !== undefined && !validDate(event.at))
|
|
56
|
+
return "at must be an RFC 3339 date-time";
|
|
57
|
+
for (const field of ["run_id", "task_id", "workspace_id", "context_id", "finding_id", "parent_id", "child_id", "execution_id", "task_from_execution_id", "workflow_fact_id"]) {
|
|
58
|
+
if (event[field] !== undefined && (typeof event[field] !== "string" || !ID_RE.test(event[field])))
|
|
59
|
+
return `${field} is not a valid bounded identifier`;
|
|
60
|
+
}
|
|
61
|
+
if (event.parent_execution_id !== undefined && event.parent_execution_id !== null && (typeof event.parent_execution_id !== "string" || !ID_RE.test(event.parent_execution_id)))
|
|
62
|
+
return "parent_execution_id is not a valid bounded identifier or null";
|
|
63
|
+
if (event.deadline_at !== undefined && !validDate(event.deadline_at))
|
|
64
|
+
return "deadline_at must be an RFC 3339 date-time";
|
|
65
|
+
for (const field of ["phase", "tool", "capability"])
|
|
66
|
+
if (event[field] !== undefined && (typeof event[field] !== "string" || !event[field]))
|
|
67
|
+
return `${field} must be a non-empty string`;
|
|
68
|
+
for (const field of ["requested_capabilities", "effective_capabilities", "requirements"]) {
|
|
69
|
+
const values = event[field];
|
|
70
|
+
if (values !== undefined && (!Array.isArray(values) || values.some((value) => typeof value !== "string" || !value) || new Set(values).size !== values.length))
|
|
71
|
+
return `${field} must be an array of unique non-empty strings`;
|
|
72
|
+
}
|
|
73
|
+
if (event.refusal_code !== undefined && !REFUSAL_RE.test(event.refusal_code))
|
|
74
|
+
return "refusal_code is invalid";
|
|
75
|
+
if (event.exit_code !== undefined && !Number.isInteger(event.exit_code))
|
|
76
|
+
return "exit_code must be an integer";
|
|
77
|
+
if (event.digests !== undefined) {
|
|
78
|
+
if (!event.digests || typeof event.digests !== "object" || Array.isArray(event.digests))
|
|
79
|
+
return "digests must be an object";
|
|
80
|
+
for (const [key, value] of Object.entries(event.digests)) {
|
|
81
|
+
if (typeof value !== "string")
|
|
82
|
+
return `digests.${key} must be a string`;
|
|
83
|
+
if (["plan", "task", "definition"].includes(key) && !SHA256_RE.test(value))
|
|
84
|
+
return `digests.${key} must be sha256`;
|
|
85
|
+
if (["head", "tree"].includes(key) && !GIT_SHA_RE.test(value))
|
|
86
|
+
return `digests.${key} must be a git object id`;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (event.approval !== undefined) {
|
|
90
|
+
if (!event.approval || typeof event.approval !== "object" || Array.isArray(event.approval))
|
|
91
|
+
return "approval must be an object";
|
|
92
|
+
const unknownApproval = Object.keys(event.approval).find((key) => !APPROVAL_KEYS.has(key));
|
|
93
|
+
if (unknownApproval)
|
|
94
|
+
return `approval.${unknownApproval} is unknown`;
|
|
95
|
+
for (const [key, value] of Object.entries(event.approval)) {
|
|
96
|
+
if (typeof value !== "string" || !value)
|
|
97
|
+
return `approval.${key} must be a non-empty string`;
|
|
98
|
+
if (key === "id" && !ID_RE.test(value))
|
|
99
|
+
return "approval.id is invalid";
|
|
100
|
+
if (["approved_at", "expires_at", "used_at"].includes(key) && !validDate(value))
|
|
101
|
+
return `approval.${key} must be an RFC 3339 date-time`;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (event.attributes !== undefined && (!event.attributes || typeof event.attributes !== "object" || Array.isArray(event.attributes)))
|
|
105
|
+
return "attributes must be an object";
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
function validDate(value) {
|
|
109
|
+
if (typeof value !== "string")
|
|
110
|
+
return false;
|
|
111
|
+
const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.exec(value);
|
|
112
|
+
if (!match || !Number.isFinite(Date.parse(value)))
|
|
113
|
+
return false;
|
|
114
|
+
const [, year, month, day, hour, minute, second] = match.map(Number);
|
|
115
|
+
if (month < 1 || month > 12 || hour > 23 || minute > 59 || second > 59)
|
|
116
|
+
return false;
|
|
117
|
+
return day >= 1 && day <= new Date(Date.UTC(year, month, 0)).getUTCDate();
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=trajectory-events.js.map
|