@skill-harness/core 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +1 -2
  2. package/dist/adapters/types.d.ts +0 -12
  3. package/dist/capture-trace-types.d.ts +0 -96
  4. package/dist/capture-trace-types.js +0 -4
  5. package/dist/command-cost.d.ts +1 -1
  6. package/dist/command-cost.js +0 -5
  7. package/dist/execution-trace.js +1 -1
  8. package/dist/index.d.ts +3 -23
  9. package/dist/index.js +3 -23
  10. package/dist/instruction-coverage.d.ts +0 -7
  11. package/dist/instruction-coverage.js +5 -7
  12. package/dist/journal.d.ts +0 -8
  13. package/dist/metrics.d.ts +18 -0
  14. package/dist/metrics.js +28 -0
  15. package/dist/redaction.d.ts +6 -0
  16. package/dist/redaction.js +55 -0
  17. package/dist/regate.js +6 -71
  18. package/dist/regrade.d.ts +1 -3
  19. package/dist/regrade.js +1 -21
  20. package/dist/report.d.ts +1 -15
  21. package/dist/report.js +1 -13
  22. package/dist/reps.d.ts +1 -2
  23. package/dist/reps.js +0 -2
  24. package/dist/results.d.ts +0 -9
  25. package/dist/results.js +2 -19
  26. package/dist/run.js +11 -72
  27. package/dist/seeded.d.ts +1 -7
  28. package/dist/seeded.js +5 -11
  29. package/dist/sources.js +4 -9
  30. package/dist/spec.d.ts +0 -13
  31. package/dist/spec.js +6 -48
  32. package/dist/trajectory-events.d.ts +58 -0
  33. package/dist/trajectory-events.js +119 -0
  34. package/package.json +1 -1
  35. package/schemas/results-v2.schema.json +376 -75
  36. package/schemas/specification-v1.schema.json +128 -174
  37. package/dist/adjudication.d.ts +0 -219
  38. package/dist/adjudication.js +0 -422
  39. package/dist/adoption.d.ts +0 -128
  40. package/dist/adoption.js +0 -102
  41. package/dist/affected.d.ts +0 -88
  42. package/dist/affected.js +0 -222
  43. package/dist/capture.d.ts +0 -193
  44. package/dist/capture.js +0 -344
  45. package/dist/comparison.d.ts +0 -97
  46. package/dist/comparison.js +0 -189
  47. package/dist/factory-calibration.d.ts +0 -63
  48. package/dist/factory-calibration.js +0 -115
  49. package/dist/intervention-results.d.ts +0 -6
  50. package/dist/intervention-results.js +0 -52
  51. package/dist/intervention.d.ts +0 -123
  52. package/dist/intervention.js +0 -218
  53. package/dist/investigation.d.ts +0 -94
  54. package/dist/investigation.js +0 -148
  55. package/dist/judge-agreement.d.ts +0 -19
  56. package/dist/judge-agreement.js +0 -31
  57. package/dist/qualification-capture.d.ts +0 -44
  58. package/dist/qualification-capture.js +0 -163
  59. package/dist/qualification-config.d.ts +0 -233
  60. package/dist/qualification-config.js +0 -694
  61. package/dist/qualification-lock.d.ts +0 -15
  62. package/dist/qualification-lock.js +0 -87
  63. package/dist/qualification-oauth-directory.d.ts +0 -79
  64. package/dist/qualification-oauth-directory.js +0 -444
  65. package/dist/qualification-panel-store.d.ts +0 -14
  66. package/dist/qualification-panel-store.js +0 -188
  67. package/dist/qualification-panels.d.ts +0 -99
  68. package/dist/qualification-panels.js +0 -91
  69. package/dist/qualification-process.d.ts +0 -13
  70. package/dist/qualification-process.js +0 -190
  71. package/dist/qualification-runner.d.ts +0 -203
  72. package/dist/qualification-runner.js +0 -2037
  73. package/dist/qualification-store.d.ts +0 -187
  74. package/dist/qualification-store.js +0 -833
  75. package/dist/qualification-validation.d.ts +0 -12
  76. package/dist/qualification-validation.js +0 -8
  77. package/dist/screen.d.ts +0 -31
  78. package/dist/screen.js +0 -105
  79. package/dist/trajectory-gates.d.ts +0 -137
  80. package/dist/trajectory-gates.js +0 -576
  81. package/dist/trusted-host-supervision.d.ts +0 -25
  82. package/dist/trusted-host-supervision.js +0 -94
  83. package/dist/work-capture-schema.d.ts +0 -38
  84. package/dist/work-capture-schema.js +0 -33
  85. package/dist/work-capture.d.ts +0 -43
  86. package/dist/work-capture.js +0 -81
  87. package/dist/work-signals.d.ts +0 -55
  88. package/dist/work-signals.js +0 -71
  89. package/schemas/qualification-cell-result-v1.schema.json +0 -32
  90. package/schemas/qualification-config-v1.schema.json +0 -244
  91. package/schemas/qualification-invocation-request-v1.schema.json +0 -59
  92. package/schemas/qualification-judge-panel-result-v1.schema.json +0 -42
  93. package/schemas/trajectory-event-v1.schema.json +0 -80
package/dist/spec.js CHANGED
@@ -1,7 +1,6 @@
1
1
  import { readFileSync } from "node:fs";
2
2
  import yaml from "js-yaml";
3
3
  import { parseTraceAssert } from "./trace-gates.js";
4
- import { parseTrajectoryAssert } from "./trajectory-gates.js";
5
4
  /** Thrown on any validation failure. Message always carries the spec file path. */
6
5
  export class SpecError extends Error {
7
6
  constructor(message, file) {
@@ -89,46 +88,6 @@ function resolveExtensions(env, hasSystemPrompt, id, file) {
89
88
  * with empty-git or a fixture — asking for one on a bare cwd is an authoring mistake,
90
89
  * not something to silently ignore.
91
90
  */
92
- function resolveEventSources(env, id, file) {
93
- const raw = env && typeof env === "object" ? env.event_sources : undefined;
94
- if (raw === undefined)
95
- return undefined;
96
- if (!Array.isArray(raw) || raw.length === 0) {
97
- throw new SpecError(`scenario \`${id}\` env.event_sources must be a non-empty list`, file);
98
- }
99
- const allowedAdapters = new Set([
100
- "normalized-v1", "principal-assurance-v1", "pi-daddy-v1", "pi-daddy-ledger-v3",
101
- ]);
102
- return raw.map((entry, index) => {
103
- if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
104
- throw new SpecError(`scenario \`${id}\` env.event_sources[${index}] must be a mapping`, file);
105
- }
106
- const source = entry;
107
- for (const key of Object.keys(source)) {
108
- if (!["adapter", "path", "required"].includes(key)) {
109
- throw new SpecError(`scenario \`${id}\` env.event_sources[${index}] has unknown key \`${key}\``, file);
110
- }
111
- }
112
- if (!allowedAdapters.has(source.adapter)) {
113
- throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].adapter is unsupported`, file);
114
- }
115
- if (typeof source.path !== "string" || !source.path.trim()) {
116
- throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].path must be non-empty`, file);
117
- }
118
- const path = source.path.trim();
119
- if (path.startsWith("/") || /^[A-Za-z]:[\\/]/.test(path) || path.includes("\\") || path.split("/").some((part) => part === ".." || part === "." || part === "")) {
120
- throw new SpecError(`scenario \`${id}\` env.event_sources paths must be workspace-relative without traversal (got \`${path}\`)`, file);
121
- }
122
- if (source.required !== undefined && typeof source.required !== "boolean") {
123
- throw new SpecError(`scenario \`${id}\` env.event_sources[${index}].required must be true or false`, file);
124
- }
125
- return {
126
- adapter: source.adapter,
127
- path,
128
- required: source.required !== false,
129
- };
130
- });
131
- }
132
91
  function resolveRemote(env, workspace, id, file) {
133
92
  const raw = env && typeof env === "object" ? env.remote : undefined;
134
93
  if (raw === undefined)
@@ -220,12 +179,12 @@ export function parseSpec(text, file) {
220
179
  // `assert.trace` is legal for inline AND seeded scenarios — it reads the
221
180
  // execution trace, which every run produces, not a staged diff.
222
181
  const rawAssert = s.assert;
182
+ if (rawAssert?.trajectory !== undefined) {
183
+ throw new SpecError(`scenario \`${id}\` uses removed \`assert.trajectory\`; delete it or replace it with an active objective gate`, file);
184
+ }
223
185
  if (rawAssert?.trace !== undefined) {
224
186
  scenario.traceAssert = parseTraceAssert(rawAssert.trace, `${file}: scenario \`${id}\``);
225
187
  }
226
- if (rawAssert?.trajectory !== undefined) {
227
- scenario.trajectoryAssert = parseTrajectoryAssert(rawAssert.trajectory, `${file}: scenario \`${id}\``);
228
- }
229
188
  if (mode === "seeded") {
230
189
  if (typeof s.fixture !== "string" || s.fixture.length === 0) {
231
190
  throw new SpecError(`seeded scenario \`${id}\` requires a \`fixture\` path`, file);
@@ -276,12 +235,11 @@ export function parseSpec(text, file) {
276
235
  scenario.assert = assertObj;
277
236
  }
278
237
  }
238
+ if (s.env && typeof s.env === "object" && Object.hasOwn(s.env, "event_sources")) {
239
+ throw new SpecError(`scenario \`${id}\` uses removed \`env.event_sources\`; it is no longer collected`, file);
240
+ }
279
241
  scenario.workspace = resolveWorkspace(s.env, mode, scenario.fixture, id, file);
280
242
  scenario.remote = resolveRemote(s.env, scenario.workspace, id, file);
281
- scenario.eventSources = resolveEventSources(s.env, id, file);
282
- if (scenario.eventSources && !scenario.trajectoryAssert) {
283
- throw new SpecError(`scenario \`${id}\` declares env.event_sources without assert.trajectory`, file);
284
- }
285
243
  if (s.system_prompt_file !== undefined) {
286
244
  if (typeof s.system_prompt_file !== "string" || !s.system_prompt_file.trim()) {
287
245
  throw new SpecError(`scenario \`${id}\` \`system_prompt_file\` must be a non-empty string`, file);
@@ -0,0 +1,58 @@
1
+ export declare const LEGACY_TRAJECTORY_EVENT_VERSION: "1.0";
2
+ export declare const TRAJECTORY_EVENT_VERSION: "1.1";
3
+ export type TrajectoryEventVersion = typeof LEGACY_TRAJECTORY_EVENT_VERSION | typeof TRAJECTORY_EVENT_VERSION;
4
+ export interface TrajectoryEventSource {
5
+ adapter: "normalized-v1" | "principal-assurance-v1" | "pi-daddy-v1" | "pi-daddy-ledger-v3";
6
+ path: string;
7
+ required: boolean;
8
+ }
9
+ export interface TrajectoryApproval {
10
+ id?: string;
11
+ capability?: string;
12
+ subject?: string;
13
+ source?: string;
14
+ scope?: string;
15
+ approved_at?: string;
16
+ expires_at?: string;
17
+ used_at?: string;
18
+ }
19
+ export interface TrajectoryDigests {
20
+ plan?: string;
21
+ task?: string;
22
+ definition?: string;
23
+ head?: string;
24
+ tree?: string;
25
+ [key: string]: string | undefined;
26
+ }
27
+ /** Normalized pi-daddy/principal event retained by adapter consumers. */
28
+ export interface TrajectoryEventV1 {
29
+ event_version: TrajectoryEventVersion;
30
+ seq: number;
31
+ type: string;
32
+ source: string;
33
+ at?: string;
34
+ run_id?: string;
35
+ task_id?: string;
36
+ workspace_id?: string;
37
+ context_id?: string;
38
+ finding_id?: string;
39
+ parent_id?: string;
40
+ child_id?: string;
41
+ execution_id?: string;
42
+ parent_execution_id?: string | null;
43
+ task_from_execution_id?: string;
44
+ workflow_fact_id?: string;
45
+ deadline_at?: string;
46
+ phase?: string;
47
+ tool?: string;
48
+ capability?: string;
49
+ requested_capabilities?: string[];
50
+ effective_capabilities?: string[];
51
+ refusal_code?: string;
52
+ exit_code?: number;
53
+ digests?: TrajectoryDigests;
54
+ approval?: TrajectoryApproval;
55
+ requirements?: string[];
56
+ attributes?: Record<string, unknown>;
57
+ }
58
+ export declare function deserializeTrajectoryEvents(text: string): TrajectoryEventV1[] | null;
@@ -0,0 +1,119 @@
1
+ export const LEGACY_TRAJECTORY_EVENT_VERSION = "1.0";
2
+ export const TRAJECTORY_EVENT_VERSION = "1.1";
3
+ export function deserializeTrajectoryEvents(text) {
4
+ const out = [];
5
+ try {
6
+ for (const line of text.split("\n").filter((entry) => entry.trim())) {
7
+ const event = JSON.parse(line);
8
+ if (validateEvent(event) !== null)
9
+ return null;
10
+ out.push(event);
11
+ }
12
+ }
13
+ catch {
14
+ return null;
15
+ }
16
+ if (!out.length)
17
+ return null;
18
+ const sequences = out.map((event) => event.seq).sort((a, b) => a - b);
19
+ if (new Set(sequences).size !== out.length || sequences.some((seq, index) => seq !== index + 1))
20
+ return null;
21
+ return out;
22
+ }
23
+ const V11_EVENT_KEYS = new Set(["execution_id", "parent_execution_id", "task_from_execution_id", "workflow_fact_id", "deadline_at"]);
24
+ const EVENT_KEYS = new Set([
25
+ "event_version", "seq", "type", "source", "at", "run_id", "task_id", "workspace_id", "context_id", "finding_id",
26
+ "parent_id", "child_id", ...V11_EVENT_KEYS,
27
+ "phase", "tool", "capability", "requested_capabilities", "effective_capabilities", "refusal_code",
28
+ "exit_code", "digests", "approval", "requirements", "attributes",
29
+ ]);
30
+ const APPROVAL_KEYS = new Set(["id", "capability", "subject", "source", "scope", "approved_at", "expires_at", "used_at"]);
31
+ const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
32
+ const SHA256_RE = /^[a-fA-F0-9]{64}$/;
33
+ const GIT_SHA_RE = /^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/;
34
+ const REFUSAL_RE = /^[A-Z][A-Z0-9_]*$/;
35
+ function validateEvent(event) {
36
+ if (!event || typeof event !== "object" || Array.isArray(event))
37
+ return "event must be an object";
38
+ const object = event;
39
+ const unknown = Object.keys(object).find((key) => !EVENT_KEYS.has(key));
40
+ if (unknown)
41
+ return `unknown field ${unknown}`;
42
+ if (event.event_version !== LEGACY_TRAJECTORY_EVENT_VERSION && event.event_version !== TRAJECTORY_EVENT_VERSION)
43
+ return `unsupported event_version ${String(event.event_version)}`;
44
+ if (event.event_version === LEGACY_TRAJECTORY_EVENT_VERSION) {
45
+ const versionedField = Object.keys(object).find((key) => V11_EVENT_KEYS.has(key));
46
+ if (versionedField)
47
+ return `${versionedField} requires event_version ${TRAJECTORY_EVENT_VERSION}`;
48
+ }
49
+ if (!Number.isInteger(event.seq) || event.seq < 1)
50
+ return "seq must be a positive integer";
51
+ if (typeof event.type !== "string" || !event.type)
52
+ return "type must be a non-empty string";
53
+ if (typeof event.source !== "string" || !event.source)
54
+ return "source must be a non-empty string";
55
+ if (event.at !== undefined && !validDate(event.at))
56
+ return "at must be an RFC 3339 date-time";
57
+ for (const field of ["run_id", "task_id", "workspace_id", "context_id", "finding_id", "parent_id", "child_id", "execution_id", "task_from_execution_id", "workflow_fact_id"]) {
58
+ if (event[field] !== undefined && (typeof event[field] !== "string" || !ID_RE.test(event[field])))
59
+ return `${field} is not a valid bounded identifier`;
60
+ }
61
+ if (event.parent_execution_id !== undefined && event.parent_execution_id !== null && (typeof event.parent_execution_id !== "string" || !ID_RE.test(event.parent_execution_id)))
62
+ return "parent_execution_id is not a valid bounded identifier or null";
63
+ if (event.deadline_at !== undefined && !validDate(event.deadline_at))
64
+ return "deadline_at must be an RFC 3339 date-time";
65
+ for (const field of ["phase", "tool", "capability"])
66
+ if (event[field] !== undefined && (typeof event[field] !== "string" || !event[field]))
67
+ return `${field} must be a non-empty string`;
68
+ for (const field of ["requested_capabilities", "effective_capabilities", "requirements"]) {
69
+ const values = event[field];
70
+ if (values !== undefined && (!Array.isArray(values) || values.some((value) => typeof value !== "string" || !value) || new Set(values).size !== values.length))
71
+ return `${field} must be an array of unique non-empty strings`;
72
+ }
73
+ if (event.refusal_code !== undefined && !REFUSAL_RE.test(event.refusal_code))
74
+ return "refusal_code is invalid";
75
+ if (event.exit_code !== undefined && !Number.isInteger(event.exit_code))
76
+ return "exit_code must be an integer";
77
+ if (event.digests !== undefined) {
78
+ if (!event.digests || typeof event.digests !== "object" || Array.isArray(event.digests))
79
+ return "digests must be an object";
80
+ for (const [key, value] of Object.entries(event.digests)) {
81
+ if (typeof value !== "string")
82
+ return `digests.${key} must be a string`;
83
+ if (["plan", "task", "definition"].includes(key) && !SHA256_RE.test(value))
84
+ return `digests.${key} must be sha256`;
85
+ if (["head", "tree"].includes(key) && !GIT_SHA_RE.test(value))
86
+ return `digests.${key} must be a git object id`;
87
+ }
88
+ }
89
+ if (event.approval !== undefined) {
90
+ if (!event.approval || typeof event.approval !== "object" || Array.isArray(event.approval))
91
+ return "approval must be an object";
92
+ const unknownApproval = Object.keys(event.approval).find((key) => !APPROVAL_KEYS.has(key));
93
+ if (unknownApproval)
94
+ return `approval.${unknownApproval} is unknown`;
95
+ for (const [key, value] of Object.entries(event.approval)) {
96
+ if (typeof value !== "string" || !value)
97
+ return `approval.${key} must be a non-empty string`;
98
+ if (key === "id" && !ID_RE.test(value))
99
+ return "approval.id is invalid";
100
+ if (["approved_at", "expires_at", "used_at"].includes(key) && !validDate(value))
101
+ return `approval.${key} must be an RFC 3339 date-time`;
102
+ }
103
+ }
104
+ if (event.attributes !== undefined && (!event.attributes || typeof event.attributes !== "object" || Array.isArray(event.attributes)))
105
+ return "attributes must be an object";
106
+ return null;
107
+ }
108
+ function validDate(value) {
109
+ if (typeof value !== "string")
110
+ return false;
111
+ const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.exec(value);
112
+ if (!match || !Number.isFinite(Date.parse(value)))
113
+ return false;
114
+ const [, year, month, day, hour, minute, second] = match.map(Number);
115
+ if (month < 1 || month > 12 || hour > 23 || minute > 59 || second > 59)
116
+ return false;
117
+ return day >= 1 && day <= new Date(Date.UTC(year, month, 0)).getUTCDate();
118
+ }
119
+ //# sourceMappingURL=trajectory-events.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@skill-harness/core",
3
- "version": "0.17.0",
3
+ "version": "0.18.0",
4
4
  "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
5
5
  "type": "module",
6
6
  "license": "MIT",