@skill-harness/adapters 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/closed-schema.d.ts +6 -0
- package/dist/closed-schema.js +70 -14
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/pi-daddy-ledger-v2.d.ts +4 -4
- package/dist/pi-daddy-ledger-v2.js +9 -7
- package/dist/pi-daddy-ledger-v3.d.ts +683 -0
- package/dist/pi-daddy-ledger-v3.js +961 -0
- package/dist/pi-json.d.ts +12 -0
- package/dist/pi-json.js +6 -2
- package/dist/pi.js +171 -30
- package/dist/prompt-capture-extension.d.ts +10 -0
- package/dist/prompt-capture-extension.js +26 -0
- package/dist/prompt-provenance.d.ts +22 -0
- package/dist/prompt-provenance.js +146 -0
- package/dist/trajectory.d.ts +15 -9
- package/dist/trajectory.js +375 -55
- package/package.json +3 -3
package/dist/pi-json.d.ts
CHANGED
|
@@ -32,6 +32,11 @@ export interface PiJsonRunOptions {
|
|
|
32
32
|
turn: number;
|
|
33
33
|
changedPaths?: string[];
|
|
34
34
|
homeDir?: string;
|
|
35
|
+
/**
|
|
36
|
+
* Extra env for the subject process, from the arm. `spawn` treats `undefined`
|
|
37
|
+
* as "inherit", so the control arm (which never sets this) is unaffected.
|
|
38
|
+
*/
|
|
39
|
+
env?: NodeJS.ProcessEnv;
|
|
35
40
|
}
|
|
36
41
|
export interface PiJsonRunResult {
|
|
37
42
|
trace: ExecutionTraceV1;
|
|
@@ -39,5 +44,12 @@ export interface PiJsonRunResult {
|
|
|
39
44
|
malformedLines: number;
|
|
40
45
|
code: number | null;
|
|
41
46
|
stderr: string;
|
|
47
|
+
/**
|
|
48
|
+
* Set when a line on the stream carried a provider-side failure diagnostic
|
|
49
|
+
* (auth, transport) rather than the model answering badly. `runStructured`
|
|
50
|
+
* collects this across turns; `run.ts` turns it into ERROR — never a model
|
|
51
|
+
* verdict.
|
|
52
|
+
*/
|
|
53
|
+
providerFailure: string | null;
|
|
42
54
|
}
|
|
43
55
|
export declare function runPiJson(opts: PiJsonRunOptions): Promise<PiJsonRunResult>;
|
package/dist/pi-json.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { createInterface } from "node:readline";
|
|
3
|
-
import { parseTrace } from "@skill-harness/core";
|
|
3
|
+
import { parseTrace, providerFailureFromJsonLine } from "@skill-harness/core";
|
|
4
4
|
/**
|
|
5
5
|
* Run `pi --mode json` and build an execution trace, **streaming**.
|
|
6
6
|
*
|
|
@@ -28,12 +28,14 @@ export function runPiJson(opts) {
|
|
|
28
28
|
return new Promise((resolve, reject) => {
|
|
29
29
|
const child = spawn("pi", opts.args, {
|
|
30
30
|
cwd: opts.cwd,
|
|
31
|
+
env: opts.env,
|
|
31
32
|
// stdin from /dev/null: pi hangs waiting on it otherwise, and a hang in a
|
|
32
33
|
// wave is indistinguishable from a slow model until the timeout fires.
|
|
33
34
|
stdio: ["ignore", "pipe", "pipe"],
|
|
34
35
|
});
|
|
35
36
|
const kept = [];
|
|
36
37
|
let stderr = "";
|
|
38
|
+
let providerFailure = null;
|
|
37
39
|
let settled = false;
|
|
38
40
|
const timer = setTimeout(() => {
|
|
39
41
|
if (settled)
|
|
@@ -58,6 +60,8 @@ export function runPiJson(opts) {
|
|
|
58
60
|
if (SKIPPED_TYPE_RE.test(line))
|
|
59
61
|
return;
|
|
60
62
|
kept.push(line);
|
|
63
|
+
if (providerFailure === null)
|
|
64
|
+
providerFailure = providerFailureFromJsonLine(line);
|
|
61
65
|
});
|
|
62
66
|
child.stderr.on("data", (chunk) => {
|
|
63
67
|
if (stderr.length < MAX_STDERR_CHARS)
|
|
@@ -85,7 +89,7 @@ export function runPiJson(opts) {
|
|
|
85
89
|
changedPaths: opts.changedPaths,
|
|
86
90
|
homeDir: opts.homeDir,
|
|
87
91
|
});
|
|
88
|
-
resolve({ ...parsed, code, stderr: stderr.slice(0, MAX_STDERR_CHARS) });
|
|
92
|
+
resolve({ ...parsed, code, stderr: stderr.slice(0, MAX_STDERR_CHARS), providerFailure });
|
|
89
93
|
});
|
|
90
94
|
});
|
|
91
95
|
}
|
package/dist/pi.js
CHANGED
|
@@ -1,10 +1,88 @@
|
|
|
1
|
-
import { existsSync, mkdtempSync, readFileSync, statSync } from "node:fs";
|
|
1
|
+
import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { tmpdir, homedir } from "node:os";
|
|
3
|
+
import { randomBytes } from "node:crypto";
|
|
3
4
|
import { join, resolve } from "node:path";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
4
6
|
import { runPiJson } from "./pi-json.js";
|
|
5
7
|
import { collectTrajectorySources, normalizePiTraces, resequence } from "./trajectory.js";
|
|
6
|
-
import {
|
|
8
|
+
import { bindPromptObservation, observeProviderPayload, promptCaptureIsTrusted, verifyPromptSummary } from "./prompt-provenance.js";
|
|
9
|
+
import { exec, onPath, envNum, traceSha256, withProviderFailure, splitPromptDoc } from "@skill-harness/core";
|
|
7
10
|
const PI_TIMEOUT_MS = envNum("PI_TIMEOUT_MS", 300_000);
|
|
11
|
+
const PROMPT_CAPTURE_EXTENSION = fileURLToPath(new URL("./prompt-capture-extension.js", import.meta.url));
|
|
12
|
+
function contractFor(req) {
|
|
13
|
+
if (req.systemPromptFile) {
|
|
14
|
+
const raw = readFileSync(req.systemPromptFile, "utf8");
|
|
15
|
+
return { text: raw, raw, mechanism: "system-prompt-file" };
|
|
16
|
+
}
|
|
17
|
+
const raw = readFileSync(join(requireSkillDir(req.skillDir, req.mode), "SKILL.md"), "utf8");
|
|
18
|
+
const body = splitPromptDoc(raw).body;
|
|
19
|
+
if (req.mode === "red")
|
|
20
|
+
return { text: body, raw, mechanism: "none" };
|
|
21
|
+
return { text: body, raw, mechanism: req.mode === "green" ? "pi-skill" : "append-system-prompt" };
|
|
22
|
+
}
|
|
23
|
+
const RUNTIME_INJECTION_ENV = ["NODE_OPTIONS", "NODE_PATH", "LD_PRELOAD", "DYLD_INSERT_LIBRARIES"];
|
|
24
|
+
function hasRuntimeInjection(req) {
|
|
25
|
+
return Boolean(req.armEnv && Object.keys(req.armEnv).length) || RUNTIME_INJECTION_ENV.some(key => Boolean(process.env[key]));
|
|
26
|
+
}
|
|
27
|
+
function captureSetup(req, env, contract, counter) {
|
|
28
|
+
if (!req.onPromptObservation)
|
|
29
|
+
return { env, finish: () => { } };
|
|
30
|
+
// Arbitrary scenario/arm extensions and runtime-injection env execute in Pi's
|
|
31
|
+
// process with full Node authority. Refuse a forgeable positive observation.
|
|
32
|
+
if (!promptCaptureIsTrusted(req.extensions?.length ?? 0, hasRuntimeInjection(req)))
|
|
33
|
+
return { env, finish: () => {
|
|
34
|
+
const empty = observeProviderPayload({}, contract.text, contract.mechanism, counter.value++);
|
|
35
|
+
req.onPromptObservation?.({ ...empty, status: "ERROR", error: "prompt delivery provenance is unauthenticated when subject extensions or runtime-injection env share Pi's process" });
|
|
36
|
+
} };
|
|
37
|
+
const dir = mkdtempSync(join(tmpdir(), "skill-harness-prompt-"));
|
|
38
|
+
const path = join(dir, "observations.jsonl"), contractPath = join(dir, "contract.json");
|
|
39
|
+
const authenticationKey = randomBytes(32).toString("hex");
|
|
40
|
+
writeFileSync(path, "", { mode: 0o600 });
|
|
41
|
+
writeFileSync(contractPath, JSON.stringify({ text: contract.text, mechanism: contract.mechanism, authentication_key: authenticationKey }), { mode: 0o600 });
|
|
42
|
+
const finish = () => {
|
|
43
|
+
try {
|
|
44
|
+
const lines = readFileSync(path, "utf8").split("\n").filter(Boolean);
|
|
45
|
+
const parsed = lines.map(line => { try {
|
|
46
|
+
return JSON.parse(line);
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return null;
|
|
50
|
+
} });
|
|
51
|
+
const records = parsed.slice(0, -1), summary = parsed.at(-1);
|
|
52
|
+
if (!verifyPromptSummary(summary, records.length, authenticationKey)) {
|
|
53
|
+
const empty = observeProviderPayload({}, contract.text, contract.mechanism, counter.value++);
|
|
54
|
+
req.onPromptObservation?.({ ...empty, status: "ERROR", error: "Pi prompt observation log is missing, truncated, replayed, or unauthenticated" });
|
|
55
|
+
}
|
|
56
|
+
else
|
|
57
|
+
records.forEach((record, observerRequestIndex) => {
|
|
58
|
+
req.onPromptObservation?.(bindPromptObservation(record, contract.text, contract.mechanism, counter.value++, authenticationKey, observerRequestIndex));
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
rmSync(dir, { recursive: true, force: true });
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
return { env: { ...(env ?? process.env), SKILL_HARNESS_PROMPT_CAPTURE_FILE: path, SKILL_HARNESS_PROMPT_CONTRACT_FILE: contractPath }, finish };
|
|
66
|
+
}
|
|
67
|
+
function observerFlags(req) {
|
|
68
|
+
return req.onPromptObservation && promptCaptureIsTrusted(req.extensions?.length ?? 0, hasRuntimeInjection(req)) ? ["--extension", PROMPT_CAPTURE_EXTENSION] : [];
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* stderr fragments that mean the provider refused the request, so the run measured
|
|
72
|
+
* nothing about the model. Substring matching on a message pi passes through from
|
|
73
|
+
* the provider — deliberately narrow: a stderr line we cannot classify stays an
|
|
74
|
+
* ordinary non-zero exit, because calling a real model failure "infrastructure"
|
|
75
|
+
* would hide a regression.
|
|
76
|
+
*/
|
|
77
|
+
const PROVIDER_STDERR_SIGNATURES = [
|
|
78
|
+
"invalidated oauth token",
|
|
79
|
+
"invalid_api_key",
|
|
80
|
+
"insufficient_quota",
|
|
81
|
+
];
|
|
82
|
+
function providerStderr(stderr) {
|
|
83
|
+
const hay = stderr.toLowerCase();
|
|
84
|
+
return PROVIDER_STDERR_SIGNATURES.some((sig) => hay.includes(sig)) ? stderr.trim() : null;
|
|
85
|
+
}
|
|
8
86
|
/**
|
|
9
87
|
* Refuse to hand pi a skill dir it will silently ignore.
|
|
10
88
|
*
|
|
@@ -41,14 +119,15 @@ function requireSkillDir(skillDir, mode) {
|
|
|
41
119
|
* made conditional. Both are checked the same way, because the failure being
|
|
42
120
|
* prevented — a skill dir that isn't there — costs a whole wave either way.
|
|
43
121
|
*/
|
|
44
|
-
function skillFlags(mode, skillDir) {
|
|
122
|
+
function skillFlags(mode, skillDir, boundRaw) {
|
|
45
123
|
switch (mode) {
|
|
46
124
|
case "red":
|
|
47
125
|
return ["--no-skills"];
|
|
48
126
|
case "green":
|
|
49
127
|
return ["--skill", requireSkillDir(skillDir, mode)];
|
|
50
128
|
case "force": {
|
|
51
|
-
|
|
129
|
+
requireSkillDir(skillDir, mode);
|
|
130
|
+
const body = boundRaw ?? readFileSync(join(resolve(skillDir), "SKILL.md"), "utf8");
|
|
52
131
|
return ["--no-skills", "--append-system-prompt", body];
|
|
53
132
|
}
|
|
54
133
|
}
|
|
@@ -81,6 +160,7 @@ function header(turnNo, total, text) {
|
|
|
81
160
|
}
|
|
82
161
|
export const piAdapter = {
|
|
83
162
|
name: "pi",
|
|
163
|
+
observesPrompts: true,
|
|
84
164
|
available() {
|
|
85
165
|
return Promise.resolve(onPath("pi"));
|
|
86
166
|
},
|
|
@@ -116,37 +196,74 @@ export const piAdapter = {
|
|
|
116
196
|
"--no-context-files",
|
|
117
197
|
"--no-extensions",
|
|
118
198
|
...extensionFlags(req.extensions),
|
|
199
|
+
...observerFlags(req),
|
|
119
200
|
"--provider",
|
|
120
201
|
req.model.provider,
|
|
121
202
|
"--model",
|
|
122
203
|
req.model.model,
|
|
123
204
|
];
|
|
124
|
-
//
|
|
205
|
+
// Bind bytes once: argv and provenance must describe the same contract even if a file changes mid-run.
|
|
206
|
+
const contract = contractFor(req);
|
|
125
207
|
const flags = req.systemPromptFile
|
|
126
|
-
? ["--no-skills", "--append-system-prompt",
|
|
127
|
-
: skillFlags(req.mode, req.skillDir);
|
|
208
|
+
? ["--no-skills", "--append-system-prompt", contract.raw]
|
|
209
|
+
: skillFlags(req.mode, req.skillDir, contract.raw);
|
|
210
|
+
const requestCounter = { value: 0 };
|
|
128
211
|
const total = req.turns.length;
|
|
129
212
|
const parts = [];
|
|
213
|
+
// The arm's env, merged over the harness's own — undefined (not `process.env`)
|
|
214
|
+
// when there is none, so `exec`'s `env: opts.env ?? process.env` inherits
|
|
215
|
+
// normally and the control arm is unaffected.
|
|
216
|
+
const env = req.armEnv ? { ...process.env, ...req.armEnv } : undefined;
|
|
217
|
+
// Collected across turns and written by `withProviderFailure` into the
|
|
218
|
+
// transcript PREAMBLE at the end, never inline after an assistant turn: the
|
|
219
|
+
// preamble is the only region of the transcript the model provably cannot
|
|
220
|
+
// reach, and a marker anywhere else is forgeable by a model that types the
|
|
221
|
+
// words (which would convert a FAIL into ERROR and mute the judge forever).
|
|
222
|
+
let providerFailure = null;
|
|
130
223
|
if (total === 1) {
|
|
131
224
|
const args = [...flags, ...common, "--no-session", "-p", req.turns[0]];
|
|
132
|
-
const
|
|
225
|
+
const capture = captureSetup(req, env, contract, requestCounter);
|
|
226
|
+
let r;
|
|
227
|
+
try {
|
|
228
|
+
r = await exec("pi", args, { cwd: req.cwd, timeoutMs: PI_TIMEOUT_MS, env: capture.env });
|
|
229
|
+
}
|
|
230
|
+
finally {
|
|
231
|
+
capture.finish();
|
|
232
|
+
}
|
|
133
233
|
parts.push(header(1, 1, req.turns[0]));
|
|
134
234
|
parts.push(`<<< ASSISTANT:\n${r.stdout.trim()}\n`);
|
|
135
|
-
if (r.code !== 0)
|
|
136
|
-
|
|
137
|
-
|
|
235
|
+
if (r.code !== 0) {
|
|
236
|
+
providerFailure = providerStderr(r.stderr);
|
|
237
|
+
if (!providerFailure)
|
|
238
|
+
parts.push(`[pi exited ${r.code}]\n${r.stderr.trim()}\n`);
|
|
239
|
+
}
|
|
240
|
+
return withProviderFailure(parts.join("\n"), providerFailure);
|
|
138
241
|
}
|
|
139
242
|
const session = mkdtempSync(join(tmpdir(), "sc-pi-session-"));
|
|
140
243
|
for (let i = 0; i < total; i++) {
|
|
141
244
|
const turnFlags = i === 0 ? ["--session-dir", session] : ["--session-dir", session, "-c"];
|
|
142
245
|
const args = [...flags, ...common, ...turnFlags, "-p", req.turns[i]];
|
|
143
|
-
const
|
|
246
|
+
const capture = captureSetup(req, env, contract, requestCounter);
|
|
247
|
+
let r;
|
|
248
|
+
try {
|
|
249
|
+
r = await exec("pi", args, { cwd: req.cwd, timeoutMs: PI_TIMEOUT_MS, env: capture.env });
|
|
250
|
+
}
|
|
251
|
+
finally {
|
|
252
|
+
capture.finish();
|
|
253
|
+
}
|
|
144
254
|
parts.push(header(i + 1, total, req.turns[i]));
|
|
145
255
|
parts.push(`<<< ASSISTANT:\n${r.stdout.trim()}\n`);
|
|
146
|
-
if (r.code !== 0)
|
|
147
|
-
|
|
256
|
+
if (r.code !== 0) {
|
|
257
|
+
const provider = providerStderr(r.stderr);
|
|
258
|
+
// First failure wins, same as the structured path: the turns after an
|
|
259
|
+
// outage are downstream of it, not independent evidence.
|
|
260
|
+
if (provider && providerFailure === null)
|
|
261
|
+
providerFailure = provider;
|
|
262
|
+
if (!provider)
|
|
263
|
+
parts.push(`[pi exited ${r.code} on turn ${i + 1}]\n${r.stderr.trim()}\n`);
|
|
264
|
+
}
|
|
148
265
|
}
|
|
149
|
-
return parts.join("\n");
|
|
266
|
+
return withProviderFailure(parts.join("\n"), providerFailure);
|
|
150
267
|
},
|
|
151
268
|
/**
|
|
152
269
|
* Structured run: same flags, same turn loop, plus `--mode json` and a trace
|
|
@@ -165,19 +282,26 @@ export const piAdapter = {
|
|
|
165
282
|
"--no-context-files",
|
|
166
283
|
"--no-extensions",
|
|
167
284
|
...extensionFlags(req.extensions),
|
|
285
|
+
...observerFlags(req),
|
|
168
286
|
"--provider",
|
|
169
287
|
req.model.provider,
|
|
170
288
|
"--model",
|
|
171
289
|
req.model.model,
|
|
172
290
|
];
|
|
291
|
+
const contract = contractFor(req);
|
|
173
292
|
const flags = req.systemPromptFile
|
|
174
|
-
? ["--no-skills", "--append-system-prompt",
|
|
175
|
-
: skillFlags(req.mode, req.skillDir);
|
|
293
|
+
? ["--no-skills", "--append-system-prompt", contract.raw]
|
|
294
|
+
: skillFlags(req.mode, req.skillDir, contract.raw);
|
|
295
|
+
const requestCounter = { value: 0 };
|
|
176
296
|
const piVersion = await this.version();
|
|
177
297
|
const total = req.turns.length;
|
|
178
298
|
const traces = [];
|
|
179
299
|
const parts = [];
|
|
180
300
|
const session = total === 1 ? null : mkdtempSync(join(tmpdir(), "sc-pi-session-"));
|
|
301
|
+
let providerFailure = null;
|
|
302
|
+
// Same merge as `run()`: undefined when the arm carries no env, so `spawn`
|
|
303
|
+
// (which treats `undefined` as "inherit") leaves the control arm untouched.
|
|
304
|
+
const env = req.armEnv ? { ...process.env, ...req.armEnv } : undefined;
|
|
181
305
|
for (let i = 0; i < total; i++) {
|
|
182
306
|
const turnFlags = session === null
|
|
183
307
|
? ["--no-session"]
|
|
@@ -185,18 +309,26 @@ export const piAdapter = {
|
|
|
185
309
|
? ["--session-dir", session]
|
|
186
310
|
: ["--session-dir", session, "-c"];
|
|
187
311
|
const args = [...flags, ...common, "--mode", "json", ...turnFlags, "-p", req.turns[i]];
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
312
|
+
const capture = captureSetup(req, env, contract, requestCounter);
|
|
313
|
+
let r;
|
|
314
|
+
try {
|
|
315
|
+
r = await runPiJson({
|
|
316
|
+
args,
|
|
317
|
+
cwd: req.cwd,
|
|
318
|
+
timeoutMs: PI_TIMEOUT_MS,
|
|
319
|
+
piVersion,
|
|
320
|
+
subject: req.model,
|
|
321
|
+
scenarioId: req.scenarioId ?? "(unknown)",
|
|
322
|
+
mode: req.mode,
|
|
323
|
+
rep: req.rep ?? 0,
|
|
324
|
+
turn: i,
|
|
325
|
+
homeDir: homedir(),
|
|
326
|
+
env: capture.env,
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
finally {
|
|
330
|
+
capture.finish();
|
|
331
|
+
}
|
|
200
332
|
// A stream with no terminal events at all is not evidence of a clean run.
|
|
201
333
|
// Fail loudly here rather than let an empty trace satisfy a `forbid_calls`
|
|
202
334
|
// gate — "the model called nothing" and "we recorded nothing" must not
|
|
@@ -210,6 +342,14 @@ export const piAdapter = {
|
|
|
210
342
|
r.trace.capture_errors = [`pi JSONL contained ${r.malformedLines} malformed line(s); absence-based trace assertions are unsafe`];
|
|
211
343
|
r.trace.trace_sha256 = traceSha256(r.trace);
|
|
212
344
|
}
|
|
345
|
+
// Recorded here, written into the transcript preamble by
|
|
346
|
+
// `withProviderFailure` below — not just returned on `providerFailure`: the
|
|
347
|
+
// artifact on disk is the only thing a later `grade`/`regrade` call ever
|
|
348
|
+
// reads (see `judgeOneRep` in core/regrade.ts), and the structured path
|
|
349
|
+
// exits 0 while carrying the evidence, so a field a re-judge never sees
|
|
350
|
+
// leaves it unrecoverable from the saved transcript.
|
|
351
|
+
if (providerFailure === null && r.providerFailure)
|
|
352
|
+
providerFailure = r.providerFailure;
|
|
213
353
|
traces.push(r.trace);
|
|
214
354
|
parts.push(header(i + 1, total, req.turns[i]));
|
|
215
355
|
parts.push(`<<< ASSISTANT:\n${r.trace.final_text.trim()}\n`);
|
|
@@ -235,10 +375,11 @@ export const piAdapter = {
|
|
|
235
375
|
}
|
|
236
376
|
const eventErrors = [...native.errors, ...chronologyErrors];
|
|
237
377
|
return {
|
|
238
|
-
transcript: parts.join("\n"),
|
|
378
|
+
transcript: withProviderFailure(parts.join("\n"), providerFailure),
|
|
239
379
|
traces,
|
|
240
380
|
events: resequence(combined),
|
|
241
381
|
...(eventErrors.length ? { eventErrors } : {}),
|
|
382
|
+
...(providerFailure ? { providerFailure } : {}),
|
|
242
383
|
};
|
|
243
384
|
},
|
|
244
385
|
/**
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
interface PromptEvent {
|
|
2
|
+
payload: unknown;
|
|
3
|
+
}
|
|
4
|
+
interface MinimalExtensionApi {
|
|
5
|
+
on(name: "before_provider_request", handler: (event: PromptEvent) => void): void;
|
|
6
|
+
on(name: "session_shutdown", handler: () => void): void;
|
|
7
|
+
}
|
|
8
|
+
/** Internal observation extension: hashes the final provider prompt in-process; never writes payload plaintext. */
|
|
9
|
+
export default function promptCapture(pi: MinimalExtensionApi): void;
|
|
10
|
+
export {};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { appendFileSync, readFileSync, rmSync } from "node:fs";
|
|
2
|
+
import { authenticatePromptObservation, authenticatePromptSummary, observeProviderPayload } from "./prompt-provenance.js";
|
|
3
|
+
/** Internal observation extension: hashes the final provider prompt in-process; never writes payload plaintext. */
|
|
4
|
+
export default function promptCapture(pi) {
|
|
5
|
+
const target = process.env.SKILL_HARNESS_PROMPT_CAPTURE_FILE;
|
|
6
|
+
const contractPath = process.env.SKILL_HARNESS_PROMPT_CONTRACT_FILE;
|
|
7
|
+
if (!target || !contractPath)
|
|
8
|
+
throw new Error("prompt capture paths are required");
|
|
9
|
+
const contract = JSON.parse(readFileSync(contractPath, "utf8"));
|
|
10
|
+
// Remove the capability before any subject/tool execution. The observer keeps
|
|
11
|
+
// only closed-over values; child processes cannot discover contract/evidence paths.
|
|
12
|
+
delete process.env.SKILL_HARNESS_PROMPT_CAPTURE_FILE;
|
|
13
|
+
delete process.env.SKILL_HARNESS_PROMPT_CONTRACT_FILE;
|
|
14
|
+
rmSync(contractPath, { force: true });
|
|
15
|
+
let requestIndex = 0;
|
|
16
|
+
pi.on("before_provider_request", (event) => {
|
|
17
|
+
const observation = observeProviderPayload(event.payload, contract.text, contract.mechanism, requestIndex++);
|
|
18
|
+
appendFileSync(target, JSON.stringify(authenticatePromptObservation(observation, contract.authentication_key)) + "\n", { encoding: "utf8", mode: 0o600 });
|
|
19
|
+
});
|
|
20
|
+
// Written after the agent loop, so a tool subprocess can at most corrupt the
|
|
21
|
+
// log into ERROR; it cannot truncate/replay a valid prefix into trusted PASS.
|
|
22
|
+
pi.on("session_shutdown", () => {
|
|
23
|
+
appendFileSync(target, JSON.stringify(authenticatePromptSummary(requestIndex, contract.authentication_key)) + "\n", { encoding: "utf8", mode: 0o600 });
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=prompt-capture-extension.js.map
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { PROMPT_NORMALIZATION_RULE, type PromptMechanism, type PromptProvenance } from "@skill-harness/core";
|
|
2
|
+
export { PROMPT_NORMALIZATION_RULE } from "@skill-harness/core";
|
|
3
|
+
/** Registry entry 1: replace exactly Pi's dynamic cwd line, preserving every other byte. */
|
|
4
|
+
export declare function normalizePromptPayload(value: unknown, rule: typeof PROMPT_NORMALIZATION_RULE): unknown;
|
|
5
|
+
export declare function promptCaptureIsTrusted(extensionCount: number, hasRuntimeInjection?: boolean): boolean;
|
|
6
|
+
export interface AuthenticatedPromptObservation {
|
|
7
|
+
observation: PromptProvenance;
|
|
8
|
+
mac: string;
|
|
9
|
+
}
|
|
10
|
+
export interface AuthenticatedPromptSummary {
|
|
11
|
+
summary: {
|
|
12
|
+
count: number;
|
|
13
|
+
};
|
|
14
|
+
mac: string;
|
|
15
|
+
}
|
|
16
|
+
export declare function authenticatePromptObservation(observation: PromptProvenance, authenticationKey: string): AuthenticatedPromptObservation;
|
|
17
|
+
export declare function authenticatePromptSummary(count: number, authenticationKey: string): AuthenticatedPromptSummary;
|
|
18
|
+
export declare function verifyPromptSummary(value: unknown, expectedCount: number, authenticationKey: string): boolean;
|
|
19
|
+
/** Compute observation from final provider payload prompt bytes; caller supplies content, never status. */
|
|
20
|
+
export declare function observeProviderPayload(payload: unknown, contract: string, mechanism: PromptMechanism, requestIndex: number): PromptProvenance;
|
|
21
|
+
/** Parent-side binding: never accept contract identity or status from the child observer. */
|
|
22
|
+
export declare function bindPromptObservation(value: unknown, contract: string, mechanism: PromptMechanism, requestIndex: number, authenticationKey: string, observerRequestIndex?: number): PromptProvenance;
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { createHash, createHmac, timingSafeEqual } from "node:crypto";
|
|
2
|
+
import { PROMPT_NORMALIZATION_FLAGS, PROMPT_NORMALIZATION_PATTERN, PROMPT_NORMALIZATION_REPLACEMENT, PROMPT_NORMALIZATION_RULE } from "@skill-harness/core";
|
|
3
|
+
export { PROMPT_NORMALIZATION_RULE } from "@skill-harness/core";
|
|
4
|
+
function sha(bytes) {
|
|
5
|
+
return createHash("sha256").update(bytes, "utf8").digest("hex");
|
|
6
|
+
}
|
|
7
|
+
/** Registry entry 1: replace exactly Pi's dynamic cwd line, preserving every other byte. */
|
|
8
|
+
export function normalizePromptPayload(value, rule) {
|
|
9
|
+
if (rule !== PROMPT_NORMALIZATION_RULE)
|
|
10
|
+
throw new Error(`unknown prompt normalization rule ${String(rule)}`);
|
|
11
|
+
if (typeof value === "string")
|
|
12
|
+
return value.replace(new RegExp(PROMPT_NORMALIZATION_PATTERN, PROMPT_NORMALIZATION_FLAGS), PROMPT_NORMALIZATION_REPLACEMENT);
|
|
13
|
+
if (Array.isArray(value))
|
|
14
|
+
return value.map((entry) => normalizePromptPayload(entry, rule));
|
|
15
|
+
if (value && typeof value === "object")
|
|
16
|
+
return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, normalizePromptPayload(entry, rule)]));
|
|
17
|
+
return value;
|
|
18
|
+
}
|
|
19
|
+
function countInStrings(value, needle) {
|
|
20
|
+
if (typeof value === "string") {
|
|
21
|
+
if (!needle)
|
|
22
|
+
return 0;
|
|
23
|
+
let count = 0, at = 0;
|
|
24
|
+
while ((at = value.indexOf(needle, at)) !== -1) {
|
|
25
|
+
count++;
|
|
26
|
+
at += needle.length;
|
|
27
|
+
}
|
|
28
|
+
return count;
|
|
29
|
+
}
|
|
30
|
+
if (Array.isArray(value))
|
|
31
|
+
return value.reduce((sum, entry) => sum + countInStrings(entry, needle), 0);
|
|
32
|
+
if (value && typeof value === "object") {
|
|
33
|
+
const record = value;
|
|
34
|
+
// A scenario may quote the contract. User-authored text is stimulus, never
|
|
35
|
+
// delivery, but providers also place tool results inside role=user messages.
|
|
36
|
+
if (record.role === "user") {
|
|
37
|
+
const content = Array.isArray(record.content) ? record.content : Array.isArray(record.parts) ? record.parts : [];
|
|
38
|
+
return content
|
|
39
|
+
.filter(block => block && typeof block === "object" && (["tool_result", "tool_response", "function_response"].includes(String(block.type ?? "")) ||
|
|
40
|
+
"functionResponse" in block || "function_response" in block))
|
|
41
|
+
.reduce((sum, block) => sum + countInStrings(block, needle), 0);
|
|
42
|
+
}
|
|
43
|
+
return Object.values(record).reduce((sum, entry) => sum + countInStrings(entry, needle), 0);
|
|
44
|
+
}
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
const PROMPT_FIELDS = new Set(["instructions", "input", "system", "messages", "prompt", "contents"]);
|
|
48
|
+
function promptProjection(payload) {
|
|
49
|
+
if (!payload || typeof payload !== "object" || Array.isArray(payload))
|
|
50
|
+
return null;
|
|
51
|
+
const record = payload;
|
|
52
|
+
const projected = Object.fromEntries(Object.entries(record).filter(([key]) => PROMPT_FIELDS.has(key)));
|
|
53
|
+
const config = record.config && typeof record.config === "object" ? record.config : null;
|
|
54
|
+
if (config?.systemInstruction !== undefined)
|
|
55
|
+
projected.systemInstruction = config.systemInstruction;
|
|
56
|
+
return Object.keys(projected).length ? projected : null;
|
|
57
|
+
}
|
|
58
|
+
export function promptCaptureIsTrusted(extensionCount, hasRuntimeInjection = false) {
|
|
59
|
+
return extensionCount === 0 && !hasRuntimeInjection;
|
|
60
|
+
}
|
|
61
|
+
function observationMac(observation, authenticationKey) {
|
|
62
|
+
return createHmac("sha256", authenticationKey).update(JSON.stringify(observation)).digest("hex");
|
|
63
|
+
}
|
|
64
|
+
export function authenticatePromptObservation(observation, authenticationKey) {
|
|
65
|
+
return { observation, mac: observationMac(observation, authenticationKey) };
|
|
66
|
+
}
|
|
67
|
+
export function authenticatePromptSummary(count, authenticationKey) {
|
|
68
|
+
const summary = { count };
|
|
69
|
+
return { summary, mac: createHmac("sha256", authenticationKey).update(JSON.stringify(summary)).digest("hex") };
|
|
70
|
+
}
|
|
71
|
+
function validMac(supplied, expected) {
|
|
72
|
+
if (typeof supplied !== "string" || !/^[a-f0-9]{64}$/i.test(supplied))
|
|
73
|
+
return false;
|
|
74
|
+
const suppliedBytes = Buffer.from(supplied, "hex"), expectedBytes = Buffer.from(expected, "hex");
|
|
75
|
+
return suppliedBytes.length === expectedBytes.length && timingSafeEqual(suppliedBytes, expectedBytes);
|
|
76
|
+
}
|
|
77
|
+
export function verifyPromptSummary(value, expectedCount, authenticationKey) {
|
|
78
|
+
if (!value || typeof value !== "object")
|
|
79
|
+
return false;
|
|
80
|
+
const envelope = value;
|
|
81
|
+
if (!envelope.summary || envelope.summary.count !== expectedCount)
|
|
82
|
+
return false;
|
|
83
|
+
const expected = createHmac("sha256", authenticationKey).update(JSON.stringify(envelope.summary)).digest("hex");
|
|
84
|
+
return validMac(envelope.mac, expected);
|
|
85
|
+
}
|
|
86
|
+
function statusFor(occurrences, mechanism, observable) {
|
|
87
|
+
if (!observable)
|
|
88
|
+
return "ERROR";
|
|
89
|
+
return occurrences === (mechanism === "none" ? 0 : 1) ? "PASS" : "NOT-MEASURED";
|
|
90
|
+
}
|
|
91
|
+
/** Compute observation from final provider payload prompt bytes; caller supplies content, never status. */
|
|
92
|
+
export function observeProviderPayload(payload, contract, mechanism, requestIndex) {
|
|
93
|
+
const projection = promptProjection(payload);
|
|
94
|
+
const raw = JSON.stringify(projection ?? {});
|
|
95
|
+
const normalized = JSON.stringify(normalizePromptPayload(projection ?? {}, PROMPT_NORMALIZATION_RULE));
|
|
96
|
+
const occurrences = countInStrings(projection ?? {}, contract);
|
|
97
|
+
const observable = projection !== null;
|
|
98
|
+
return {
|
|
99
|
+
capture_version: "prompt-provenance-v1",
|
|
100
|
+
request_index: requestIndex,
|
|
101
|
+
raw_sha256: sha(raw),
|
|
102
|
+
normalized_sha256: sha(normalized),
|
|
103
|
+
normalization_rule: PROMPT_NORMALIZATION_RULE,
|
|
104
|
+
bytes: Buffer.byteLength(raw),
|
|
105
|
+
contract_sha256: sha(contract),
|
|
106
|
+
contract_bytes: Buffer.byteLength(contract),
|
|
107
|
+
contract_occurrences: occurrences,
|
|
108
|
+
mechanism,
|
|
109
|
+
status: statusFor(occurrences, mechanism, observable),
|
|
110
|
+
...(observable ? {} : { error: "provider payload has no supported model-visible prompt field" }),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
/** Parent-side binding: never accept contract identity or status from the child observer. */
|
|
114
|
+
export function bindPromptObservation(value, contract, mechanism, requestIndex, authenticationKey, observerRequestIndex = requestIndex) {
|
|
115
|
+
const fallback = { ...observeProviderPayload({}, contract, mechanism, requestIndex), status: "ERROR" };
|
|
116
|
+
if (!value || typeof value !== "object")
|
|
117
|
+
return { ...fallback, error: "prompt observer emitted a non-object record" };
|
|
118
|
+
const envelope = value;
|
|
119
|
+
if (!envelope.observation)
|
|
120
|
+
return { ...fallback, error: "prompt observation authentication missing" };
|
|
121
|
+
const expectedMac = observationMac(envelope.observation, authenticationKey);
|
|
122
|
+
if (!validMac(envelope.mac, expectedMac))
|
|
123
|
+
return { ...fallback, error: "prompt observation authentication failed" };
|
|
124
|
+
const record = envelope.observation;
|
|
125
|
+
if (record.request_index !== observerRequestIndex)
|
|
126
|
+
return { ...fallback, error: "prompt observation replay or ordering mismatch" };
|
|
127
|
+
const expectedDigest = sha(contract), expectedBytes = Buffer.byteLength(contract);
|
|
128
|
+
const valid = record.capture_version === "prompt-provenance-v1" &&
|
|
129
|
+
record.normalization_rule === PROMPT_NORMALIZATION_RULE && record.mechanism === mechanism &&
|
|
130
|
+
record.contract_sha256 === expectedDigest && record.contract_bytes === expectedBytes &&
|
|
131
|
+
typeof record.raw_sha256 === "string" && /^[a-f0-9]{64}$/i.test(record.raw_sha256) &&
|
|
132
|
+
typeof record.normalized_sha256 === "string" && /^[a-f0-9]{64}$/i.test(record.normalized_sha256) &&
|
|
133
|
+
Number.isInteger(record.bytes) && record.bytes >= 0 &&
|
|
134
|
+
Number.isInteger(record.contract_occurrences) && record.contract_occurrences >= 0;
|
|
135
|
+
if (!valid)
|
|
136
|
+
return { ...fallback, error: "prompt observation failed parent contract/provenance binding" };
|
|
137
|
+
const occurrences = record.contract_occurrences;
|
|
138
|
+
return {
|
|
139
|
+
capture_version: "prompt-provenance-v1", request_index: requestIndex,
|
|
140
|
+
raw_sha256: record.raw_sha256, normalized_sha256: record.normalized_sha256,
|
|
141
|
+
normalization_rule: PROMPT_NORMALIZATION_RULE, bytes: record.bytes,
|
|
142
|
+
contract_sha256: expectedDigest, contract_bytes: expectedBytes, contract_occurrences: occurrences,
|
|
143
|
+
mechanism, status: statusFor(occurrences, mechanism, true),
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
//# sourceMappingURL=prompt-provenance.js.map
|
package/dist/trajectory.d.ts
CHANGED
|
@@ -12,20 +12,26 @@ export declare function normalizePiTraces(traces: ExecutionTraceV1[]): Trajector
|
|
|
12
12
|
/** Normalize principal-pi-skills' immutable assurance event schema v1.0. */
|
|
13
13
|
export declare function normalizePrincipalAssuranceLedger(text: string): TrajectoryEventV1[];
|
|
14
14
|
/**
|
|
15
|
-
* Normalize pi-daddy's public ledgers: unversioned 0.17 GrantRecord lines
|
|
16
|
-
* ledgerVersion 2
|
|
17
|
-
*
|
|
15
|
+
* Normalize pi-daddy's public ledgers: unversioned 0.17 GrantRecord lines,
|
|
16
|
+
* frozen ledgerVersion 2 events, and separately pinned production ledgerVersion
|
|
17
|
+
* 3 events. Explicit version dispatch precedes legacy fallback, so an unknown or
|
|
18
|
+
* malformed new record is never reinterpreted as an old grant.
|
|
18
19
|
*/
|
|
20
|
+
/** Historical selector behavior: unversioned 0.17 and frozen ledger v2 only. */
|
|
21
|
+
export declare function normalizePiDaddyLegacyLedger(text: string): TrajectoryEventV1[];
|
|
22
|
+
/** Separately versioned selector: every line must be an explicit ledgerVersion 3 event. */
|
|
23
|
+
export declare function normalizePiDaddyLedgerV3(text: string): TrajectoryEventV1[];
|
|
24
|
+
/** Programmatic mixed-version dispatcher; source declarations use explicit selectors above. */
|
|
19
25
|
export declare function normalizePiDaddyLedger(text: string): TrajectoryEventV1[];
|
|
20
26
|
/**
|
|
21
|
-
* pi-daddy's canonical refusal vocabulary,
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* exactly how `GRANT_ID_MALFORMED` came to be rejected as "unsupported".
|
|
27
|
+
* pi-daddy's canonical refusal vocabulary, derived directly from the immutable
|
|
28
|
+
* pinned schema. The producer's `src/refusals.ts` is vendored and digest-checked
|
|
29
|
+
* separately, while the real-builder verifier proves its production export is
|
|
30
|
+
* set-equal to this schema-derived runtime set. There is no second list here to
|
|
31
|
+
* forget when pi-daddy adds a contract-valid code.
|
|
27
32
|
*/
|
|
28
33
|
export declare const V2_REFUSAL_CODES: Set<string>;
|
|
34
|
+
export declare const V3_REFUSAL_CODES: Set<string>;
|
|
29
35
|
/**
|
|
30
36
|
* Every vocabulary the adapter restates from the pinned contract, paired with the
|
|
31
37
|
* place in the schema it must equal.
|